package models import ( "errors" "eta/eta_index_lib/utils" "fmt" "github.com/beego/beego/v2/client/orm" "github.com/shopspring/decimal" "math" "strings" "time" ) // EdbDataPredictCalculateNhcc 拟合残差数据结构体 type EdbDataPredictCalculateNhcc struct { EdbDataId int `orm:"column(edb_data_id);pk"` EdbInfoId int EdbCode string DataTime string Value float64 Status int CreateTime time.Time ModifyTime time.Time DataTimestamp int64 } // SavePredictCalculateNhcc 新增拟合残差数据 func SavePredictCalculateNhcc(req *EdbInfoCalculateBatchSaveReq, firstEdbInfo, secondEdbInfo *EdbInfo, edbCode, uniqueCode string, nhccDate NhccDate, sysUserId int, sysUserRealName, lang string) (edbInfo *EdbInfo, latestDateStr string, latestValue float64, err error) { o := orm.NewOrm() to, err := o.Begin() if err != nil { return } defer func() { if err != nil { fmt.Println("SavePredictCalculateNhcc,Err:" + err.Error()) _ = to.Rollback() } else { _ = to.Commit() } }() var existItemA, existItemB *EdbInfoCalculateMapping if req.EdbInfoId <= 0 { edbInfo = &EdbInfo{ EdbInfoType: 1, SourceName: "预测拟合残差", Source: utils.DATA_SOURCE_PREDICT_CALCULATE_NHCC, EdbCode: edbCode, EdbName: req.EdbName, EdbNameSource: req.EdbName, Frequency: req.Frequency, Unit: req.Unit, StartDate: firstEdbInfo.StartDate, EndDate: firstEdbInfo.EndDate, ClassifyId: req.ClassifyId, SysUserId: sysUserId, SysUserRealName: sysUserRealName, UniqueCode: uniqueCode, CreateTime: time.Now(), ModifyTime: time.Now(), CalculateFormula: req.Formula, EdbNameEn: req.EdbName, UnitEn: req.Unit, EdbType: 2, Sort: GetAddEdbMaxSortByClassifyId(req.ClassifyId, utils.PREDICT_EDB_INFO_TYPE), } newEdbInfoId, tmpErr := to.Insert(edbInfo) if tmpErr != nil { err = tmpErr return } edbInfo.EdbInfoId = int(newEdbInfoId) //第一个指标 { existItemA = &EdbInfoCalculateMapping{ EdbInfoCalculateMappingId: 0, EdbInfoId: edbInfo.EdbInfoId, Source: edbInfo.Source, SourceName: edbInfo.SourceName, EdbCode: edbInfo.EdbCode, FromEdbInfoId: firstEdbInfo.EdbInfoId, FromEdbCode: firstEdbInfo.EdbCode, FromEdbName: firstEdbInfo.EdbName, FromSource: firstEdbInfo.Source, FromSourceName: firstEdbInfo.SourceName, FromTag: "A", MoveValue: req.EdbInfoIdArr[0].MoveValue, Sort: 1, CreateTime: time.Now(), ModifyTime: time.Now(), } insertId, tmpErr := to.Insert(existItemA) if tmpErr != nil { err = tmpErr return } existItemA.EdbInfoCalculateMappingId = int(insertId) } //第二个指标 { existItemB = &EdbInfoCalculateMapping{ EdbInfoCalculateMappingId: 0, EdbInfoId: edbInfo.EdbInfoId, Source: edbInfo.Source, SourceName: edbInfo.SourceName, EdbCode: edbInfo.EdbCode, FromEdbInfoId: secondEdbInfo.EdbInfoId, FromEdbCode: secondEdbInfo.EdbCode, FromEdbName: secondEdbInfo.EdbName, FromSource: secondEdbInfo.Source, FromSourceName: secondEdbInfo.SourceName, FromTag: "B", MoveValue: req.EdbInfoIdArr[1].MoveValue, Sort: 1, CreateTime: time.Now(), ModifyTime: time.Now(), } insertId, tmpErr := to.Insert(existItemB) if tmpErr != nil { err = tmpErr return } existItemB.EdbInfoCalculateMappingId = int(insertId) } } else { edbInfo, err = GetEdbInfoById(req.EdbInfoId) if err != nil { return } latestDateStr = edbInfo.LatestDate latestValue = edbInfo.LatestValue nowEdbInfo := *edbInfo // 现在的指标信息 //修改指标信息 switch lang { case utils.EnLangVersion: edbInfo.EdbNameEn = req.EdbName edbInfo.UnitEn = req.Unit default: edbInfo.EdbName = req.EdbName edbInfo.Unit = req.Unit edbInfo.EdbNameSource = req.EdbName } edbInfo.Frequency = req.Frequency edbInfo.ClassifyId = req.ClassifyId edbInfo.CalculateFormula = req.Formula edbInfo.ModifyTime = time.Now() _, err = to.Update(edbInfo, "EdbName", "EdbNameSource", "Frequency", "Unit", "ClassifyId", "CalculateFormula", "ModifyTime", "EdbNameEn", "UnitEn") if err != nil { return } var existCondition string var existPars []interface{} existCondition += " AND edb_info_id=? " existPars = append(existPars, edbInfo.EdbInfoId) //查询出所有的关联指标 var existList []*EdbInfoCalculateMapping existList, err = GetEdbInfoCalculateListByCondition(existCondition, existPars) if err != nil { err = fmt.Errorf("判断指标是否改变失败,Err:" + err.Error()) return } for _, existItem := range existList { if existItem.FromTag == "A" { existItemA = existItem } else if existItem.FromTag == "B" { existItemB = existItem } } if existItemA == nil { err = errors.New("原自变量指标不存在") return } if existItemB == nil { err = errors.New("原因变量指标不存在") return } // 是否需要删除数据重新计算 isNeedCalculateData := false // 如果截止日期变更,那么需要重新计算 if req.Formula != nowEdbInfo.CalculateFormula { isNeedCalculateData = true } var isDeleteA, isDeleteB bool // 如果指标变了,那么需要删除关系 { if existItemA.FromEdbInfoId != firstEdbInfo.EdbInfoId { //删除之前的A指标关联关系 sql := ` DELETE FROM edb_info_calculate_mapping WHERE edb_info_id = ? and from_edb_info_id = ?` _, err = to.Raw(sql, edbInfo.EdbInfoId, existItemA.FromEdbInfoId).Exec() if err != nil { err = fmt.Errorf("删除指标A关联关系失败,Err:" + err.Error()) return } isDeleteA = true } //并重新计算 if existItemB.FromEdbInfoId != secondEdbInfo.EdbInfoId { //删除之前的B指标关联关系 sql := ` DELETE FROM edb_info_calculate_mapping WHERE edb_info_id = ? and from_edb_info_id = ?` _, err = to.Raw(sql, edbInfo.EdbInfoId, existItemB.FromEdbInfoId).Exec() if err != nil { err = fmt.Errorf("删除指标B关联关系失败,Err:" + err.Error()) return } isDeleteB = true } } //第一个指标数据 { // 如果指标变了,那么需要删除关系,并重新计算 if isDeleteA { //添加新的指标关系 { existItemA = &EdbInfoCalculateMapping{ EdbInfoCalculateMappingId: 0, EdbInfoId: edbInfo.EdbInfoId, Source: edbInfo.Source, SourceName: edbInfo.SourceName, EdbCode: edbInfo.EdbCode, FromEdbInfoId: firstEdbInfo.EdbInfoId, FromEdbCode: firstEdbInfo.EdbCode, FromEdbName: firstEdbInfo.EdbName, FromSource: firstEdbInfo.Source, FromSourceName: firstEdbInfo.SourceName, FromTag: "A", MoveValue: req.EdbInfoIdArr[0].MoveValue, Sort: 1, CreateTime: time.Now(), ModifyTime: time.Now(), } insertId, tmpErr := to.Insert(existItemA) if tmpErr != nil { err = tmpErr return } existItemA.EdbInfoCalculateMappingId = int(insertId) isNeedCalculateData = true } } else if existItemA.MoveValue != req.EdbInfoIdArr[0].MoveValue { // 如果平移天数不一致的话, existItemA.ModifyTime = time.Now() existItemA.MoveValue = req.EdbInfoIdArr[0].MoveValue _, err = to.Update(existItemA, "ModifyTime", "MoveValue") if err != nil { return } isNeedCalculateData = true } } //第二个指标数据 { // 如果指标变了,那么需要删除关系,并重新计算 if isDeleteB { // 添加新的指标关联关系 existItemB = &EdbInfoCalculateMapping{ EdbInfoCalculateMappingId: 0, EdbInfoId: edbInfo.EdbInfoId, Source: edbInfo.Source, SourceName: edbInfo.SourceName, EdbCode: edbInfo.EdbCode, FromEdbInfoId: secondEdbInfo.EdbInfoId, FromEdbCode: secondEdbInfo.EdbCode, FromEdbName: secondEdbInfo.EdbName, FromSource: secondEdbInfo.Source, FromSourceName: secondEdbInfo.SourceName, FromTag: "B", MoveValue: req.EdbInfoIdArr[1].MoveValue, Sort: 2, CreateTime: time.Now(), ModifyTime: time.Now(), } insertId, tmpErr := to.Insert(existItemB) if tmpErr != nil { err = tmpErr return } existItemB.EdbInfoCalculateMappingId = int(insertId) isNeedCalculateData = true } else if existItemB.MoveValue != req.EdbInfoIdArr[1].MoveValue { // 如果平移天数不一致的话, existItemB.ModifyTime = time.Now() existItemB.MoveValue = req.EdbInfoIdArr[1].MoveValue _, err = to.Update(existItemB, "ModifyTime", "MoveValue") if err != nil { return } isNeedCalculateData = true } } // 如果需要重新计算,那么先删除所有的指标数据,然后再重新计算 if isNeedCalculateData { // 删除之前所有的指标数据 tableName := GetEdbDataTableName(edbInfo.Source, edbInfo.SubSource) sql := fmt.Sprintf(` DELETE FROM %s WHERE edb_info_id = ? `, tableName) _, err = to.Raw(sql, edbInfo.EdbInfoId).Exec() if err != nil { err = fmt.Errorf("删除历史数据失败,Err:" + err.Error()) return } } else { return } } //拼接数据 latestDateStr, latestValue, err = refreshAllPredictCalculateNhcc(to, edbInfo, firstEdbInfo, secondEdbInfo, existItemA, existItemB, nhccDate) return } // RefreshAllPredictCalculateNhcc 刷新所有 拟合残差 数据 func RefreshAllPredictCalculateNhcc(edbInfo *EdbInfo) (latestDateStr string, latestValue float64, err error) { o := orm.NewOrm() to, err := o.Begin() defer func() { if err != nil { fmt.Println("RefreshAllPredictCalculateNhcc,Err:" + err.Error()) _ = to.Rollback() } else { _ = to.Commit() } }() //查询关联指标信息 var existCondition string var existPars []interface{} existCondition += " AND edb_info_id=? " existPars = append(existPars, edbInfo.EdbInfoId) existList, err := GetEdbInfoCalculateListByCondition(existCondition, existPars) if err != nil { err = fmt.Errorf("判断指标是否改变失败,Err:" + err.Error()) return } var existItemA, existItemB *EdbInfoCalculateMapping for _, existItem := range existList { if existItem.FromTag == "A" { existItemA = existItem } else if existItem.FromTag == "B" { existItemB = existItem } } if existItemA == nil { err = errors.New("原自变量指标不存在") return } if existItemB == nil { err = errors.New("原因变量指标不存在") return } fromEdbInfo, err := GetEdbInfoById(existItemA.FromEdbInfoId) if err != nil { err = fmt.Errorf("GetEdbInfoById Err:" + err.Error()) return } secondEdbInfo, err := GetEdbInfoById(existItemB.FromEdbInfoId) if err != nil { err = fmt.Errorf("GetEdbInfoById Err:" + err.Error()) return } timeList := strings.Split(edbInfo.CalculateFormula, ",") startDate, err := time.ParseInLocation(utils.FormatDate, timeList[0], time.Local) if err != nil { return } endDate, err := time.ParseInLocation(utils.FormatDate, timeList[1], time.Local) if err != nil { return } if endDate.Sub(startDate).Hours() < 48 { return } nhccDate := NhccDate{ StartDate: startDate, EndDate: endDate, } // 刷新数据 latestDateStr, latestValue, err = refreshAllPredictCalculateNhcc(to, edbInfo, fromEdbInfo, secondEdbInfo, existItemA, existItemB, nhccDate) return } // refreshAllPredictCalculateNhcc 刷新所有 拟合残差 数据 func refreshAllPredictCalculateNhcc(to orm.TxOrmer, edbInfo, firstEdbInfo, secondEdbInfo *EdbInfo, existItemA, existItemB *EdbInfoCalculateMapping, nhccDate NhccDate) (latestDateStr string, latestValue float64, err error) { var dataList []*EdbDataPredictCalculateNhcc sql := ` SELECT * FROM edb_data_predict_calculate_nhcc WHERE edb_info_id=? ORDER BY data_time DESC ` _, err = to.Raw(sql, edbInfo.EdbInfoId).QueryRows(&dataList) if err != nil { return } // 计算最新的实际值的日期 latestDateA, _ := time.ParseInLocation(utils.FormatDate, firstEdbInfo.LatestDate, time.Local) if existItemA.MoveValue != 0 { latestDateA = latestDateA.AddDate(0, 0, existItemA.MoveValue) } latestDateB, _ := time.ParseInLocation(utils.FormatDate, secondEdbInfo.LatestDate, time.Local) if existItemB.MoveValue != 0 { latestDateB = latestDateA.AddDate(0, 0, existItemB.MoveValue) } if latestDateA.Before(latestDateB) { latestDateStr = latestDateA.Format(utils.FormatDate) } else { latestDateStr = latestDateB.Format(utils.FormatDate) } var dateArr []string dataMap := make(map[string]*EdbDataPredictCalculateNhcc) removeDataTimeMap := make(map[string]int) //需要移除的日期数据 for _, v := range dataList { dateArr = append(dateArr, v.DataTime) dataMap[v.DataTime] = v removeDataTimeMap[v.DataTime] = 1 } addDataList := make([]*EdbDataPredictCalculateNhcc, 0) aDataList, secondDataList, aDataMap, bDataMap, err := getPredictNhccData(firstEdbInfo, secondEdbInfo, existItemA, existItemB, nhccDate) // 计算线性方程公式 var a, b float64 { coordinateData := make([]utils.Coordinate, 0) for i := nhccDate.StartDate; i.Before(nhccDate.EndDate) || i.Equal(nhccDate.EndDate); i = i.AddDate(0, 0, 1) { dateStr := i.Format(utils.FormatDate) xValue, ok := aDataMap[dateStr] if !ok { err = errors.New("指标A日期:" + dateStr + "数据异常,导致计算线性方程公式失败") return } yValue, ok := bDataMap[dateStr] if !ok { err = errors.New("指标B日期:" + dateStr + "数据异常,导致计算线性方程公式失败") return } tmpCoordinate := utils.Coordinate{ X: xValue, Y: yValue, } coordinateData = append(coordinateData, tmpCoordinate) } a, b = utils.GetLinearResult(coordinateData) } if math.IsNaN(a) || math.IsNaN(b) { err = errors.New("线性方程公式生成失败") return } //fmt.Println("a:", a, ";======b:", b) //计算B’ newBDataMap := make(map[string]float64) { //B’=aA+b aDecimal := decimal.NewFromFloat(a) bDecimal := decimal.NewFromFloat(b) for _, aData := range aDataList { xDecimal := decimal.NewFromFloat(aData.Value) val, _ := aDecimal.Mul(xDecimal).Add(bDecimal).RoundCeil(4).Float64() newBDataMap[aData.DataTime] = val } } // Delta=B-B‘ for _, bData := range secondDataList { currDate := bData.DataTime //校验待删除日期数据里面是否存在该元素,如果存在的话,那么移除该元素 if _, ok := removeDataTimeMap[currDate]; ok { delete(removeDataTimeMap, currDate) } b2Val, ok := newBDataMap[currDate] if !ok { // 如果B`不存在数据,那么就退出当前循环 continue } bDecimal := decimal.NewFromFloat(bData.Value) b2Decimal := decimal.NewFromFloat(b2Val) val, _ := bDecimal.Sub(b2Decimal).RoundCeil(4).Float64() // 判断之前有没有该数据 existData, ok := dataMap[currDate] if !ok { //不存在那么就添加吧 currentDate, _ := time.ParseInLocation(utils.FormatDate, currDate, time.Local) timestamp := currentDate.UnixNano() / 1e6 edbDataNhcc := &EdbDataPredictCalculateNhcc{ EdbInfoId: edbInfo.EdbInfoId, EdbCode: edbInfo.EdbCode, DataTime: currDate, Value: val, Status: 1, CreateTime: time.Now(), ModifyTime: time.Now(), DataTimestamp: timestamp, } addDataList = append(addDataList, edbDataNhcc) } else { // 如果有的话,还需要判断值是否一致,一致则不处理,不一致则修改 if existData.Value != val { existData.Value = val _, err = to.Update(existData, "Value") if err != nil { return } } } } //删除已经不存在的拟合残差指标数据(由于该指标当日的数据删除了) { removeDateList := make([]string, 0) for dateTime := range removeDataTimeMap { removeDateList = append(removeDateList, dateTime) } removeNum := len(removeDateList) if removeNum > 0 { //如果拼接指标变更了,那么需要删除所有的指标数据 tableName := GetEdbDataTableName(edbInfo.Source, edbInfo.SubSource) sql := fmt.Sprintf(` DELETE FROM %s WHERE edb_info_id = ? and data_time in (`+utils.GetOrmInReplace(removeNum)+`) `, tableName) _, err = to.Raw(sql, edbInfo.EdbInfoId, removeDateList).Exec() if err != nil { err = fmt.Errorf("删除不存在的拟合残差指标数据失败,Err:" + err.Error()) return } } } //数据入库 if len(addDataList) > 0 { tmpAddDataList := make([]*EdbDataPredictCalculateNhcc, 0) i := 0 for _, v := range addDataList { tmpAddDataList = append(tmpAddDataList, v) i++ if i >= 500 { _, err = to.InsertMulti(len(tmpAddDataList), tmpAddDataList) if err != nil { return } i = 0 tmpAddDataList = make([]*EdbDataPredictCalculateNhcc, 0) } } if len(tmpAddDataList) > 0 { _, err = to.InsertMulti(len(tmpAddDataList), tmpAddDataList) if err != nil { return } } } //确定最终值 var finalLast EdbInfoSearchData sql = ` SELECT data_time , value FROM edb_data_predict_calculate_nhcc WHERE edb_info_id=? and data_time<=? ORDER BY data_time DESC ` tmpErr := to.Raw(sql, edbInfo.EdbInfoId, latestDateStr).QueryRow(&finalLast) if tmpErr != nil { if tmpErr.Error() != utils.ErrNoRow() { err = tmpErr } return } else { latestDateStr = finalLast.DataTime latestValue = finalLast.Value } return } // CalculatePredictComputeCorrelation 计算相关性结果 func CalculatePredictComputeCorrelation(req *EdbInfoCalculateBatchSaveReq, firstEdbInfo, secondEdbInfo *EdbInfo, nhccDate NhccDate) (r float64, err error) { var existItemA, existItemB *EdbInfoCalculateMapping //第一个指标 { existItemA = &EdbInfoCalculateMapping{ EdbInfoCalculateMappingId: 0, FromEdbInfoId: firstEdbInfo.EdbInfoId, FromEdbCode: firstEdbInfo.EdbCode, FromEdbName: firstEdbInfo.EdbName, FromSource: firstEdbInfo.Source, FromSourceName: firstEdbInfo.SourceName, FromTag: "A", MoveValue: req.EdbInfoIdArr[0].MoveValue, Sort: 1, CreateTime: time.Now(), ModifyTime: time.Now(), } } //第二个指标 { existItemB = &EdbInfoCalculateMapping{ EdbInfoCalculateMappingId: 0, FromEdbInfoId: secondEdbInfo.EdbInfoId, FromEdbCode: secondEdbInfo.EdbCode, FromEdbName: secondEdbInfo.EdbName, FromSource: secondEdbInfo.Source, FromSourceName: secondEdbInfo.SourceName, FromTag: "B", MoveValue: req.EdbInfoIdArr[1].MoveValue, Sort: 1, CreateTime: time.Now(), ModifyTime: time.Now(), } } o := orm.NewOrm() to, err := o.Begin() defer func() { if err != nil { fmt.Println("RefreshAllCalculateNhcc,Err:" + err.Error()) _ = to.Rollback() } else { _ = to.Commit() } }() // 获取相关数据 _, _, aDataMap, bDataMap, err := getPredictNhccData(firstEdbInfo, secondEdbInfo, existItemA, existItemB, nhccDate) // 计算相关性 coordinateData := make([]utils.Coordinate, 0) for i := nhccDate.StartDate; i.Before(nhccDate.EndDate) || i.Equal(nhccDate.EndDate); i = i.AddDate(0, 0, 1) { dateStr := i.Format(utils.FormatDate) xValue, ok := aDataMap[dateStr] if !ok { err = errors.New("指标A日期:" + dateStr + "数据异常,导致计算线性方程公式失败") return } yValue, ok := bDataMap[dateStr] if !ok { err = errors.New("指标B日期:" + dateStr + "数据异常,导致计算线性方程公式失败") return } tmpCoordinate := utils.Coordinate{ X: xValue, Y: yValue, } coordinateData = append(coordinateData, tmpCoordinate) } r = utils.ComputeCorrelation(coordinateData) return } // getNhccData 获取拟合残差需要的数据 func getPredictNhccData(firstEdbInfo, secondEdbInfo *EdbInfo, existItemA, existItemB *EdbInfoCalculateMapping, nhccDate NhccDate) (aDataList []EdbInfoSearchData, secondDataList []*EdbInfoSearchData, aDataMap, bDataMap map[string]float64, err error) { //第一个指标 aDataList = make([]EdbInfoSearchData, 0) aDataMap = make(map[string]float64) { /*var condition string var pars []interface{} condition += " AND edb_info_id=? " pars = append(pars, existItemA.FromEdbInfoId) //第一个指标的数据列表 firstDataList, tmpErr := GetEdbDataListAllByTo(to, condition, pars, existItemA.FromSource, 0) if tmpErr != nil { return tmpErr }*/ var firstDataList []*EdbInfoSearchData firstDataList, err = GetPredictEdbDataListAllByStartDate(firstEdbInfo, 0, "") if err != nil { return } aDataList, aDataMap = handleNhccData(firstDataList, existItemA.MoveValue) } //第二个指标 bDataList := make([]EdbInfoSearchData, 0) secondDataList = make([]*EdbInfoSearchData, 0) bDataMap = make(map[string]float64) { /*condition = `` pars = make([]interface{}, 0) condition += " AND edb_info_id = ? " pars = append(pars, existItemB.FromEdbInfoId) //第二个指标的数据列表 secondDataList, err = GetEdbDataListAllByTo(to, condition, pars, existItemB.FromSource, 0) if err != nil { return }*/ secondDataList, err = GetPredictEdbDataListAllByStartDate(secondEdbInfo, 0, "") if err != nil { return } bDataList, bDataMap = handleNhccData(secondDataList, existItemB.MoveValue) } if len(aDataList) <= 0 { err = errors.New("指标A没有数据") return } if len(bDataList) <= 0 { err = errors.New("指标B没有数据") return } // 拟合残差计算的结束日期判断 { endAData := aDataList[len(aDataList)-1] tmpEndDate, tmpErr := time.ParseInLocation(utils.FormatDate, endAData.DataTime, time.Local) if tmpErr != nil { err = tmpErr return } // 如果A指标的最新数据日期早于拟合残差的结束日期,那么就用A指标的最新数据日期 if tmpEndDate.Before(nhccDate.EndDate) { nhccDate.EndDate = tmpEndDate } endBData := bDataList[len(bDataList)-1] tmpEndDate, tmpErr = time.ParseInLocation(utils.FormatDate, endBData.DataTime, time.Local) if tmpErr != nil { err = tmpErr return } // 如果B指标的最新数据日期早于拟合残差的结束日期,那么就用B指标的最新数据日期 if tmpEndDate.Before(nhccDate.EndDate) { nhccDate.EndDate = tmpEndDate } } return }