package models import ( "errors" "eta_gn/eta_index_lib/global" "eta_gn/eta_index_lib/utils" "fmt" "github.com/shopspring/decimal" "gorm.io/gorm" "math" "strings" "time" ) // EdbDataPredictCalculateNhcc 拟合残差数据结构体 type EdbDataPredictCalculateNhcc struct { EdbDataId int `gorm:"primaryKey;autoIncrement;column:edb_data_id"` EdbInfoId int `gorm:"column:edb_info_id" description:"指标ID"` EdbCode string `gorm:"column:edb_code" description:"指标编码"` DataTime string `gorm:"column:data_time" description:"数据日期"` Value float64 `gorm:"column:value" description:"数据值"` Status int `gorm:"column:status" description:"状态"` CreateTime time.Time `gorm:"column:status" description:"创建时间"` ModifyTime time.Time `gorm:"column:create_time" description:"修改时间"` DataTimestamp int64 `gorm:"column:modify_time" description:"数据日期时间戳"` } // SavePredictCalculateNhcc 新增拟合残差数据 func SavePredictCalculateNhcc(req *EdbInfoCalculateBatchSaveReq, firstEdbInfo, secondEdbInfo *EdbInfo, edbCode, uniqueCode string, nhccDate NhccDate, sysUserId int, sysUserRealName, lang string) (edbInfo *EdbInfo, latestDateStr string, latestValue float64, err error) { to := global.DEFAULT_DmSQL.Begin() defer func() { if err != nil { to.Rollback() } else { to.Commit() } }() var existItemA, existItemB *EdbInfoCalculateMapping if req.EdbInfoId <= 0 { edbInfo = &EdbInfo{ EdbInfoType: 1, SourceName: "预测拟合残差", Source: utils.DATA_SOURCE_PREDICT_CALCULATE_NHCC, EdbCode: edbCode, EdbName: req.EdbName, EdbNameSource: req.EdbName, Frequency: req.Frequency, Unit: req.Unit, StartDate: firstEdbInfo.StartDate, EndDate: firstEdbInfo.EndDate, ClassifyId: req.ClassifyId, SysUserId: sysUserId, SysUserRealName: sysUserRealName, UniqueCode: uniqueCode, CreateTime: time.Now(), ModifyTime: time.Now(), CalculateFormula: req.Formula, EdbNameEn: req.EdbName, UnitEn: req.Unit, EdbType: 2, Sort: GetAddEdbMaxSortByClassifyId(req.ClassifyId, utils.PREDICT_EDB_INFO_TYPE), } tmpErr := to.Create(edbInfo).Error if tmpErr != nil { err = tmpErr return } //第一个指标 { existItemA = &EdbInfoCalculateMapping{ EdbInfoCalculateMappingId: 0, EdbInfoId: edbInfo.EdbInfoId, Source: edbInfo.Source, SourceName: edbInfo.SourceName, EdbCode: edbInfo.EdbCode, FromEdbInfoId: firstEdbInfo.EdbInfoId, FromEdbCode: firstEdbInfo.EdbCode, FromEdbName: firstEdbInfo.EdbName, FromSource: firstEdbInfo.Source, FromSourceName: firstEdbInfo.SourceName, FromTag: "A", MoveValue: req.EdbInfoIdArr[0].MoveValue, Sort: 1, CreateTime: time.Now(), ModifyTime: time.Now(), } tmpErr := to.Create(existItemA).Error if tmpErr != nil { err = tmpErr return } } //第二个指标 { existItemB = &EdbInfoCalculateMapping{ EdbInfoCalculateMappingId: 0, EdbInfoId: edbInfo.EdbInfoId, Source: edbInfo.Source, SourceName: edbInfo.SourceName, EdbCode: edbInfo.EdbCode, FromEdbInfoId: secondEdbInfo.EdbInfoId, FromEdbCode: secondEdbInfo.EdbCode, FromEdbName: secondEdbInfo.EdbName, FromSource: secondEdbInfo.Source, FromSourceName: secondEdbInfo.SourceName, FromTag: "B", MoveValue: req.EdbInfoIdArr[1].MoveValue, Sort: 1, CreateTime: time.Now(), ModifyTime: time.Now(), } tmpErr := to.Create(existItemB).Error if tmpErr != nil { err = tmpErr return } } } else { edbInfo, err = GetEdbInfoById(req.EdbInfoId) if err != nil { return } latestDateStr = edbInfo.LatestDate latestValue = edbInfo.LatestValue nowEdbInfo := *edbInfo // 现在的指标信息 //修改指标信息 switch lang { case utils.EnLangVersion: edbInfo.EdbNameEn = req.EdbName edbInfo.UnitEn = req.Unit default: edbInfo.EdbName = req.EdbName edbInfo.Unit = req.Unit edbInfo.EdbNameSource = req.EdbName } edbInfo.Frequency = req.Frequency edbInfo.ClassifyId = req.ClassifyId edbInfo.CalculateFormula = req.Formula edbInfo.ModifyTime = time.Now() err = to.Model(edbInfo).Select([]string{"EdbName", "EdbNameSource", "Frequency", "Unit", "ClassifyId", "CalculateFormula", "ModifyTime", "EdbNameEn", "UnitEn"}).Updates(edbInfo).Error if err != nil { return } var existCondition string var existPars []interface{} existCondition += " AND edb_info_id=? " existPars = append(existPars, edbInfo.EdbInfoId) //查询出所有的关联指标 var existList []*EdbInfoCalculateMapping existList, err = GetEdbInfoCalculateListByCondition(existCondition, existPars) if err != nil { err = fmt.Errorf("判断指标是否改变失败,Err:" + err.Error()) return } for _, existItem := range existList { if existItem.FromTag == "A" { existItemA = existItem } else if existItem.FromTag == "B" { existItemB = existItem } } if existItemA == nil { err = errors.New("原自变量指标不存在") return } if existItemB == nil { err = errors.New("原因变量指标不存在") return } // 是否需要删除数据重新计算 isNeedCalculateData := false // 如果截止日期变更,那么需要重新计算 if req.Formula != nowEdbInfo.CalculateFormula { isNeedCalculateData = true } var isDeleteA, isDeleteB bool // 如果指标变了,那么需要删除关系 { if existItemA.FromEdbInfoId != firstEdbInfo.EdbInfoId { //删除之前的A指标关联关系 sql := ` DELETE FROM edb_info_calculate_mapping WHERE edb_info_id = ? and from_edb_info_id = ?` err = to.Exec(sql, edbInfo.EdbInfoId, existItemA.FromEdbInfoId).Error if err != nil { err = fmt.Errorf("删除指标A关联关系失败,Err:" + err.Error()) return } isDeleteA = true } //并重新计算 if existItemB.FromEdbInfoId != secondEdbInfo.EdbInfoId { //删除之前的B指标关联关系 sql := ` DELETE FROM edb_info_calculate_mapping WHERE edb_info_id = ? and from_edb_info_id = ?` err = to.Exec(sql, edbInfo.EdbInfoId, existItemB.FromEdbInfoId).Error if err != nil { err = fmt.Errorf("删除指标B关联关系失败,Err:" + err.Error()) return } isDeleteB = true } } //第一个指标数据 { // 如果指标变了,那么需要删除关系,并重新计算 if isDeleteA { //添加新的指标关系 { existItemA = &EdbInfoCalculateMapping{ EdbInfoCalculateMappingId: 0, EdbInfoId: edbInfo.EdbInfoId, Source: edbInfo.Source, SourceName: edbInfo.SourceName, EdbCode: edbInfo.EdbCode, FromEdbInfoId: firstEdbInfo.EdbInfoId, FromEdbCode: firstEdbInfo.EdbCode, FromEdbName: firstEdbInfo.EdbName, FromSource: firstEdbInfo.Source, FromSourceName: firstEdbInfo.SourceName, FromTag: "A", MoveValue: req.EdbInfoIdArr[0].MoveValue, Sort: 1, CreateTime: time.Now(), ModifyTime: time.Now(), } tmpErr := to.Create(existItemA).Error if tmpErr != nil { err = tmpErr return } isNeedCalculateData = true } } else if existItemA.MoveValue != req.EdbInfoIdArr[0].MoveValue { // 如果平移天数不一致的话, existItemA.ModifyTime = time.Now() existItemA.MoveValue = req.EdbInfoIdArr[0].MoveValue //_, err = to.Update(existItemA, "ModifyTime", "MoveValue") err = to.Model(existItemA).Select([]string{"ModifyTime", "MoveValue"}).Updates(existItemA).Error if err != nil { return } isNeedCalculateData = true } } //第二个指标数据 { // 如果指标变了,那么需要删除关系,并重新计算 if isDeleteB { // 添加新的指标关联关系 existItemB = &EdbInfoCalculateMapping{ EdbInfoCalculateMappingId: 0, EdbInfoId: edbInfo.EdbInfoId, Source: edbInfo.Source, SourceName: edbInfo.SourceName, EdbCode: edbInfo.EdbCode, FromEdbInfoId: secondEdbInfo.EdbInfoId, FromEdbCode: secondEdbInfo.EdbCode, FromEdbName: secondEdbInfo.EdbName, FromSource: secondEdbInfo.Source, FromSourceName: secondEdbInfo.SourceName, FromTag: "B", MoveValue: req.EdbInfoIdArr[1].MoveValue, Sort: 2, CreateTime: time.Now(), ModifyTime: time.Now(), } tmpErr := to.Create(existItemB).Error if tmpErr != nil { err = tmpErr return } isNeedCalculateData = true } else if existItemB.MoveValue != req.EdbInfoIdArr[1].MoveValue { // 如果平移天数不一致的话, existItemB.ModifyTime = time.Now() existItemB.MoveValue = req.EdbInfoIdArr[1].MoveValue //_, err = to.Update(existItemB, "ModifyTime", "MoveValue") err = to.Model(existItemB).Select([]string{"ModifyTime", "MoveValue"}).Updates(existItemB).Error if err != nil { return } isNeedCalculateData = true } } // 如果需要重新计算,那么先删除所有的指标数据,然后再重新计算 if isNeedCalculateData { // 删除之前所有的指标数据 tableName := GetEdbDataTableName(edbInfo.Source, edbInfo.SubSource) sql := fmt.Sprintf(` DELETE FROM %s WHERE edb_info_id = ? `, tableName) err = to.Exec(sql, edbInfo.EdbInfoId).Error if err != nil { err = fmt.Errorf("删除历史数据失败,Err:" + err.Error()) return } } else { return } } //拼接数据 latestDateStr, latestValue, err = refreshAllPredictCalculateNhcc(to, edbInfo, firstEdbInfo, secondEdbInfo, existItemA, existItemB, nhccDate) return } // RefreshAllPredictCalculateNhcc 刷新所有 拟合残差 数据 func RefreshAllPredictCalculateNhcc(edbInfo *EdbInfo) (latestDateStr string, latestValue float64, err error) { to := global.DEFAULT_DmSQL.Begin() defer func() { if err != nil { to.Rollback() } else { to.Commit() } }() //查询关联指标信息 var existCondition string var existPars []interface{} existCondition += " AND edb_info_id=? " existPars = append(existPars, edbInfo.EdbInfoId) existList, err := GetEdbInfoCalculateListByCondition(existCondition, existPars) if err != nil { err = fmt.Errorf("判断指标是否改变失败,Err:" + err.Error()) return } var existItemA, existItemB *EdbInfoCalculateMapping for _, existItem := range existList { if existItem.FromTag == "A" { existItemA = existItem } else if existItem.FromTag == "B" { existItemB = existItem } } if existItemA == nil { err = errors.New("原自变量指标不存在") return } if existItemB == nil { err = errors.New("原因变量指标不存在") return } fromEdbInfo, err := GetEdbInfoById(existItemA.FromEdbInfoId) if err != nil { err = fmt.Errorf("GetEdbInfoById Err:" + err.Error()) return } secondEdbInfo, err := GetEdbInfoById(existItemB.FromEdbInfoId) if err != nil { err = fmt.Errorf("GetEdbInfoById Err:" + err.Error()) return } timeList := strings.Split(edbInfo.CalculateFormula, ",") startDate, err := time.ParseInLocation(utils.FormatDate, timeList[0], time.Local) if err != nil { return } endDate, err := time.ParseInLocation(utils.FormatDate, timeList[1], time.Local) if err != nil { return } if endDate.Sub(startDate).Hours() < 48 { return } nhccDate := NhccDate{ StartDate: startDate, EndDate: endDate, } // 刷新数据 latestDateStr, latestValue, err = refreshAllPredictCalculateNhcc(to, edbInfo, fromEdbInfo, secondEdbInfo, existItemA, existItemB, nhccDate) return } // refreshAllPredictCalculateNhcc 刷新所有 拟合残差 数据 func refreshAllPredictCalculateNhcc(to *gorm.DB, edbInfo, firstEdbInfo, secondEdbInfo *EdbInfo, existItemA, existItemB *EdbInfoCalculateMapping, nhccDate NhccDate) (latestDateStr string, latestValue float64, err error) { var dataList []*EdbDataPredictCalculateNhcc sql := ` SELECT * FROM edb_data_predict_calculate_nhcc WHERE edb_info_id=? ORDER BY data_time DESC ` err = to.Raw(sql, edbInfo.EdbInfoId).Scan(&dataList).Error if err != nil { return } // 计算最新的实际值的日期 latestDateA, _ := time.ParseInLocation(utils.FormatDate, firstEdbInfo.LatestDate, time.Local) if existItemA.MoveValue != 0 { latestDateA = latestDateA.AddDate(0, 0, existItemA.MoveValue) } latestDateB, _ := time.ParseInLocation(utils.FormatDate, secondEdbInfo.LatestDate, time.Local) if existItemB.MoveValue != 0 { latestDateB = latestDateA.AddDate(0, 0, existItemB.MoveValue) } if latestDateA.Before(latestDateB) { latestDateStr = latestDateA.Format(utils.FormatDate) } else { latestDateStr = latestDateB.Format(utils.FormatDate) } var dateArr []string dataMap := make(map[string]*EdbDataPredictCalculateNhcc) removeDataTimeMap := make(map[string]int) //需要移除的日期数据 for _, v := range dataList { dateArr = append(dateArr, v.DataTime) dataMap[v.DataTime] = v removeDataTimeMap[v.DataTime] = 1 } addDataList := make([]*EdbDataPredictCalculateNhcc, 0) aDataList, secondDataList, aDataMap, bDataMap, err := getPredictNhccData(firstEdbInfo, secondEdbInfo, existItemA, existItemB, nhccDate) // 计算线性方程公式 var a, b float64 { coordinateData := make([]utils.Coordinate, 0) for i := nhccDate.StartDate; i.Before(nhccDate.EndDate) || i.Equal(nhccDate.EndDate); i = i.AddDate(0, 0, 1) { dateStr := i.Format(utils.FormatDate) xValue, ok := aDataMap[dateStr] if !ok { err = errors.New("指标A日期:" + dateStr + "数据异常,导致计算线性方程公式失败") return } yValue, ok := bDataMap[dateStr] if !ok { err = errors.New("指标B日期:" + dateStr + "数据异常,导致计算线性方程公式失败") return } tmpCoordinate := utils.Coordinate{ X: xValue, Y: yValue, } coordinateData = append(coordinateData, tmpCoordinate) } a, b = utils.GetLinearResult(coordinateData) } if math.IsNaN(a) || math.IsNaN(b) { err = errors.New("线性方程公式生成失败") return } //fmt.Println("a:", a, ";======b:", b) //计算B’ newBDataMap := make(map[string]float64) { //B’=aA+b aDecimal := decimal.NewFromFloat(a) bDecimal := decimal.NewFromFloat(b) for _, aData := range aDataList { xDecimal := decimal.NewFromFloat(aData.Value) val, _ := aDecimal.Mul(xDecimal).Add(bDecimal).Round(4).Float64() newBDataMap[aData.DataTime] = val } } // Delta=B-B‘ for _, bData := range secondDataList { currDate := bData.DataTime //校验待删除日期数据里面是否存在该元素,如果存在的话,那么移除该元素 if _, ok := removeDataTimeMap[currDate]; ok { delete(removeDataTimeMap, currDate) } b2Val, ok := newBDataMap[currDate] if !ok { // 如果B`不存在数据,那么就退出当前循环 continue } bDecimal := decimal.NewFromFloat(bData.Value) b2Decimal := decimal.NewFromFloat(b2Val) val, _ := bDecimal.Sub(b2Decimal).Round(4).Float64() // 判断之前有没有该数据 existData, ok := dataMap[currDate] if !ok { //不存在那么就添加吧 currentDate, _ := time.ParseInLocation(utils.FormatDate, currDate, time.Local) timestamp := currentDate.UnixNano() / 1e6 edbDataNhcc := &EdbDataPredictCalculateNhcc{ EdbInfoId: edbInfo.EdbInfoId, EdbCode: edbInfo.EdbCode, DataTime: currDate, Value: val, Status: 1, CreateTime: time.Now(), ModifyTime: time.Now(), DataTimestamp: timestamp, } addDataList = append(addDataList, edbDataNhcc) } else { // 如果有的话,还需要判断值是否一致,一致则不处理,不一致则修改 if existData.Value != val { existData.Value = val //_, err = to.Update(existData, "Value") err = to.Model(existData).Select([]string{"Value"}).Updates(existData).Error if err != nil { return } } } } //删除已经不存在的拟合残差指标数据(由于该指标当日的数据删除了) { removeDateList := make([]string, 0) for dateTime := range removeDataTimeMap { removeDateList = append(removeDateList, dateTime) } removeNum := len(removeDateList) if removeNum > 0 { //如果拼接指标变更了,那么需要删除所有的指标数据 tableName := GetEdbDataTableName(edbInfo.Source, edbInfo.SubSource) sql := fmt.Sprintf(` DELETE FROM %s WHERE edb_info_id = ? and data_time in (`+utils.GetOrmInReplace(removeNum)+`) `, tableName) err = to.Exec(sql, edbInfo.EdbInfoId, removeDateList).Error if err != nil { err = fmt.Errorf("删除不存在的拟合残差指标数据失败,Err:" + err.Error()) return } } } //数据入库 if len(addDataList) > 0 { tmpAddDataList := make([]*EdbDataPredictCalculateNhcc, 0) i := 0 for _, v := range addDataList { tmpAddDataList = append(tmpAddDataList, v) i++ if i >= 500 { err = to.CreateInBatches(tmpAddDataList, 500).Error if err != nil { return } i = 0 tmpAddDataList = make([]*EdbDataPredictCalculateNhcc, 0) } } if len(tmpAddDataList) > 0 { err = to.CreateInBatches(tmpAddDataList, 500).Error if err != nil { return } } } //确定最终值 var finalLast EdbInfoSearchData sql = ` SELECT data_time , value FROM edb_data_predict_calculate_nhcc WHERE edb_info_id=? and data_time<=? ORDER BY data_time DESC ` tmpErr := to.Raw(sql, edbInfo.EdbInfoId, latestDateStr).First(&finalLast).Error if tmpErr != nil { if tmpErr.Error() != utils.ErrNoRow() { err = tmpErr } return } else { latestDateStr = finalLast.DataTime latestValue = finalLast.Value } return } // CalculatePredictComputeCorrelation 计算相关性结果 func CalculatePredictComputeCorrelation(req *EdbInfoCalculateBatchSaveReq, firstEdbInfo, secondEdbInfo *EdbInfo, nhccDate NhccDate) (r float64, err error) { var existItemA, existItemB *EdbInfoCalculateMapping //第一个指标 { existItemA = &EdbInfoCalculateMapping{ EdbInfoCalculateMappingId: 0, FromEdbInfoId: firstEdbInfo.EdbInfoId, FromEdbCode: firstEdbInfo.EdbCode, FromEdbName: firstEdbInfo.EdbName, FromSource: firstEdbInfo.Source, FromSourceName: firstEdbInfo.SourceName, FromTag: "A", MoveValue: req.EdbInfoIdArr[0].MoveValue, Sort: 1, CreateTime: time.Now(), ModifyTime: time.Now(), } } //第二个指标 { existItemB = &EdbInfoCalculateMapping{ EdbInfoCalculateMappingId: 0, FromEdbInfoId: secondEdbInfo.EdbInfoId, FromEdbCode: secondEdbInfo.EdbCode, FromEdbName: secondEdbInfo.EdbName, FromSource: secondEdbInfo.Source, FromSourceName: secondEdbInfo.SourceName, FromTag: "B", MoveValue: req.EdbInfoIdArr[1].MoveValue, Sort: 1, CreateTime: time.Now(), ModifyTime: time.Now(), } } to := global.DEFAULT_DmSQL.Begin() defer func() { if err != nil { to.Rollback() } else { to.Commit() } }() // 获取相关数据 _, _, aDataMap, bDataMap, err := getPredictNhccData(firstEdbInfo, secondEdbInfo, existItemA, existItemB, nhccDate) // 计算相关性 coordinateData := make([]utils.Coordinate, 0) for i := nhccDate.StartDate; i.Before(nhccDate.EndDate) || i.Equal(nhccDate.EndDate); i = i.AddDate(0, 0, 1) { dateStr := i.Format(utils.FormatDate) xValue, ok := aDataMap[dateStr] if !ok { err = errors.New("指标A日期:" + dateStr + "数据异常,导致计算线性方程公式失败") return } yValue, ok := bDataMap[dateStr] if !ok { err = errors.New("指标B日期:" + dateStr + "数据异常,导致计算线性方程公式失败") return } tmpCoordinate := utils.Coordinate{ X: xValue, Y: yValue, } coordinateData = append(coordinateData, tmpCoordinate) } r = utils.ComputeCorrelation(coordinateData) return } // getNhccData 获取拟合残差需要的数据 func getPredictNhccData(firstEdbInfo, secondEdbInfo *EdbInfo, existItemA, existItemB *EdbInfoCalculateMapping, nhccDate NhccDate) (aDataList []EdbInfoSearchData, secondDataList []*EdbInfoSearchData, aDataMap, bDataMap map[string]float64, err error) { //第一个指标 aDataList = make([]EdbInfoSearchData, 0) aDataMap = make(map[string]float64) { var firstDataList []*EdbInfoSearchData firstDataList, err = GetPredictEdbDataListAllByStartDate(firstEdbInfo, 0, "") if err != nil { return } aDataList, aDataMap = handleNhccData(firstDataList, existItemA.MoveValue) } //第二个指标 bDataList := make([]EdbInfoSearchData, 0) secondDataList = make([]*EdbInfoSearchData, 0) bDataMap = make(map[string]float64) { secondDataList, err = GetPredictEdbDataListAllByStartDate(secondEdbInfo, 0, "") if err != nil { return } bDataList, bDataMap = handleNhccData(secondDataList, existItemB.MoveValue) } if len(aDataList) <= 0 { err = errors.New("指标A没有数据") return } if len(bDataList) <= 0 { err = errors.New("指标B没有数据") return } // 拟合残差计算的结束日期判断 { endAData := aDataList[len(aDataList)-1] tmpEndDate, tmpErr := time.ParseInLocation(utils.FormatDate, endAData.DataTime, time.Local) if tmpErr != nil { err = tmpErr return } // 如果A指标的最新数据日期早于拟合残差的结束日期,那么就用A指标的最新数据日期 if tmpEndDate.Before(nhccDate.EndDate) { nhccDate.EndDate = tmpEndDate } endBData := bDataList[len(bDataList)-1] tmpEndDate, tmpErr = time.ParseInLocation(utils.FormatDate, endBData.DataTime, time.Local) if tmpErr != nil { err = tmpErr return } // 如果B指标的最新数据日期早于拟合残差的结束日期,那么就用B指标的最新数据日期 if tmpEndDate.Before(nhccDate.EndDate) { nhccDate.EndDate = tmpEndDate } } return }