package models import ( "encoding/json" "errors" "eta/eta_index_lib/utils" "fmt" "github.com/beego/beego/v2/client/orm" "github.com/dengsgo/math-engine/engine" "github.com/shopspring/decimal" "strings" "time" ) // CalculateRule 预测指标 规则 计算 type CalculateRule struct { EdbInfoId int `description:"指标id"` ConfigId int `description:"配置id"` TrendsCalculateMappingList []*PredictEdbConfCalculateMapping EdbInfoList []*EdbInfo EdbInfoIdBytes []string Formula string RuleType int `description:"预测规则,1:最新,2:固定值,3:同比,4:同差,5:环比,6:环差,7:N期移动均值,8:N期段线性外推值,9:动态环差"` EndDate string `description:"截止日期"` EdbInfoIdArr []EdbInfoFromTag `description:"指标信息"` EmptyType int `description:"空值处理类型(0查找前后35天,1不计算,2前值填充,3后值填充,4等于0)"` MaxEmptyType int `description:"MAX、MIN公式空值处理类型(1、等于0;2、跳过空值)"` } // RefreshCalculateByRuleBy9 刷新计算 func RefreshCalculateByRuleBy9(rule CalculateRule) (resultDataList []*EdbInfoSearchData, err error) { o := orm.NewOrm() to, err := o.Begin() if err != nil { return } defer func() { if err != nil { to.Rollback() } else { err = to.Commit() } }() resultDataList, err = CalculateByRuleBy9(to, rule) return } // CalculateByRuleBy9 动态环差规则计算入库 func CalculateByRuleBy9(to orm.TxOrmer, rule CalculateRule) (resultDataList []*EdbInfoSearchData, err error) { realSaveDataMap := make(map[string]map[int]float64) saveDataMap := make(map[string]map[int]float64) // 最小的结束日期 , 最晚的数据开始日期 var minLatestDate, maxStartDate time.Time dateList := make([]string, 0) // 第一个指标的日期数据 formulaStr := strings.ToUpper(rule.Formula) // 获取关联指标数据 for edbInfoIndex, v := range rule.EdbInfoList { dataList, tmpErr := GetPredictEdbDataListAll(v, 1) if tmpErr != nil { err = tmpErr return } //lenData := len(dataList) for _, dv := range dataList { // 现有实际数据 if val, ok := realSaveDataMap[dv.DataTime]; ok { if _, ok := val[v.EdbInfoId]; !ok { val[v.EdbInfoId] = dv.Value } } else { temp := make(map[int]float64) temp[v.EdbInfoId] = dv.Value realSaveDataMap[dv.DataTime] = temp } // 待处理的数据 if val, ok := saveDataMap[dv.DataTime]; ok { if _, ok := val[v.EdbInfoId]; !ok { val[v.EdbInfoId] = dv.Value } } else { temp := make(map[int]float64) temp[v.EdbInfoId] = dv.Value saveDataMap[dv.DataTime] = temp } // 以第一个指标的日期作为基准日期 if edbInfoIndex == 0 { dateList = append(dateList, dv.DataTime) tmpDate, _ := time.ParseInLocation(utils.FormatDate, dv.DataTime, time.Local) if minLatestDate.IsZero() || tmpDate.After(minLatestDate) { minLatestDate = tmpDate } if maxStartDate.IsZero() || tmpDate.Before(maxStartDate) { maxStartDate = tmpDate } } } /*if lenData > 0 { tmpLatestDate, _ := time.ParseInLocation(utils.FormatDate, dataList[lenData-1].DataTime, time.Local) if minLatestDate.IsZero() || minLatestDate.After(tmpLatestDate) { minLatestDate = tmpLatestDate } tmpStartDate, _ := time.ParseInLocation(utils.FormatDate, dataList[0].DataTime, time.Local) if maxStartDate.IsZero() || maxStartDate.Before(tmpStartDate) { maxStartDate = tmpStartDate } }*/ } // todo 数据处理,将日期内不全的数据做填补 HandleDateSaveDataMap(dateList, maxStartDate, minLatestDate, realSaveDataMap, saveDataMap, rule.EdbInfoList, rule.EmptyType) // 添加数据 addDataList := make([]*PredictEdbRuleData, 0) // 计算规则 formulaDateSlice, formulaDateMap, err := utils.HandleFormulaJson(formulaStr, minLatestDate) if err != nil { return } //获取指标所有数据 dataList := make([]*PredictEdbRuleData, 0) sql := `SELECT * FROM predict_edb_rule_data WHERE config_id = ?` _, err = to.Raw(sql, rule.ConfigId).QueryRows(&dataList) if err != nil { return } dataMap := make(map[string]*PredictEdbRuleData) removeDateMap := make(map[string]*PredictEdbRuleData) //需要移除的日期 for _, v := range dataList { dataMap[v.DataTime] = v removeDateMap[v.DataTime] = v } existDataMap := make(map[string]string) // 判断是否特殊处理max和min函数 maxDealFlag := false if rule.EmptyType == 4 && rule.MaxEmptyType == 2 { maxDealFlag = true } for sk, sv := range saveDataMap { // 当空值处理类型选择了不计算时,只要有一个指标在某个日期没有值(即空值),则计算指标在该日期没有值 if rule.EmptyType == 1 { if len(sv) != len(rule.EdbInfoList) { continue } } //fmt.Println(sk, sv) // 根据时间范围,选择对应的公式 formulaMap := make(map[string]string) formulaStr = "" for _, fv := range formulaDateSlice { if sk < fv { if f, ok := formulaDateMap[fv]; ok { formulaStr = f formulaMap, err = utils.CheckFormula(formulaStr) if err != nil { err = fmt.Errorf("公式错误,请重新填写") return } } break } } if formulaStr == "" { continue } svMax := make(map[int]float64) if maxDealFlag { // 特殊处理max和min函数,如果原本的值为空,则选择空值参与运算 if svMaxData, ok := realSaveDataMap[sk]; ok { svMax = svMaxData } } formulaStr = strings.ToUpper(formulaStr) //fmt.Println(sk, sv) formulaFormStr := ReplaceFormula(rule.EdbInfoList, sv, svMax, formulaMap, formulaStr, rule.EdbInfoIdBytes, maxDealFlag) //计算公式异常,那么就移除该指标 if formulaFormStr == "" { continue } //utils.FileLog.Info(fmt.Sprintf("formulaFormStr:%s", formulaFormStr)) //expression := formula.NewExpression(formulaFormStr) //calResult, tmpErr := expression.Evaluate() //if tmpErr != nil { // // 分母为0的报错 // if strings.Contains(tmpErr.Error(), "divide by zero") { // continue // } // err = errors.New("计算失败:Err:" + tmpErr.Error() + ";formulaStr:" + formulaFormStr) // //fmt.Println(err) // return //} //calVal, tmpErr := calResult.Float64() //if tmpErr != nil { // err = errors.New("计算失败:获取计算值失败 Err:" + tmpErr.Error() + ";formulaStr:" + formulaFormStr) // //fmt.Println(err) // return //} calVal, err := engine.ParseAndExec(formulaFormStr) //calVal, err := calResult.Float64() if err != nil { // 分母为0的报错,忽略该循环 if utils.IsDivideZero(err) { //removeDateList = append(removeDateList, sk) continue } err = errors.New("计算失败:获取计算值失败 Err:" + err.Error() + ";formulaStr:" + formulaFormStr) fmt.Println(err) return nil, err } nanCheck := fmt.Sprintf("%0.f", calVal) if nanCheck == "NaN" || nanCheck == "+Inf" || nanCheck == "-Inf" { continue } // 移除不存在的日期 delete(removeDateMap, sk) saveValue := decimal.NewFromFloat(calVal).Round(4).String() //utils.SubFloatToString(calVal, 4) existPredictEdbRuleData, ok := dataMap[sk] if !ok { dataTime, _ := time.ParseInLocation(utils.FormatDate, sk, time.Local) timestamp := dataTime.UnixNano() / 1e6 if _, existOk := existDataMap[sk]; !existOk { tmpPredictEdbRuleData := &PredictEdbRuleData{ //PredictEdbRuleDataId: 0, EdbInfoId: rule.EdbInfoId, ConfigId: rule.ConfigId, DataTime: sk, Value: saveValue, CreateTime: time.Now(), ModifyTime: time.Now(), DataTimestamp: timestamp, } addDataList = append(addDataList, tmpPredictEdbRuleData) } existDataMap[sk] = sk } else { existValDecimal, tmpErr := decimal.NewFromString(existPredictEdbRuleData.Value) if tmpErr != nil { err = tmpErr return nil, tmpErr } existStr := existValDecimal.String() if existStr != saveValue { existPredictEdbRuleData.Value = saveValue existPredictEdbRuleData.ModifyTime = time.Now() _, err = to.Update(existPredictEdbRuleData, "Value", "ModifyTime") if err != nil { return nil, err } } } // 计算出来的结果集 resultDataList = append(resultDataList, &EdbInfoSearchData{ //EdbDataId: 0, DataTime: sk, Value: calVal, }) } // 添加计算出来的值入库 lenAddDataList := len(addDataList) if lenAddDataList > 0 { _, err = to.InsertMulti(lenAddDataList, addDataList) if err != nil { return } } //删除多余的值 lenRemoveDateList := len(removeDateMap) if lenRemoveDateList > 0 { removeDateList := make([]string, 0) //需要移除的日期 for date, _ := range removeDateMap { removeDateList = append(removeDateList, date) } //如果拼接指标变更了,那么需要删除所有的指标数据 sql := ` DELETE FROM predict_edb_rule_data WHERE config_id = ? and data_time in (` + utils.GetOrmInReplace(lenRemoveDateList) + `) ` _, err = to.Raw(sql, rule.ConfigId, removeDateList).Exec() if err != nil { err = fmt.Errorf("删除计算失败的预测规则计算指标数据失败,Err:" + err.Error()) return } } return } // RefreshCalculateByRuleByLineNh 刷新动态结果计算(线性拟合) func RefreshCalculateByRuleByLineNh(predictEdbInfo EdbInfo, predictEdbConfAndDataList []*PredictEdbConfAndData, rule PredictEdbConf) (err error, errMsg string) { o := orm.NewOrm() to, err := o.Begin() if err != nil { return } defer func() { if err != nil { to.Rollback() } else { err = to.Commit() } }() err, errMsg = CalculateByRuleByRuleLineNh(to, predictEdbInfo, predictEdbConfAndDataList, rule) return } // CalculateByRuleByRuleLineNh 一元线性拟合规则计算入库 func CalculateByRuleByRuleLineNh(to orm.TxOrmer, predictEdbInfo EdbInfo, predictEdbConfAndDataList []*PredictEdbConfAndData, rule PredictEdbConf) (err error, errMsg string) { var secondDataList []*EdbInfoSearchData predictEdbInfoId := predictEdbInfo.EdbInfoId // 预测指标id // 规则 var ruleConf RuleLineNhConf tmpErr := json.Unmarshal([]byte(rule.Value), &ruleConf) if tmpErr != nil { errMsg = `季节性配置信息异常` err = errors.New("季节性配置信息异常:" + tmpErr.Error()) return } // 获取自身指标的数据 { // 来源指标 var sourceEdbInfoItem *EdbInfo sql := ` SELECT * FROM edb_info WHERE edb_info_id=? ` err = to.Raw(sql, rule.SourceEdbInfoId).QueryRow(&sourceEdbInfoItem) if err != nil { return } predictEdbInfo.EdbInfoId = 0 secondDataList, err, _ = GetPredictDataListByPredictEdbConfList(&predictEdbInfo, sourceEdbInfoItem, predictEdbConfAndDataList, 1, ``) if err != nil { return } } lenSecondData := len(secondDataList) if lenSecondData <= 0 { return } newNhccDataMap, err, errMsg := getCalculateNhccData(secondDataList, ruleConf) if err != nil { return } //将最后计算出来的结果数据处理(新增入库、编辑日期的值、删除日期) { // 获取需要预测的日期 startDateStr := secondDataList[lenSecondData-1].DataTime startDate, _ := time.ParseInLocation(utils.FormatDate, startDateStr, time.Local) //endDate, _ := time.ParseInLocation(utils.FormatDate, ruleConf.EndDate, time.Local) endDate := rule.EndDate dayList := getPredictEdbDayList(startDate, endDate, predictEdbInfo.Frequency, predictEdbInfo.DataDateType) if len(dayList) <= 0 { // 如果未来没有日期的话,那么就退出当前循环,进入下一个循环 return } //获取该配置的所有数据 dataList := make([]*PredictEdbRuleData, 0) sql := `SELECT * FROM predict_edb_rule_data WHERE config_id = ?` _, err = to.Raw(sql, rule.ConfigId).QueryRows(&dataList) if err != nil { return } dataMap := make(map[string]*PredictEdbRuleData) for _, v := range dataList { dataMap[v.DataTime] = v } //需要移除的日期 removeDateList := make([]string, 0) // 已经操作过的日期 existDataMap := make(map[string]string) // 添加数据 addDataList := make([]*PredictEdbRuleData, 0) for _, currentDate := range dayList { // 动态拟合残差值数据 currentDateStr := currentDate.Format(utils.FormatDate) val, ok := newNhccDataMap[currentDateStr] // 找不到数据,那么就移除该日期的数据 if !ok { removeDateList = append(removeDateList, currentDateStr) continue } saveValue := decimal.NewFromFloat(val).Round(4).String() //utils.SubFloatToString(calVal, 4) existPredictEdbRuleData, ok := dataMap[currentDateStr] if !ok { timestamp := currentDate.UnixNano() / 1e6 if _, existOk := existDataMap[currentDateStr]; !existOk { tmpPredictEdbRuleData := &PredictEdbRuleData{ //PredictEdbRuleDataId: 0, EdbInfoId: predictEdbInfoId, ConfigId: rule.ConfigId, DataTime: currentDateStr, Value: saveValue, CreateTime: time.Now(), ModifyTime: time.Now(), DataTimestamp: timestamp, } addDataList = append(addDataList, tmpPredictEdbRuleData) } existDataMap[currentDateStr] = currentDateStr } else { existValDecimal, tmpErr := decimal.NewFromString(existPredictEdbRuleData.Value) if tmpErr != nil { err = tmpErr return } existStr := existValDecimal.String() if existStr != saveValue { existPredictEdbRuleData.Value = saveValue existPredictEdbRuleData.ModifyTime = time.Now() _, err = to.Update(existPredictEdbRuleData, "Value", "ModifyTime") if err != nil { return } } } } // 添加计算出来的值入库 lenAddDataList := len(addDataList) if lenAddDataList > 0 { _, err = to.InsertMulti(lenAddDataList, addDataList) if err != nil { return } } //删除多余的值 lenRemoveDateList := len(removeDateList) if lenRemoveDateList > 0 { //如果拼接指标变更了,那么需要删除所有的指标数据 sql := ` DELETE FROM predict_edb_rule_data WHERE config_id = ? and data_time in (` + utils.GetOrmInReplace(lenRemoveDateList) + `) ` _, err = to.Raw(sql, rule.ConfigId, removeDateList).Exec() if err != nil { err = fmt.Errorf("删除计算失败的预测规则计算指标数据失败,Err:" + err.Error()) return } } } return }