123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466 |
- package models
- import (
- "encoding/json"
- "errors"
- "eta/eta_index_lib/utils"
- "fmt"
- "github.com/beego/beego/v2/client/orm"
- "github.com/dengsgo/math-engine/engine"
- "github.com/shopspring/decimal"
- "strings"
- "time"
- )
- // CalculateRule 预测指标 规则 计算
- type CalculateRule struct {
- EdbInfoId int `description:"指标id"`
- ConfigId int `description:"配置id"`
- TrendsCalculateMappingList []*PredictEdbConfCalculateMapping
- EdbInfoList []*EdbInfo
- EdbInfoIdBytes []string
- Formula string
- RuleType int `description:"预测规则,1:最新,2:固定值,3:同比,4:同差,5:环比,6:环差,7:N期移动均值,8:N期段线性外推值,9:动态环差"`
- EndDate string `description:"截止日期"`
- EdbInfoIdArr []EdbInfoFromTag `description:"指标信息"`
- EmptyType int `description:"空值处理类型(0查找前后35天,1不计算,2前值填充,3后值填充,4等于0)"`
- MaxEmptyType int `description:"MAX、MIN公式空值处理类型(1、等于0;2、跳过空值)"`
- }
- // RefreshCalculateByRuleBy9 刷新计算
- func RefreshCalculateByRuleBy9(rule CalculateRule) (resultDataList []*EdbInfoSearchData, err error) {
- o := orm.NewOrm()
- to, err := o.Begin()
- if err != nil {
- return
- }
- defer func() {
- if err != nil {
- to.Rollback()
- } else {
- err = to.Commit()
- }
- }()
- resultDataList, err = CalculateByRuleBy9(to, rule)
- return
- }
- // CalculateByRuleBy9 动态环差规则计算入库
- func CalculateByRuleBy9(to orm.TxOrmer, rule CalculateRule) (resultDataList []*EdbInfoSearchData, err error) {
- realSaveDataMap := make(map[string]map[int]float64)
- saveDataMap := make(map[string]map[int]float64)
- // 最小的结束日期 , 最晚的数据开始日期
- var minLatestDate, maxStartDate time.Time
- dateList := make([]string, 0) // 第一个指标的日期数据
- formulaStr := strings.ToUpper(rule.Formula)
- // 获取关联指标数据
- for edbInfoIndex, v := range rule.EdbInfoList {
- dataList, tmpErr := GetPredictEdbDataListAll(v, 1)
- if tmpErr != nil {
- err = tmpErr
- return
- }
- //lenData := len(dataList)
- for _, dv := range dataList {
- // 现有实际数据
- if val, ok := realSaveDataMap[dv.DataTime]; ok {
- if _, ok := val[v.EdbInfoId]; !ok {
- val[v.EdbInfoId] = dv.Value
- }
- } else {
- temp := make(map[int]float64)
- temp[v.EdbInfoId] = dv.Value
- realSaveDataMap[dv.DataTime] = temp
- }
- // 待处理的数据
- if val, ok := saveDataMap[dv.DataTime]; ok {
- if _, ok := val[v.EdbInfoId]; !ok {
- val[v.EdbInfoId] = dv.Value
- }
- } else {
- temp := make(map[int]float64)
- temp[v.EdbInfoId] = dv.Value
- saveDataMap[dv.DataTime] = temp
- }
- // 以第一个指标的日期作为基准日期
- if edbInfoIndex == 0 {
- dateList = append(dateList, dv.DataTime)
- tmpDate, _ := time.ParseInLocation(utils.FormatDate, dv.DataTime, time.Local)
- if minLatestDate.IsZero() || tmpDate.After(minLatestDate) {
- minLatestDate = tmpDate
- }
- if maxStartDate.IsZero() || tmpDate.Before(maxStartDate) {
- maxStartDate = tmpDate
- }
- }
- }
- /*if lenData > 0 {
- tmpLatestDate, _ := time.ParseInLocation(utils.FormatDate, dataList[lenData-1].DataTime, time.Local)
- if minLatestDate.IsZero() || minLatestDate.After(tmpLatestDate) {
- minLatestDate = tmpLatestDate
- }
- tmpStartDate, _ := time.ParseInLocation(utils.FormatDate, dataList[0].DataTime, time.Local)
- if maxStartDate.IsZero() || maxStartDate.Before(tmpStartDate) {
- maxStartDate = tmpStartDate
- }
- }*/
- }
- // todo 数据处理,将日期内不全的数据做填补
- HandleDateSaveDataMap(dateList, maxStartDate, minLatestDate, realSaveDataMap, saveDataMap, rule.EdbInfoList, rule.EmptyType)
- // 添加数据
- addDataList := make([]*PredictEdbRuleData, 0)
- // 计算规则
- formulaDateSlice, formulaDateMap, err := utils.HandleFormulaJson(formulaStr, minLatestDate)
- if err != nil {
- return
- }
- //获取指标所有数据
- dataList := make([]*PredictEdbRuleData, 0)
- sql := `SELECT * FROM predict_edb_rule_data WHERE config_id = ?`
- _, err = to.Raw(sql, rule.ConfigId).QueryRows(&dataList)
- if err != nil {
- return
- }
- dataMap := make(map[string]*PredictEdbRuleData)
- removeDateMap := make(map[string]*PredictEdbRuleData) //需要移除的日期
- for _, v := range dataList {
- dataMap[v.DataTime] = v
- removeDateMap[v.DataTime] = v
- }
- existDataMap := make(map[string]string)
- // 判断是否特殊处理max和min函数
- maxDealFlag := false
- if rule.EmptyType == 4 && rule.MaxEmptyType == 2 {
- maxDealFlag = true
- }
- for sk, sv := range saveDataMap {
- // 当空值处理类型选择了不计算时,只要有一个指标在某个日期没有值(即空值),则计算指标在该日期没有值
- if rule.EmptyType == 1 {
- if len(sv) != len(rule.EdbInfoList) {
- continue
- }
- }
- //fmt.Println(sk, sv)
- // 根据时间范围,选择对应的公式
- formulaMap := make(map[string]string)
- formulaStr = ""
- for _, fv := range formulaDateSlice {
- if sk < fv {
- if f, ok := formulaDateMap[fv]; ok {
- formulaStr = f
- formulaMap, err = utils.CheckFormula(formulaStr)
- if err != nil {
- err = fmt.Errorf("公式错误,请重新填写")
- return
- }
- }
- break
- }
- }
- if formulaStr == "" {
- continue
- }
- svMax := make(map[int]float64)
- if maxDealFlag {
- // 特殊处理max和min函数,如果原本的值为空,则选择空值参与运算
- if svMaxData, ok := realSaveDataMap[sk]; ok {
- svMax = svMaxData
- }
- }
- formulaStr = strings.ToUpper(formulaStr)
- //fmt.Println(sk, sv)
- formulaFormStr := ReplaceFormula(rule.EdbInfoList, sv, svMax, formulaMap, formulaStr, rule.EdbInfoIdBytes, maxDealFlag)
- //计算公式异常,那么就移除该指标
- if formulaFormStr == "" {
- continue
- }
- //utils.FileLog.Info(fmt.Sprintf("formulaFormStr:%s", formulaFormStr))
- //expression := formula.NewExpression(formulaFormStr)
- //calResult, tmpErr := expression.Evaluate()
- //if tmpErr != nil {
- // // 分母为0的报错
- // if strings.Contains(tmpErr.Error(), "divide by zero") {
- // continue
- // }
- // err = errors.New("计算失败:Err:" + tmpErr.Error() + ";formulaStr:" + formulaFormStr)
- // //fmt.Println(err)
- // return
- //}
- //calVal, tmpErr := calResult.Float64()
- //if tmpErr != nil {
- // err = errors.New("计算失败:获取计算值失败 Err:" + tmpErr.Error() + ";formulaStr:" + formulaFormStr)
- // //fmt.Println(err)
- // return
- //}
- calVal, err := engine.ParseAndExec(formulaFormStr)
- //calVal, err := calResult.Float64()
- if err != nil {
- // 分母为0的报错,忽略该循环
- if utils.IsDivideZero(err) {
- //removeDateList = append(removeDateList, sk)
- continue
- }
- err = errors.New("计算失败:获取计算值失败 Err:" + err.Error() + ";formulaStr:" + formulaFormStr)
- fmt.Println(err)
- return nil, err
- }
- nanCheck := fmt.Sprintf("%0.f", calVal)
- if nanCheck == "NaN" || nanCheck == "+Inf" || nanCheck == "-Inf" {
- continue
- }
- // 移除不存在的日期
- delete(removeDateMap, sk)
- saveValue := decimal.NewFromFloat(calVal).RoundCeil(4).String() //utils.SubFloatToString(calVal, 4)
- existPredictEdbRuleData, ok := dataMap[sk]
- if !ok {
- dataTime, _ := time.ParseInLocation(utils.FormatDate, sk, time.Local)
- timestamp := dataTime.UnixNano() / 1e6
- if _, existOk := existDataMap[sk]; !existOk {
- tmpPredictEdbRuleData := &PredictEdbRuleData{
- //PredictEdbRuleDataId: 0,
- EdbInfoId: rule.EdbInfoId,
- ConfigId: rule.ConfigId,
- DataTime: sk,
- Value: saveValue,
- CreateTime: time.Now(),
- ModifyTime: time.Now(),
- DataTimestamp: timestamp,
- }
- addDataList = append(addDataList, tmpPredictEdbRuleData)
- }
- existDataMap[sk] = sk
- } else {
- existValDecimal, tmpErr := decimal.NewFromString(existPredictEdbRuleData.Value)
- if tmpErr != nil {
- err = tmpErr
- return nil, tmpErr
- }
- existStr := existValDecimal.String()
- if existStr != saveValue {
- existPredictEdbRuleData.Value = saveValue
- existPredictEdbRuleData.ModifyTime = time.Now()
- _, err = to.Update(existPredictEdbRuleData, "Value", "ModifyTime")
- if err != nil {
- return nil, err
- }
- }
- }
- // 计算出来的结果集
- resultDataList = append(resultDataList, &EdbInfoSearchData{
- //EdbDataId: 0,
- DataTime: sk,
- Value: calVal,
- })
- }
- // 添加计算出来的值入库
- lenAddDataList := len(addDataList)
- if lenAddDataList > 0 {
- _, err = to.InsertMulti(lenAddDataList, addDataList)
- if err != nil {
- return
- }
- }
- //删除多余的值
- lenRemoveDateList := len(removeDateMap)
- if lenRemoveDateList > 0 {
- removeDateList := make([]string, 0) //需要移除的日期
- for date, _ := range removeDateMap {
- removeDateList = append(removeDateList, date)
- }
- //如果拼接指标变更了,那么需要删除所有的指标数据
- sql := ` DELETE FROM predict_edb_rule_data WHERE config_id = ? and data_time in (` + utils.GetOrmInReplace(lenRemoveDateList) + `) `
- _, err = to.Raw(sql, rule.ConfigId, removeDateList).Exec()
- if err != nil {
- err = fmt.Errorf("删除计算失败的预测规则计算指标数据失败,Err:" + err.Error())
- return
- }
- }
- return
- }
- // RefreshCalculateByRuleByLineNh 刷新动态结果计算(线性拟合)
- func RefreshCalculateByRuleByLineNh(predictEdbInfo EdbInfo, predictEdbConfAndDataList []*PredictEdbConfAndData, rule PredictEdbConf) (err error, errMsg string) {
- o := orm.NewOrm()
- to, err := o.Begin()
- if err != nil {
- return
- }
- defer func() {
- if err != nil {
- to.Rollback()
- } else {
- err = to.Commit()
- }
- }()
- err, errMsg = CalculateByRuleByRuleLineNh(to, predictEdbInfo, predictEdbConfAndDataList, rule)
- return
- }
- // CalculateByRuleByRuleLineNh 一元线性拟合规则计算入库
- func CalculateByRuleByRuleLineNh(to orm.TxOrmer, predictEdbInfo EdbInfo, predictEdbConfAndDataList []*PredictEdbConfAndData, rule PredictEdbConf) (err error, errMsg string) {
- var secondDataList []*EdbInfoSearchData
- predictEdbInfoId := predictEdbInfo.EdbInfoId // 预测指标id
- // 规则
- var ruleConf RuleLineNhConf
- tmpErr := json.Unmarshal([]byte(rule.Value), &ruleConf)
- if tmpErr != nil {
- errMsg = `季节性配置信息异常`
- err = errors.New("季节性配置信息异常:" + tmpErr.Error())
- return
- }
- // 获取自身指标的数据
- {
- // 来源指标
- var sourceEdbInfoItem *EdbInfo
- sql := ` SELECT * FROM edb_info WHERE edb_info_id=? `
- err = to.Raw(sql, rule.SourceEdbInfoId).QueryRow(&sourceEdbInfoItem)
- if err != nil {
- return
- }
- predictEdbInfo.EdbInfoId = 0
- secondDataList, err, _ = GetPredictDataListByPredictEdbConfList(&predictEdbInfo, sourceEdbInfoItem, predictEdbConfAndDataList, 1, ``)
- if err != nil {
- return
- }
- }
- lenSecondData := len(secondDataList)
- if lenSecondData <= 0 {
- return
- }
- newNhccDataMap, err, errMsg := getCalculateNhccData(secondDataList, ruleConf)
- if err != nil {
- return
- }
- //将最后计算出来的结果数据处理(新增入库、编辑日期的值、删除日期)
- {
- // 获取需要预测的日期
- startDateStr := secondDataList[lenSecondData-1].DataTime
- startDate, _ := time.ParseInLocation(utils.FormatDate, startDateStr, time.Local)
- //endDate, _ := time.ParseInLocation(utils.FormatDate, ruleConf.EndDate, time.Local)
- endDate := rule.EndDate
- dayList := getPredictEdbDayList(startDate, endDate, predictEdbInfo.Frequency, predictEdbInfo.DataDateType)
- if len(dayList) <= 0 { // 如果未来没有日期的话,那么就退出当前循环,进入下一个循环
- return
- }
- //获取该配置的所有数据
- dataList := make([]*PredictEdbRuleData, 0)
- sql := `SELECT * FROM predict_edb_rule_data WHERE config_id = ?`
- _, err = to.Raw(sql, rule.ConfigId).QueryRows(&dataList)
- if err != nil {
- return
- }
- dataMap := make(map[string]*PredictEdbRuleData)
- for _, v := range dataList {
- dataMap[v.DataTime] = v
- }
- //需要移除的日期
- removeDateList := make([]string, 0)
- // 已经操作过的日期
- existDataMap := make(map[string]string)
- // 添加数据
- addDataList := make([]*PredictEdbRuleData, 0)
- for _, currentDate := range dayList {
- // 动态拟合残差值数据
- currentDateStr := currentDate.Format(utils.FormatDate)
- val, ok := newNhccDataMap[currentDateStr]
- // 找不到数据,那么就移除该日期的数据
- if !ok {
- removeDateList = append(removeDateList, currentDateStr)
- continue
- }
- saveValue := decimal.NewFromFloat(val).RoundCeil(4).String() //utils.SubFloatToString(calVal, 4)
- existPredictEdbRuleData, ok := dataMap[currentDateStr]
- if !ok {
- timestamp := currentDate.UnixNano() / 1e6
- if _, existOk := existDataMap[currentDateStr]; !existOk {
- tmpPredictEdbRuleData := &PredictEdbRuleData{
- //PredictEdbRuleDataId: 0,
- EdbInfoId: predictEdbInfoId,
- ConfigId: rule.ConfigId,
- DataTime: currentDateStr,
- Value: saveValue,
- CreateTime: time.Now(),
- ModifyTime: time.Now(),
- DataTimestamp: timestamp,
- }
- addDataList = append(addDataList, tmpPredictEdbRuleData)
- }
- existDataMap[currentDateStr] = currentDateStr
- } else {
- existValDecimal, tmpErr := decimal.NewFromString(existPredictEdbRuleData.Value)
- if tmpErr != nil {
- err = tmpErr
- return
- }
- existStr := existValDecimal.String()
- if existStr != saveValue {
- existPredictEdbRuleData.Value = saveValue
- existPredictEdbRuleData.ModifyTime = time.Now()
- _, err = to.Update(existPredictEdbRuleData, "Value", "ModifyTime")
- if err != nil {
- return
- }
- }
- }
- }
- // 添加计算出来的值入库
- lenAddDataList := len(addDataList)
- if lenAddDataList > 0 {
- _, err = to.InsertMulti(lenAddDataList, addDataList)
- if err != nil {
- return
- }
- }
- //删除多余的值
- lenRemoveDateList := len(removeDateList)
- if lenRemoveDateList > 0 {
- //如果拼接指标变更了,那么需要删除所有的指标数据
- sql := ` DELETE FROM predict_edb_rule_data WHERE config_id = ? and data_time in (` + utils.GetOrmInReplace(lenRemoveDateList) + `) `
- _, err = to.Raw(sql, rule.ConfigId, removeDateList).Exec()
- if err != nil {
- err = fmt.Errorf("删除计算失败的预测规则计算指标数据失败,Err:" + err.Error())
- return
- }
- }
- }
- return
- }
|