eta_index_lib/models/predict_edb.go

package models

import (
	"encoding/json"
	"errors"
	"eta/eta_index_lib/global"
	"eta/eta_index_lib/utils"
	"fmt"
	"github.com/dengsgo/math-engine/engine"
	"github.com/shopspring/decimal"
	"gorm.io/gorm"
	"strings"
	"time"
)

// CalculateRule 预测指标 规则 计算
type CalculateRule struct {
	EdbInfoId                  int                               `description:"指标id"`
	ConfigId                   int                               `description:"配置id"`
	TrendsCalculateMappingList []*PredictEdbConfCalculateMapping `gorm:"-"`
	EdbInfoList                []*EdbInfo                        `gorm:"-"`
	EdbInfoIdBytes             []string
	Formula                    string
	RuleType                   int              `description:"预测规则，1：最新，2：固定值，3：同比，4：同差，5：环比，6：环差，7：N期移动均值，8：N期段线性外推值，9：动态环差"`
	EndDate                    string           `description:"截止日期"`
	EdbInfoIdArr               []EdbInfoFromTag `description:"指标信息" gorm:"-"`
	EmptyType                  int              `description:"空值处理类型（0查找前后35天，1不计算，2前值填充，3后值填充，4等于0）"`
	MaxEmptyType               int              `description:"MAX、MIN公式空值处理类型（1、等于0；2、跳过空值）"`
}

// AfterFind 在该模型上设置钩子函数，把日期转成正确的string，所以查询函数只能用Find函数，First或者Scan是不会触发该函数的来获取数据
func (m *CalculateRule) AfterFind(db *gorm.DB) (err error) {
	m.EndDate = utils.GormDateStrToDateStr(m.EndDate)

	return
}

// RefreshCalculateByRuleBy9 刷新计算
func RefreshCalculateByRuleBy9(rule CalculateRule) (resultDataList []*EdbInfoSearchData, err error) {
	to := global.DEFAULT_DB.Begin()
	defer func() {
		if err != nil {
			to.Rollback()
		} else {
			to.Commit()
		}
	}()

	resultDataList, err = CalculateByRuleBy9(to, rule)

	return
}

// CalculateByRuleBy9 动态环差规则计算入库
func CalculateByRuleBy9(to *gorm.DB, rule CalculateRule) (resultDataList []*EdbInfoSearchData, err error) {
	realSaveDataMap := make(map[string]map[int]float64)
	saveDataMap := make(map[string]map[int]float64)

	// 最小的结束日期 , 最晚的数据开始日期
	var minLatestDate, maxStartDate time.Time
	dateList := make([]string, 0) // 第一个指标的日期数据

	formulaStr := strings.ToUpper(rule.Formula)
	// 获取关联指标数据
	for edbInfoIndex, v := range rule.EdbInfoList {
		dataList, tmpErr := GetPredictEdbDataListAll(v, 1)
		if tmpErr != nil {
			err = tmpErr
			return
		}

		//lenData := len(dataList)
		for _, dv := range dataList {
			// 现有实际数据
			if val, ok := realSaveDataMap[dv.DataTime]; ok {
				if _, ok := val[v.EdbInfoId]; !ok {
					val[v.EdbInfoId] = dv.Value
				}
			} else {
				temp := make(map[int]float64)
				temp[v.EdbInfoId] = dv.Value
				realSaveDataMap[dv.DataTime] = temp
			}

			// 待处理的数据
			if val, ok := saveDataMap[dv.DataTime]; ok {
				if _, ok := val[v.EdbInfoId]; !ok {
					val[v.EdbInfoId] = dv.Value
				}
			} else {
				temp := make(map[int]float64)
				temp[v.EdbInfoId] = dv.Value
				saveDataMap[dv.DataTime] = temp
			}

			// 以第一个指标的日期作为基准日期
			if edbInfoIndex == 0 {
				dateList = append(dateList, dv.DataTime)
				tmpDate, _ := time.ParseInLocation(utils.FormatDate, dv.DataTime, time.Local)
				if minLatestDate.IsZero() || tmpDate.After(minLatestDate) {
					minLatestDate = tmpDate
				}
				if maxStartDate.IsZero() || tmpDate.Before(maxStartDate) {
					maxStartDate = tmpDate
				}
			}
		}

		/*if lenData > 0 {
			tmpLatestDate, _ := time.ParseInLocation(utils.FormatDate, dataList[lenData-1].DataTime, time.Local)
			if minLatestDate.IsZero() || minLatestDate.After(tmpLatestDate) {
				minLatestDate = tmpLatestDate
			}

			tmpStartDate, _ := time.ParseInLocation(utils.FormatDate, dataList[0].DataTime, time.Local)
			if maxStartDate.IsZero() || maxStartDate.Before(tmpStartDate) {
				maxStartDate = tmpStartDate
			}
		}*/
	}

	// todo 数据处理，将日期内不全的数据做填补
	HandleDateSaveDataMap(dateList, maxStartDate, minLatestDate, realSaveDataMap, saveDataMap, rule.EdbInfoList, rule.EmptyType)

	// 添加数据
	addDataList := make([]*PredictEdbRuleData, 0)

	// 计算规则
	formulaDateSlice, formulaDateMap, err := utils.HandleFormulaJson(formulaStr, minLatestDate)
	if err != nil {
		return
	}
	//获取指标所有数据
	dataList := make([]*PredictEdbRuleData, 0)
	sql := `SELECT * FROM predict_edb_rule_data WHERE config_id = ?`
	err = to.Raw(sql, rule.ConfigId).Find(&dataList).Error
	if err != nil {
		return
	}
	dataMap := make(map[string]*PredictEdbRuleData)
	removeDateMap := make(map[string]*PredictEdbRuleData) //需要移除的日期
	for _, v := range dataList {
		dataMap[v.DataTime] = v
		removeDateMap[v.DataTime] = v
	}
	existDataMap := make(map[string]string)

	// 判断是否特殊处理max和min函数
	maxDealFlag := false
	if rule.EmptyType == 4 && rule.MaxEmptyType == 2 {
		maxDealFlag = true
	}
	for sk, sv := range saveDataMap {
		// 当空值处理类型选择了不计算时，只要有一个指标在某个日期没有值(即空值)，则计算指标在该日期没有值
		if rule.EmptyType == 1 {
			if len(sv) != len(rule.EdbInfoList) {
				continue
			}
		}
		//fmt.Println(sk, sv)
		// 根据时间范围，选择对应的公式
		formulaMap := make(map[string]string)
		formulaStr = ""
		for _, fv := range formulaDateSlice {
			if sk < fv {
				if f, ok := formulaDateMap[fv]; ok {
					formulaStr = f
					formulaMap, err = utils.CheckFormula(formulaStr)
					if err != nil {
						err = fmt.Errorf("公式错误，请重新填写")
						return
					}
				}
				break
			}
		}
		if formulaStr == "" {
			continue
		}
		svMax := make(map[int]float64)
		if maxDealFlag {
			// 特殊处理max和min函数，如果原本的值为空，则选择空值参与运算
			if svMaxData, ok := realSaveDataMap[sk]; ok {
				svMax = svMaxData
			}
		}
		formulaStr = strings.ToUpper(formulaStr)
		//fmt.Println(sk, sv)
		formulaFormStr := ReplaceFormula(rule.EdbInfoList, sv, svMax, formulaMap, formulaStr, rule.EdbInfoIdBytes, maxDealFlag)
		//计算公式异常，那么就移除该指标
		if formulaFormStr == "" {
			continue
		}

		//utils.FileLog.Info(fmt.Sprintf("formulaFormStr:%s", formulaFormStr))
		//expression := formula.NewExpression(formulaFormStr)
		//calResult, tmpErr := expression.Evaluate()
		//if tmpErr != nil {
		//	// 分母为0的报错
		//	if strings.Contains(tmpErr.Error(), "divide by zero") {
		//		continue
		//	}
		//	err = errors.New("计算失败：Err:" + tmpErr.Error() + ";formulaStr:" + formulaFormStr)
		//	//fmt.Println(err)
		//	return
		//}
		//calVal, tmpErr := calResult.Float64()
		//if tmpErr != nil {
		//	err = errors.New("计算失败：获取计算值失败 Err:" + tmpErr.Error() + ";formulaStr:" + formulaFormStr)
		//	//fmt.Println(err)
		//	return
		//}

		calVal, err := engine.ParseAndExec(formulaFormStr)
		//calVal, err := calResult.Float64()
		if err != nil {
			// 分母为0的报错，忽略该循环
			if utils.IsDivideZero(err) {
				//removeDateList = append(removeDateList, sk)
				continue
			}
			err = errors.New("计算失败：获取计算值失败 Err:" + err.Error() + ";formulaStr:" + formulaFormStr)
			fmt.Println(err)
			return nil, err
		}

		nanCheck := fmt.Sprintf("%0.f", calVal)
		if nanCheck == "NaN" || nanCheck == "+Inf" || nanCheck == "-Inf" {
			continue
		}

		// 移除不存在的日期
		delete(removeDateMap, sk)

		saveValue := decimal.NewFromFloat(calVal).Round(4).String() //utils.SubFloatToString(calVal, 4)
		existPredictEdbRuleData, ok := dataMap[sk]
		if !ok {
			dataTime, _ := time.ParseInLocation(utils.FormatDate, sk, time.Local)
			timestamp := dataTime.UnixNano() / 1e6

			if _, existOk := existDataMap[sk]; !existOk {
				tmpPredictEdbRuleData := &PredictEdbRuleData{
					//PredictEdbRuleDataId: 0,
					EdbInfoId:     rule.EdbInfoId,
					ConfigId:      rule.ConfigId,
					DataTime:      sk,
					Value:         saveValue,
					CreateTime:    time.Now(),
					ModifyTime:    time.Now(),
					DataTimestamp: timestamp,
				}
				addDataList = append(addDataList, tmpPredictEdbRuleData)
			}
			existDataMap[sk] = sk
		} else {
			existValDecimal, tmpErr := decimal.NewFromString(existPredictEdbRuleData.Value)
			if tmpErr != nil {
				err = tmpErr
				return nil, tmpErr
			}
			existStr := existValDecimal.String()
			if existStr != saveValue {
				existPredictEdbRuleData.Value = saveValue
				existPredictEdbRuleData.ModifyTime = time.Now()
				err = to.Model(existPredictEdbRuleData).Select([]string{"Value", "ModifyTime"}).Updates(existPredictEdbRuleData).Error
				if err != nil {
					return nil, err
				}
			}
		}

		// 计算出来的结果集
		resultDataList = append(resultDataList, &EdbInfoSearchData{
			//EdbDataId: 0,
			DataTime: sk,
			Value:    calVal,
		})
	}

	// 添加计算出来的值入库
	lenAddDataList := len(addDataList)
	if lenAddDataList > 0 {
		err = to.CreateInBatches(addDataList, utils.MultiAddNum).Error
		if err != nil {
			return
		}
	}

	//删除多余的值
	lenRemoveDateList := len(removeDateMap)
	if lenRemoveDateList > 0 {
		removeDateList := make([]string, 0) //需要移除的日期
		for date, _ := range removeDateMap {
			removeDateList = append(removeDateList, date)
		}
		//如果拼接指标变更了，那么需要删除所有的指标数据
		sql := ` DELETE FROM predict_edb_rule_data WHERE config_id = ? and data_time in (` + utils.GetOrmInReplace(lenRemoveDateList) + `) `
		err = to.Exec(sql, rule.ConfigId, removeDateList).Error
		if err != nil {
			err = fmt.Errorf("删除计算失败的预测规则计算指标数据失败,Err:" + err.Error())
			return
		}
	}

	return
}

// RefreshCalculateByRuleByLineNh 刷新动态结果计算（线性拟合）
func RefreshCalculateByRuleByLineNh(predictEdbInfo EdbInfo, predictEdbConfAndDataList []*PredictEdbConfAndData, rule PredictEdbConf) (err error, errMsg string) {
	to := global.DEFAULT_DB.Begin()
	defer func() {
		if err != nil {
			to.Rollback()
		} else {
			to.Commit()
		}
	}()
	err, errMsg = CalculateByRuleByRuleLineNh(to, predictEdbInfo, predictEdbConfAndDataList, rule)
	return
}

// CalculateByRuleByRuleLineNh 一元线性拟合规则计算入库
func CalculateByRuleByRuleLineNh(to *gorm.DB, predictEdbInfo EdbInfo, predictEdbConfAndDataList []*PredictEdbConfAndData, rule PredictEdbConf) (err error, errMsg string) {
	var secondDataList []*EdbInfoSearchData
	predictEdbInfoId := predictEdbInfo.EdbInfoId // 预测指标id

	// 规则
	var ruleConf RuleLineNhConf
	tmpErr := json.Unmarshal([]byte(rule.Value), &ruleConf)
	if tmpErr != nil {
		errMsg = `季节性配置信息异常`
		err = errors.New("季节性配置信息异常:" + tmpErr.Error())
		return
	}

	// 获取自身指标的数据
	{
		// 来源指标
		var sourceEdbInfoItem *EdbInfo
		sql := ` SELECT * FROM edb_info WHERE edb_info_id=? `
		err = to.Raw(sql, rule.SourceEdbInfoId).First(&sourceEdbInfoItem).Error
		if err != nil {
			return
		}

		predictEdbInfo.EdbInfoId = 0
		secondDataList, err, _ = GetPredictDataListByPredictEdbConfList(&predictEdbInfo, sourceEdbInfoItem, predictEdbConfAndDataList, 1, ``)
		if err != nil {
			return
		}

	}
	lenSecondData := len(secondDataList)
	if lenSecondData <= 0 {
		return
	}

	newNhccDataMap, err, errMsg := getCalculateNhccData(secondDataList, ruleConf)
	if err != nil {
		return
	}

	//将最后计算出来的结果数据处理（新增入库、编辑日期的值、删除日期）
	{
		// 获取需要预测的日期
		startDateStr := secondDataList[lenSecondData-1].DataTime
		startDate, _ := time.ParseInLocation(utils.FormatDate, startDateStr, time.Local)
		//endDate, _ := time.ParseInLocation(utils.FormatDate, ruleConf.EndDate, time.Local)
		endDate := rule.EndDate
		// todo 拟合时间配置
		dayList := getPredictEdbDayList(startDate, endDate, predictEdbInfo.Frequency, predictEdbInfo.DataDateType, predictEdbInfo.EndDateType, rule.EndNum)
		if len(dayList) <= 0 { // 如果未来没有日期的话，那么就退出当前循环，进入下一个循环
			return
		}

		//获取该配置的所有数据
		dataList := make([]*PredictEdbRuleData, 0)
		sql := `SELECT * FROM predict_edb_rule_data WHERE config_id = ?`
		err = to.Raw(sql, rule.ConfigId).Find(&dataList).Error
		if err != nil {
			return
		}
		dataMap := make(map[string]*PredictEdbRuleData)
		for _, v := range dataList {
			dataMap[v.DataTime] = v
		}

		//需要移除的日期
		removeDateList := make([]string, 0)
		// 已经操作过的日期
		existDataMap := make(map[string]string)
		// 添加数据
		addDataList := make([]*PredictEdbRuleData, 0)

		for _, currentDate := range dayList {
			// 动态拟合残差值数据
			currentDateStr := currentDate.Format(utils.FormatDate)
			val, ok := newNhccDataMap[currentDateStr]
			// 找不到数据，那么就移除该日期的数据
			if !ok {
				removeDateList = append(removeDateList, currentDateStr)
				continue
			}

			saveValue := decimal.NewFromFloat(val).Round(4).String() //utils.SubFloatToString(calVal, 4)
			existPredictEdbRuleData, ok := dataMap[currentDateStr]
			if !ok {
				timestamp := currentDate.UnixNano() / 1e6

				if _, existOk := existDataMap[currentDateStr]; !existOk {
					tmpPredictEdbRuleData := &PredictEdbRuleData{
						//PredictEdbRuleDataId: 0,
						EdbInfoId:     predictEdbInfoId,
						ConfigId:      rule.ConfigId,
						DataTime:      currentDateStr,
						Value:         saveValue,
						CreateTime:    time.Now(),
						ModifyTime:    time.Now(),
						DataTimestamp: timestamp,
					}
					addDataList = append(addDataList, tmpPredictEdbRuleData)
				}
				existDataMap[currentDateStr] = currentDateStr
			} else {
				existValDecimal, tmpErr := decimal.NewFromString(existPredictEdbRuleData.Value)
				if tmpErr != nil {
					err = tmpErr
					return
				}
				existStr := existValDecimal.String()
				if existStr != saveValue {
					existPredictEdbRuleData.Value = saveValue
					existPredictEdbRuleData.ModifyTime = time.Now()
					err = to.Model(existPredictEdbRuleData).Select([]string{"Value", "ModifyTime"}).Updates(existPredictEdbRuleData).Error
					if err != nil {
						return
					}
				}
			}
		}

		// 添加计算出来的值入库
		lenAddDataList := len(addDataList)
		if lenAddDataList > 0 {
			err = to.CreateInBatches(addDataList, utils.MultiAddNum).Error
			if err != nil {
				return
			}
		}

		//删除多余的值
		lenRemoveDateList := len(removeDateList)
		if lenRemoveDateList > 0 {
			//如果拼接指标变更了，那么需要删除所有的指标数据
			sql := ` DELETE FROM predict_edb_rule_data WHERE config_id = ? and data_time in (` + utils.GetOrmInReplace(lenRemoveDateList) + `) `
			err = to.Exec(sql, rule.ConfigId, removeDateList).Error
			if err != nil {
				err = fmt.Errorf("删除计算失败的预测规则计算指标数据失败,Err:" + err.Error())
				return
			}
		}
	}

	return
}