eta_index_lib/models/base_predict_from_calculate.go

package models

import (
	"encoding/json"
	"errors"
	"eta_gn/eta_index_lib/global"
	"eta_gn/eta_index_lib/utils"
	"fmt"
	"github.com/dengsgo/math-engine/engine"
	"github.com/shopspring/decimal"
	"gorm.io/gorm"
	"strconv"
	"strings"
	"time"
)

// PredictEdbInfoCalculateSaveReq 预测计算（运算）指标请求参数
type PredictEdbInfoCalculateSaveReq struct {
	AdminId          int    `description:"添加人id"`
	AdminName        string `description:"添加人名称"`
	EdbName          string `description:"指标名称"`
	Frequency        string `description:"频率"`
	Unit             string `description:"单位"`
	ClassifyId       int    `description:"分类id"`
	CalculateFormula string `description:"计算公式"`
	EdbInfoIdArr     []struct {
		EdbInfoId int    `description:"指标id"`
		FromTag   string `description:"指标对应标签"`
	}
}

// PredictCalculateItems 预测计算（运算）指标信息
type PredictCalculateItems struct {
	EdbInfoId int
	DataMap   map[string]float64
}

// EditPredictCalculate 新增预测计算（运算）指标
func EditPredictCalculate(edbInfo *EdbInfo, calculateMappingList []*EdbInfoCalculateMapping) (err error) {
	to := global.DEFAULT_DmSQL.Begin()
	defer func() {
		if err != nil {
			tmpErr := to.Rollback().Error
			errMsg := fmt.Sprintf("AddPredictCalculate Err:%s" + err.Error())
			if tmpErr != nil {
				errMsg += `;rollback失败：` + tmpErr.Error()
			}
			utils.FileLog.Info(errMsg)
			return
		}

		tmpErr := to.Commit().Error
		if tmpErr != nil {
			errMsg := fmt.Sprintf("AddPredictCalculate commit Err:%s" + tmpErr.Error())
			utils.FileLog.Info(errMsg)
		}
	}()

	// 修改指标信息
	err = edbInfo.Update(
		[]string{"EdbName", "Frequency", "Unit", "ClassifyId", "CalculateFormula", "EmptyType", "MaxEmptyType", "Extra", "EdbNameEn", "UnitEn"},
	)
	if err != nil {
		return
	}

	//删除，计算指标关联的，基础指标的关联关系
	sql := ` DELETE FROM edb_info_calculate_mapping WHERE edb_info_id = ? `
	err = to.Exec(sql, edbInfo.EdbInfoId).Error
	if err != nil {
		err = errors.New("删除计算指标关联关系失败,Err:" + err.Error())
		return
	}

	// 插入新的关系表
	if len(calculateMappingList) > 0 {
		err = to.CreateInBatches(calculateMappingList, len(calculateMappingList)).Error
	}

	//数据表
	tableName := GetEdbDataTableName(edbInfo.Source, edbInfo.SubSource)
	//清空原有数据
	sql = ` DELETE FROM ` + tableName + ` WHERE edb_info_id = ? `
	err = to.Exec(sql, edbInfo.EdbInfoId).Error
	if err != nil {
		return
	}

	return
}

func AddPredictCalculateData(edbInfoIdList []*EdbInfo, edbInfoTag map[string]int, edbInfo *EdbInfo, edbCode, formulaStr string, edbInfoIdBytes []string) (latestDateStr string, latestValue float64, err error) {
	to := global.DEFAULT_DmSQL.Begin()
	defer func() {
		if err != nil {
			fmt.Println("AddPredictCalculateData Calculate,Err:" + err.Error())
			_ = to.Rollback()
		} else {
			_ = to.Commit()
		}
	}()

	latestDateStr, latestValue, err = refreshAllPredictCalculate(to, edbInfoIdList, edbInfoTag, edbInfo.EdbInfoId, edbInfo.Source, edbInfo.SubSource, edbCode, formulaStr, "", edbInfoIdBytes, edbInfo.EmptyType, edbInfo.MaxEmptyType, edbInfo.Extra)
	return
}

// RefreshAllPredictCalculate 刷新预测计算指标的全部数据
func RefreshAllPredictCalculate(edbInfoIdList []*EdbInfo, edbInfoTag map[string]int, edbInfoId, source, subSource int, edbCode, formulaStr, startDate string, edbInfoIdBytes []string, emptyType, maxEmptyType int, extra string) (latestDateStr string, latestValue float64, err error) {
	to := global.DEFAULT_DmSQL.Begin()
	defer func() {
		if err != nil {
			fmt.Println("RefreshAllCalculate,Err:" + err.Error())
			_ = to.Rollback()
		} else {
			_ = to.Commit()
		}
	}()

	latestDateStr, latestValue, err = refreshAllPredictCalculate(to, edbInfoIdList, edbInfoTag, edbInfoId, source, subSource, edbCode, formulaStr, startDate, edbInfoIdBytes, emptyType, maxEmptyType, extra)

	return
}

// refreshAllPredictCalculate 刷新预测计算指标的全部数据
func refreshAllPredictCalculate(to *gorm.DB, edbInfoIdList []*EdbInfo, edbInfoTag map[string]int, edbInfoId, source, subSource int, edbCode, formulaStr, startDate string, edbInfoIdBytes []string, emptyType, maxEmptyType int, extra string) (latestDateStr string, latestValue float64, err error) {
	fmt.Println("startDate:", startDate)
	// 最小的结束日期 , 最晚的数据开始日期
	var newLatestDate, minLatestDate, maxStartDate time.Time
	dateList := make([]string, 0) // 第一个指标的日期数据

	realSaveDataMap := make(map[string]map[int]float64)
	saveDataMap := make(map[string]map[int]float64)
	dateMap := make(map[string]struct{}) // 最终的日期数据

	dateTagConfig := ""
	if extra != "" {
		var dateConfig CalculateEdbExtra
		err = json.Unmarshal([]byte(extra), &dateConfig)
		if err != nil {
			err = fmt.Errorf("refreshAllCalculate,extra解析失败,Err:%s", err.Error())
			return
		}
		dateTagConfig = dateConfig.DateTag
	}

	for edbInfoIndex, v := range edbInfoIdList {
		// 单独存储max、min函数里的指标的数据
		tmpNewLatestDate, tmpErr := time.ParseInLocation(utils.FormatDate, v.LatestDate, time.Local)
		if tmpErr != nil {
			err = errors.New(`最近的日期格式化失败;日期：` + v.LatestDate + `;err:` + tmpErr.Error())
			return
		}
		if newLatestDate.IsZero() || tmpNewLatestDate.Before(newLatestDate) {
			newLatestDate = tmpNewLatestDate
		}

		// 获取关联指标数据
		dataList, tmpErr := GetPredictEdbDataListAllByStartDate(v, 1, startDate)
		if tmpErr != nil {
			err = tmpErr
			return
		}

		for _, dv := range dataList { // 现有实际数据
			if val, ok := realSaveDataMap[dv.DataTime]; ok {
				if _, ok := val[v.EdbInfoId]; !ok {
					val[v.EdbInfoId] = dv.Value
				}
			} else {
				temp := make(map[int]float64)
				temp[v.EdbInfoId] = dv.Value
				realSaveDataMap[dv.DataTime] = temp
			}

			// 待处理的数据
			if val, ok := saveDataMap[dv.DataTime]; ok {
				if _, ok := val[v.EdbInfoId]; !ok {
					val[v.EdbInfoId] = dv.Value
				}
			} else {
				temp := make(map[int]float64)
				temp[v.EdbInfoId] = dv.Value
				saveDataMap[dv.DataTime] = temp
			}

			if dateTagConfig == "all" {
				if _, ok := dateMap[dv.DataTime]; !ok {
					dateList = append(dateList, dv.DataTime)
					dateMap[dv.DataTime] = struct{}{}
				}
			} else if dateTagConfig == "" { // 默认取第一个指标的时间序列
				if edbInfoIndex == 0 {
					if _, ok := dateMap[dv.DataTime]; !ok {
						dateList = append(dateList, dv.DataTime)
						dateMap[dv.DataTime] = struct{}{}
					}
				}
			} else {
				if eId, ok := edbInfoTag[dateTagConfig]; ok {
					if v.EdbInfoId == eId {
						if _, ok1 := dateMap[dv.DataTime]; !ok1 {
							dateList = append(dateList, dv.DataTime)
							dateMap[dv.DataTime] = struct{}{}
						}
					}
				}
			}
		}
	}
	// 处理最大日期和最小日期
	for _, v := range dateList {
		tmpDate, _ := time.ParseInLocation(utils.FormatDate, v, time.Local)
		if minLatestDate.IsZero() || tmpDate.After(minLatestDate) {
			minLatestDate = tmpDate
		}
		if maxStartDate.IsZero() || tmpDate.Before(maxStartDate) {
			maxStartDate = tmpDate
		}
	}
	//数据处理，将日期内不全的数据做填补
	HandleDateSaveDataMap(dateList, maxStartDate, minLatestDate, realSaveDataMap, saveDataMap, edbInfoIdList, emptyType)

	formulaDateSlice, formulaDateMap, err := utils.HandleFormulaJson(formulaStr, minLatestDate)
	if err != nil {
		return
	}
	addSql := ` INSERT INTO edb_data_predict_calculate (edb_info_id,edb_code,data_time,value,create_time,modify_time,data_timestamp) values `
	var isAdd bool

	var sql string
	dataTableName := GetEdbDataTableName(source, subSource)
	//获取指标所有数据
	dataList, err := GetAllEdbDataListByTo(to, edbInfoId, source, subSource)
	if err != nil {
		return
	}
	dataMap := make(map[string]string)
	for _, v := range dataList {
		dataMap[v.DataTime] = v.Value
	}
	edbInfoIdStr := strconv.Itoa(edbInfoId)
	existDataMap := make(map[string]string)

	removeDateList := make([]string, 0) //需要移除的日期
	// 判断是否特殊处理max和min函数
	maxDealFlag := false
	if emptyType == 4 && maxEmptyType == 2 {
		maxDealFlag = true
	}

	for sk, sv := range saveDataMap {
		if _, ok := dateMap[sk]; !ok { //不在最终的日期序列里面不计算
			continue
		}
		// 当空值处理类型选择了不计算时，只要有一个指标在某个日期没有值(即空值)，则计算指标在该日期没有值
		if emptyType == 1 {
			if len(sv) != len(edbInfoIdList) {
				continue
			}
		}
		// 根据时间范围，选择对应的公式
		formulaMap := make(map[string]string)
		formulaStr = ""
		for _, fv := range formulaDateSlice {
			if sk < fv {
				if f, ok := formulaDateMap[fv]; ok {
					formulaStr = f
					formulaMap, err = utils.CheckFormula(formulaStr)
					if err != nil {
						err = fmt.Errorf("公式错误，请重新填写")
						return
					}
				}
				break
			}
		}
		if formulaStr == "" {
			continue
		}
		svMax := make(map[int]float64)
		if maxDealFlag {
			// 特殊处理max和min函数，如果原本的值为空，则选择空值参与运算
			if svMaxData, ok := realSaveDataMap[sk]; ok {
				svMax = svMaxData
			}
		}
		formulaStr = strings.ToUpper(formulaStr)
		formulaFormStr := ReplaceFormula(edbInfoIdList, sv, svMax, formulaMap, formulaStr, edbInfoIdBytes, maxDealFlag)
		if formulaFormStr != "" {
			fmt.Println(fmt.Sprintf("date %s:formulaFormStr:%s", sk, formulaFormStr))

			calVal, err := engine.ParseAndExec(formulaFormStr)
			if err != nil {
				// 分母为0的报错，忽略该循环
				if utils.IsDivideZero(err) {
					removeDateList = append(removeDateList, sk)
					continue
				}
				err = errors.New("计算失败：获取计算值失败 Err:" + err.Error() + ";formulaStr:" + formulaFormStr)
				return "", 0, err
			}
			nanCheck := fmt.Sprintf("%0.f", calVal)
			// 分母为0.0的报错
			if nanCheck == "NaN" || nanCheck == "+Inf" || nanCheck == "-Inf" {
				removeDateList = append(removeDateList, sk)
				continue
			}
			saveValue := decimal.NewFromFloat(calVal).Round(4).String() //utils.SubFloatToString(calVal, 4)
			if existVal, ok := dataMap[sk]; !ok {
				dataTime, _ := time.ParseInLocation(utils.FormatDate, sk, time.Local)
				timestamp := dataTime.UnixNano() / 1e6
				timeStr := fmt.Sprintf("%d", timestamp)

				if _, existOk := existDataMap[sk]; !existOk {
					addSql += GetAddSql(edbInfoIdStr, edbCode, sk, timeStr, saveValue)
					isAdd = true
				}
				existDataMap[sk] = sk
			} else {
				existValDecimal, tmpErr := decimal.NewFromString(existVal)
				if tmpErr != nil {
					err = tmpErr
					return "", 0, err
				}
				existStr := existValDecimal.String()
				if existStr != saveValue {
					sql := ` UPDATE %s SET value=?,modify_time=NOW() WHERE edb_info_id=? AND data_time=? `
					sql = fmt.Sprintf(sql, dataTableName)
					err = to.Exec(sql, saveValue, edbInfoId, sk).Error
					if err != nil {
						return "", 0, err
					}
				}
			}
		} else {
			//计算公式异常，那么就移除该指标
			removeDateList = append(removeDateList, sk)
			continue
		}
	}

	if isAdd {
		addSql = strings.TrimRight(addSql, ",")
		err = global.DEFAULT_DmSQL.Exec(addSql).Error
		if err != nil {
			fmt.Println("RefreshAllCalculate add Err", err.Error())
			return
		}
	}

	if len(removeDateList) > 0 {
		removeDateStr := strings.Join(removeDateList, `','`)
		removeDateStr = `'` + removeDateStr + `'`
		//如果拼接指标变更了，那么需要删除所有的指标数据
		tableName := GetEdbDataTableName(source, subSource)
		sql := fmt.Sprintf(` DELETE FROM %s WHERE edb_info_id = ? and data_time in (%s) `, tableName, removeDateStr)

		err = to.Exec(sql, edbInfoId).Error
		if err != nil {
			err = fmt.Errorf("删除计算失败的计算指标数据失败,Err:" + err.Error())
			return
		}
	}

	//查询预测日期
	if !newLatestDate.IsZero() {
		latestDateStr = newLatestDate.Format(utils.FormatDate)
		sql = fmt.Sprintf(` SELECT * FROM %s WHERE edb_info_id = ? and data_time <= ? ORDER BY data_time desc limit 1`, dataTableName)
		var latestDate EdbData
		e := to.Raw(sql, edbInfoId, latestDateStr).Find(&latestDate).Error
		if e != nil {
			if e.Error() == utils.ErrNoRow() {
				//首个日期的值
				sql = fmt.Sprintf(` SELECT * FROM %s WHERE edb_info_id = ? ORDER BY data_time ASC limit 1`, dataTableName)
				e = to.Raw(sql, edbInfoId).Find(&latestDate).Error
				if e != nil {
					err = fmt.Errorf("查询最新日期失败,Err:" + e.Error())
					return
				}
				floatValue, _ := strconv.ParseFloat(latestDate.Value, 64)
				latestDateStr = latestDate.DataTime
				latestValue = floatValue
			} else {
				err = fmt.Errorf("查询最新日期失败,Err:" + e.Error())
				return
			}

		} else {
			floatValue, _ := strconv.ParseFloat(latestDate.Value, 64)
			latestDateStr = latestDate.DataTime
			latestValue = floatValue
		}
	}

	return
}