eta_api/services/data/edb_info_calculate.go

package data

import (
	"encoding/json"
	"errors"
	"eta/eta_api/models/data_manage"
	"eta/eta_api/utils"
	"fmt"
	"github.com/shopspring/decimal"
	"math"
	"regexp"
	"sort"
	"strings"
	"time"
)

func CheckFormula(formula string) map[string]string {
	mathFormula := []string{"MAX", "MIN", "ABS", "ACOS", "ASIN", "CEIL", "MOD", "POW", "ROUND", "SIGN", "SIN", "TAN", "LOG10", "LOG2", "LOG", "LN", "EXP"}

	str := strings.ToUpper(formula)
	for _, v := range mathFormula {
		str = strings.Replace(str, v, "", -1)
	}
	str = strings.Replace(str, "(", "", -1)
	str = strings.Replace(str, ")", "", -1)

	byteMap := make(map[string]string)
	for i := 0; i < len(str); i++ {
		byteInt := str[i]
		if byteInt >= 65 && byteInt <= 90 {
			byteStr := string(byteInt)
			if _, ok := byteMap[byteStr]; !ok {
				byteMap[byteStr] = byteStr
			}
		}
	}
	return byteMap
}

type FormulaListItem struct {
	Formula string `json:"f"`
	Date    string `json:"d"`
}

// CheckFormulaJson 检测计算公式json串是否异常
func CheckFormulaJson(formula string) (formulaSlice []string, err error) {
	list := make([]FormulaListItem, 0)
	err = json.Unmarshal([]byte(formula), &list)
	if err != nil {
		err = fmt.Errorf("公式串解析失败： json.Unmarshal Err: %v", err)
		return
	}
	formulaSlice = make([]string, 0)
	// 日期排序
	for _, v := range list {
		formulaSlice = append(formulaSlice, v.Formula)
	}
	return
}

type CalculateItems struct {
	EdbInfoId int
	DataMap   map[string]float64
}

func GetFormulaMap() map[string]string {
	funMap := make(map[string]string)
	funMap["MAX"] = "[@@]"
	funMap["MIN"] = "[@!]"
	funMap["ABS"] = "[@#]"
	funMap["CEIL"] = "[@$]"
	funMap["COS"] = "[@%]"
	funMap["FLOOR"] = "[@^]"
	funMap["MOD"] = "[@&]"
	funMap["POW"] = "[@*]"
	funMap["ROUND"] = "[@(]"
	return funMap
}

// 处理整个数据
func handleDateSaveDataMap(dateList []string, realSaveDataMap, saveDataMap map[string]map[int]float64, edbInfoIdArr []*data_manage.EdbInfo, emptyType int) {
	var startDate, endDate string
	var startDateT, endDateT time.Time
	if emptyType == 2 || emptyType == 3 {
		for k, _ := range realSaveDataMap {
			if k > endDate {
				endDate = k
			}
			if k < startDate || startDate == "" {
				startDate = k
			}
		}

		startDateT, _ = time.ParseInLocation(utils.FormatDate, startDate, time.Local)
		endDateT, _ = time.ParseInLocation(utils.FormatDate, endDate, time.Local)
	}
	for _, date := range dateList {
		tmpDataMap := realSaveDataMap[date]
		for _, edbInfo := range edbInfoIdArr {
			tmpEdbInfoId := edbInfo.EdbInfoId // 当前指标id
			// 如果该日期不存在该指标数据，那么需要找寻前后日期的数据，进行填补
			if _, ok := tmpDataMap[tmpEdbInfoId]; !ok {
				//day := 0
				//switch edbInfo.Frequency {
				//case "周度":
				//	day = 7
				//case "旬度":
				//	day = 15
				//case "月度":
				//	day = 30
				//case "季度":
				//	day = 90
				//case "年度":
				//	day = 365
				//}
				// 需求池 255 指标运算文案修改，补数据遍历区间修改（2023-3-7 09:37:23修改）
				switch emptyType {
				case 0:
					handleDateDataMap(realSaveDataMap, saveDataMap, date, tmpEdbInfoId, 35)
				case 2:
					handleDateDataMapBefore(realSaveDataMap, saveDataMap, date, tmpEdbInfoId, startDateT, endDateT)
				case 3:
					handleDateDataMapAfter(realSaveDataMap, saveDataMap, date, tmpEdbInfoId, startDateT, endDateT)
				case 4:
					handleDateDataMapZero(saveDataMap, date, tmpEdbInfoId)
				}
			}
		}
	}
}

// handleDataMap 处理单个日期的数据
func handleDateDataMap(realSaveDataMap, saveDataMap map[string]map[int]float64, date string, edbInfoId, day int) {
	currDate, _ := time.ParseInLocation(utils.FormatDate, date, time.Local)

	// 后一天
	nextDateDayStr := currDate.AddDate(0, 0, 1).Format(utils.FormatDate)

	// 前一天
	preDateDayStr := currDate.AddDate(0, 0, -1).Format(utils.FormatDate)

	for i := 1; i <= day; i++ {

		// 下个日期的数据
		{
			if i >= 1 {
				nextDateDayStr = currDate.AddDate(0, 0, i).Format(utils.FormatDate)
			}
			if findDataMap, hasFindDataMap := realSaveDataMap[nextDateDayStr]; hasFindDataMap { // 下一个日期有数据
				if val, hasFindItem := findDataMap[edbInfoId]; hasFindItem {
					saveDataMap[date][edbInfoId] = val
					return
				}
			}
		}

		// 上个日期的数据
		{
			if i >= 1 {
				preDateDayStr = currDate.AddDate(0, 0, -i).Format(utils.FormatDate)
			}
			if findDataMap, hasFindDataMap := realSaveDataMap[preDateDayStr]; hasFindDataMap { // 下一个日期有数据
				if val, hasFindItem := findDataMap[edbInfoId]; hasFindItem {
					saveDataMap[date][edbInfoId] = val
					return
				}
			}
		}
	}
}

// handleDataByLinearRegression 插值法补充数据(线性方程式)
func handleDataByLinearRegression(edbInfoDataList []*data_manage.EdbDataList, handleDataMap map[string]float64) (err error) {
	if len(edbInfoDataList) < 2 {
		return
	}

	var startEdbInfoData *data_manage.EdbDataList
	for _, v := range edbInfoDataList {
		handleDataMap[v.DataTime] = v.Value

		// 第一个数据就给过滤了,给后面的试用
		if startEdbInfoData == nil {
			startEdbInfoData = v
			continue
		}

		// 获取两条数据之间相差的天数
		startDataTime, _ := time.ParseInLocation(utils.FormatDate, startEdbInfoData.DataTime, time.Local)
		currDataTime, _ := time.ParseInLocation(utils.FormatDate, v.DataTime, time.Local)
		betweenHour := int(currDataTime.Sub(startDataTime).Hours())
		betweenDay := betweenHour / 24

		// 如果相差一天,那么过滤
		if betweenDay <= 1 {
			startEdbInfoData = v
			continue
		}

		// 生成线性方程式
		var a, b float64
		{
			coordinateData := make([]utils.Coordinate, 0)
			tmpCoordinate1 := utils.Coordinate{
				X: 1,
				Y: startEdbInfoData.Value,
			}
			coordinateData = append(coordinateData, tmpCoordinate1)
			tmpCoordinate2 := utils.Coordinate{
				X: float64(betweenDay) + 1,
				Y: v.Value,
			}
			coordinateData = append(coordinateData, tmpCoordinate2)

			a, b = utils.GetLinearResult(coordinateData)
			if math.IsNaN(a) || math.IsNaN(b) {
				err = errors.New("线性方程公式生成失败")
				return
			}
		}

		// 生成对应的值
		{
			for i := 1; i < betweenDay; i++ {
				tmpDataTime := startDataTime.AddDate(0, 0, i)
				aDecimal := decimal.NewFromFloat(a)
				xDecimal := decimal.NewFromInt(int64(i) + 1)
				bDecimal := decimal.NewFromFloat(b)

				val, _ := aDecimal.Mul(xDecimal).Add(bDecimal).Round(4).Float64()
				handleDataMap[tmpDataTime.Format(utils.FormatDate)] = val
			}
		}

		startEdbInfoData = v
	}

	return
}

// HandleDataByLinearRegression 插值法补充数据(线性方程式)
func HandleDataByLinearRegression(edbInfoDataList []*data_manage.EdbDataList, handleDataMap map[string]float64) (err error) {
	return handleDataByLinearRegression(edbInfoDataList, handleDataMap)
}

// CallCalculateComputeCorrelation 调用计算拟合残差的相关系数
func CallCalculateComputeCorrelation(data *data_manage.EdbInfoCalculateBatchSaveReqByEdbLib, lang string) (val string, err error, errMsg string) {
	errMsg = "计算失败"
	// 调用指标库去更新
	reqJson, err := json.Marshal(data)
	if err != nil {
		errMsg = "计算相关系数参数解析异常！"
		err = errors.New("参数解析失败，Err:" + err.Error())
		return
	}
	respItem, err := CalculateComputeCorrelation(string(reqJson), lang)
	if err != nil {
		return
	}

	if respItem.Ret == 200 {
		val = respItem.Data
	}

	return
}

// handleDateDataMapBefore 前值填充：空值优先以最近的前值填充，没有前值时，用后值填充
func handleDateDataMapBefore(realSaveDataMap, saveDataMap map[string]map[int]float64, date string, edbInfoId int, startDateT, endDateT time.Time) {
	currDate, _ := time.ParseInLocation(utils.FormatDate, date, time.Local)

	// 后一天
	nextDateDay := currDate

	// 前一天
	preDateDay := currDate

	for i := 1; preDateDay.After(startDateT) || preDateDay == startDateT; i++ {
		// 上个日期的数据
		{
			preDateDay = currDate.AddDate(0, 0, -i)
			preDateDayStr := preDateDay.Format(utils.FormatDate)
			if findDataMap, hasFindDataMap := realSaveDataMap[preDateDayStr]; hasFindDataMap { // 下一个日期有数据
				if val, hasFindItem := findDataMap[edbInfoId]; hasFindItem {
					fmt.Println(fmt.Sprintf("date:%s, 无值，取%s的值%.4f", date, preDateDayStr, val))
					saveDataMap[date][edbInfoId] = val
					return
				}
			}
		}
	}

	for i := 1; nextDateDay.Before(endDateT) || nextDateDay == endDateT; i++ {
		// 下个日期的数据
		{
			nextDateDay = currDate.AddDate(0, 0, i)
			nextDateDayStr := nextDateDay.Format(utils.FormatDate)
			if findDataMap, hasFindDataMap := realSaveDataMap[nextDateDayStr]; hasFindDataMap { // 下一个日期有数据
				if val, hasFindItem := findDataMap[edbInfoId]; hasFindItem {
					fmt.Println(fmt.Sprintf("date:%s, 无值，取%s的值%.4f", date, nextDateDayStr, val))
					saveDataMap[date][edbInfoId] = val
					return
				}
			}
		}
	}
	return
}

// handleDateDataMapAfter 后值填充：空值优先以最近的后值填充，没有后值时，用前值填充
func handleDateDataMapAfter(realSaveDataMap, saveDataMap map[string]map[int]float64, date string, edbInfoId int, startDateT, endDateT time.Time) {
	currDate, _ := time.ParseInLocation(utils.FormatDate, date, time.Local)

	// 后一天
	nextDateDay := currDate

	// 前一天
	preDateDay := currDate

	for i := 1; nextDateDay.Before(endDateT) || nextDateDay == endDateT; i++ {
		// 下个日期的数据
		{
			nextDateDay = currDate.AddDate(0, 0, i)
			nextDateDayStr := nextDateDay.Format(utils.FormatDate)
			if findDataMap, hasFindDataMap := realSaveDataMap[nextDateDayStr]; hasFindDataMap { // 下一个日期有数据
				if val, hasFindItem := findDataMap[edbInfoId]; hasFindItem {
					fmt.Println(fmt.Sprintf("date:%s, 无值，取%s的值%.4f", date, nextDateDayStr, val))
					saveDataMap[date][edbInfoId] = val
					return
				}
			}
		}
	}

	for i := 1; preDateDay.After(startDateT) || preDateDay == startDateT; i++ {
		// 上个日期的数据
		{
			preDateDay = currDate.AddDate(0, 0, -i)
			preDateDayStr := preDateDay.Format(utils.FormatDate)
			if findDataMap, hasFindDataMap := realSaveDataMap[preDateDayStr]; hasFindDataMap { // 下一个日期有数据
				if val, hasFindItem := findDataMap[edbInfoId]; hasFindItem {
					fmt.Println(fmt.Sprintf("date:%s, 无值，取%s的值%.4f", date, preDateDayStr, val))
					saveDataMap[date][edbInfoId] = val
					return
				}
			}
		}
	}
	return
}

// handleDateDataMapZero 等于0
func handleDateDataMapZero(saveDataMap map[string]map[int]float64, date string, edbInfoId int) {
	saveDataMap[date][edbInfoId] = 0
	return
}

func GetMaxMinEdbInfo(formula string) string {
	//formula := "A+min(A,B,max(A,C))"
	// todo 无法处理max里嵌套max或者min的情况
	// 使用正则表达式匹配MAX和MIN函数及其参数
	regex := regexp.MustCompile(`(?i)(MAX|MIN)\((.*?)\)`)
	matches := regex.FindAllStringSubmatch(formula, -1)
	// 遍历匹配结果，输出MAX和MIN函数及其参数
	for _, match := range matches {
		if len(match) == 3 {
			parameter := strings.ToLower(match[0]) // 参数
			formula = strings.ReplaceAll(formula, match[0], parameter)
			fmt.Printf("formula: %s\n", formula)
		}
	}
	formula = strings.ReplaceAll(formula, "max", "MAX")
	formula = strings.ReplaceAll(formula, "min", "MIN")
	return formula
}

// HandleDataByLinearRegressionToList 插值法补充数据(线性方程式)
func HandleDataByLinearRegressionToList(edbInfoDataList []*data_manage.EdbDataList, handleDataMap map[string]float64) (dataTimeList []string, valueList []float64, err error) {
	if len(edbInfoDataList) < 2 {
		return
	}

	var startEdbInfoData *data_manage.EdbDataList
	for _, v := range edbInfoDataList {
		handleDataMap[v.DataTime] = v.Value
		dataTimeList = append(dataTimeList, v.DataTime)
		// 第一个数据就给过滤了,给后面的试用
		if startEdbInfoData == nil {
			startEdbInfoData = v
			//startEdbInfoData.DataTime = startEdbInfoData.DataTime[:5]+ "01-01"
			continue
		}

		// 获取两条数据之间相差的天数
		startDataTime, _ := time.ParseInLocation(utils.FormatDate, startEdbInfoData.DataTime, time.Local)
		currDataTime, _ := time.ParseInLocation(utils.FormatDate, v.DataTime, time.Local)
		betweenHour := int(currDataTime.Sub(startDataTime).Hours())
		betweenDay := betweenHour / 24

		// 如果相差一天,那么过滤
		if betweenDay <= 1 {
			startEdbInfoData = v
			continue
		}

		// 生成线性方程式
		var a, b float64
		{
			coordinateData := make([]utils.Coordinate, 0)
			tmpCoordinate1 := utils.Coordinate{
				X: 1,
				Y: startEdbInfoData.Value,
			}
			coordinateData = append(coordinateData, tmpCoordinate1)
			tmpCoordinate2 := utils.Coordinate{
				X: float64(betweenDay) + 1,
				Y: v.Value,
			}
			coordinateData = append(coordinateData, tmpCoordinate2)

			a, b = utils.GetLinearResult(coordinateData)
			if math.IsNaN(a) || math.IsNaN(b) {
				err = errors.New("线性方程公式生成失败")
				return
			}
		}

		// 生成对应的值
		{
			for i := 1; i < betweenDay; i++ {
				tmpDataTime := startDataTime.AddDate(0, 0, i)
				aDecimal := decimal.NewFromFloat(a)
				xDecimal := decimal.NewFromInt(int64(i) + 1)
				bDecimal := decimal.NewFromFloat(b)

				val, _ := aDecimal.Mul(xDecimal).Add(bDecimal).Round(4).Float64()
				handleDataMap[tmpDataTime.Format(utils.FormatDate)] = val
				dataTimeList = append(dataTimeList, tmpDataTime.Format(utils.FormatDate))
				valueList = append(valueList, val)
			}
		}

		startEdbInfoData = v
	}

	return
}

// HandleDataByLinearRegressionToList 保证生成365个数据点的线性插值法
func HandleDataByLinearRegressionToListV2(edbInfoDataList []*data_manage.EdbDataList, handleDataMap map[string]float64) (dataTimeList []string, valueList []float64, err error) {
	if len(edbInfoDataList) < 2 {
		return
	}

	// 确保至少有两天数据来生成线性方程
	if len(edbInfoDataList) < 2 {
		err = errors.New("至少需要两天的数据来执行线性插值")
		return
	}

	// 对数据按日期排序，确保顺序正确
	sort.Slice(edbInfoDataList, func(i, j int) bool {
		t1, _ := time.ParseInLocation(utils.FormatDate, edbInfoDataList[i].DataTime, time.Local)
		t2, _ := time.ParseInLocation(utils.FormatDate, edbInfoDataList[j].DataTime, time.Local)
		return t1.Before(t2)
	})

	startEdbInfoData := edbInfoDataList[0]
	endEdbInfoData := edbInfoDataList[len(edbInfoDataList)-1]

	// 计算起始和结束日期间实际的天数
	startDate, _ := time.ParseInLocation(utils.FormatDate, startEdbInfoData.DataTime, time.Local)
	endDate, _ := time.ParseInLocation(utils.FormatDate, endEdbInfoData.DataTime, time.Local)
	actualDays := endDate.Sub(startDate).Hours() / 24

	// 生成365个数据点，首先处理已有数据
	for _, v := range edbInfoDataList {
		handleDataMap[v.DataTime] = v.Value
		dataTimeList = append(dataTimeList, v.DataTime)
		valueList = append(valueList, v.Value)
	}

	// 如果已有数据跨越天数不足365天，则对缺失的日期进行线性插值
	if actualDays < 365 {
		// 使用已有数据点生成线性方程（这里简化处理，实际可能需更细致处理边界情况）
		var a, b float64
		coordinateData := []utils.Coordinate{
			{X: 1, Y: startEdbInfoData.Value},
			{X: float64(len(edbInfoDataList)), Y: endEdbInfoData.Value},
		}
		a, b = utils.GetLinearResult(coordinateData)
		if math.IsNaN(a) || math.IsNaN(b) {
			err = errors.New("线性方程公式生成失败")
			return
		}

		// 对剩余日期进行插值
		for i := 1; i < 365; i++ {
			day := startDate.AddDate(0, 0, i)
			if _, exists := handleDataMap[day.Format(utils.FormatDate)]; !exists {
				aDecimal := decimal.NewFromFloat(a)
				xDecimal := decimal.NewFromInt(int64(i) + 1)
				bDecimal := decimal.NewFromFloat(b)

				val, _ := aDecimal.Mul(xDecimal).Add(bDecimal).Round(4).Float64()
				handleDataMap[day.Format(utils.FormatDate)] = val
				dataTimeList = append(dataTimeList, day.Format(utils.FormatDate))
				valueList = append(valueList, val)
			}
		}
	}

	return
}

// HandleDataByLinearRegressionToListV3 插值法补充数据(线性方程式)-直接补充指标起始日期间的所有数据
func HandleDataByLinearRegressionToListV3(edbInfoDataList []*data_manage.EdbDataList, handleDataMap map[string]float64) (newEdbInfoDataList []*data_manage.EdbDataList, dataTimeList []string, valueList []float64, err error) {
	if len(edbInfoDataList) < 2 {
		return
	}

	var startEdbInfoData *data_manage.EdbDataList
	for _, v := range edbInfoDataList {
		handleDataMap[v.DataTime] = v.Value
		newEdbInfoDataList = append(newEdbInfoDataList, v)
		dataTimeList = append(dataTimeList, v.DataTime)
		// 第一个数据就给过滤了,给后面的试用
		if startEdbInfoData == nil {
			startEdbInfoData = v
			//startEdbInfoData.DataTime = startEdbInfoData.DataTime[:5]+ "01-01"
			continue
		}

		// 获取两条数据之间相差的天数
		startDataTime, _ := time.ParseInLocation(utils.FormatDate, startEdbInfoData.DataTime, time.Local)
		currDataTime, _ := time.ParseInLocation(utils.FormatDate, v.DataTime, time.Local)
		betweenHour := int(currDataTime.Sub(startDataTime).Hours())
		betweenDay := betweenHour / 24

		// 如果相差一天,那么过滤
		if betweenDay <= 1 {
			startEdbInfoData = v
			continue
		}

		// 生成线性方程式
		var a, b float64
		{
			coordinateData := make([]utils.Coordinate, 0)
			tmpCoordinate1 := utils.Coordinate{
				X: 1,
				Y: startEdbInfoData.Value,
			}
			coordinateData = append(coordinateData, tmpCoordinate1)
			tmpCoordinate2 := utils.Coordinate{
				X: float64(betweenDay) + 1,
				Y: v.Value,
			}
			coordinateData = append(coordinateData, tmpCoordinate2)

			a, b = utils.GetLinearResult(coordinateData)
			if math.IsNaN(a) || math.IsNaN(b) {
				err = errors.New("线性方程公式生成失败")
				return
			}
		}

		// 生成对应的值
		{
			for i := 1; i < betweenDay; i++ {
				tmpDataTime := startDataTime.AddDate(0, 0, i)
				aDecimal := decimal.NewFromFloat(a)
				xDecimal := decimal.NewFromInt(int64(i) + 1)
				bDecimal := decimal.NewFromFloat(b)

				val, _ := aDecimal.Mul(xDecimal).Add(bDecimal).Round(4).Float64()
				handleDataMap[tmpDataTime.Format(utils.FormatDate)] = val
				dataTimeList = append(dataTimeList, tmpDataTime.Format(utils.FormatDate))
				valueList = append(valueList, val)
				newEdbInfoDataList = append(newEdbInfoDataList, &data_manage.EdbDataList{
					EdbDataId:     v.EdbDataId,
					EdbInfoId:     v.EdbInfoId,
					DataTime:      tmpDataTime.Format(utils.FormatDate),
					DataTimestamp: tmpDataTime.UnixNano() / 1e6,
					Value:         val,
				})
			}
		}

		startEdbInfoData = v
	}

	return
}