package data import ( "encoding/json" "errors" "eta_gn/eta_api/models/data_manage" "eta_gn/eta_api/utils" "fmt" "github.com/shopspring/decimal" "math" "sort" "strings" "time" ) func CheckFormula(formula string) map[string]string { mathFormula := []string{"MAX", "MIN", "ABS", "ACOS", "ASIN", "CEIL", "MOD", "POW", "ROUND", "SIGN", "SIN", "TAN", "LOG10", "LOG2", "LOG", "LN", "EXP"} str := strings.ToUpper(formula) for _, v := range mathFormula { str = strings.Replace(str, v, "", -1) } str = strings.Replace(str, "(", "", -1) str = strings.Replace(str, ")", "", -1) byteMap := make(map[string]string) for i := 0; i < len(str); i++ { byteInt := str[i] if byteInt >= 65 && byteInt <= 90 { byteStr := string(byteInt) if _, ok := byteMap[byteStr]; !ok { byteMap[byteStr] = byteStr } } } return byteMap } type FormulaListItem struct { Formula string `json:"f"` Date string `json:"d"` } // CheckFormulaJson 检测计算公式json串是否异常 func CheckFormulaJson(formula string) (formulaSlice []string, err error) { list := make([]FormulaListItem, 0) err = json.Unmarshal([]byte(formula), &list) if err != nil { err = fmt.Errorf("公式串解析失败: json.Unmarshal Err: %v", err) return } formulaSlice = make([]string, 0) // 日期排序 for _, v := range list { formulaSlice = append(formulaSlice, v.Formula) } return } type CalculateItems struct { EdbInfoId int DataMap map[string]float64 } // handleDataByLinearRegression 插值法补充数据(线性方程式) func handleDataByLinearRegression(edbInfoDataList []*data_manage.EdbDataList, handleDataMap map[string]float64) (err error) { if len(edbInfoDataList) < 2 { return } var startEdbInfoData *data_manage.EdbDataList for _, v := range edbInfoDataList { handleDataMap[v.DataTime] = v.Value // 第一个数据就给过滤了,给后面的试用 if startEdbInfoData == nil { startEdbInfoData = v continue } // 获取两条数据之间相差的天数 startDataTime, _ := time.ParseInLocation(utils.FormatDate, startEdbInfoData.DataTime, time.Local) currDataTime, _ := time.ParseInLocation(utils.FormatDate, v.DataTime, time.Local) betweenHour := int(currDataTime.Sub(startDataTime).Hours()) betweenDay := betweenHour / 24 // 如果相差一天,那么过滤 if betweenDay <= 1 { startEdbInfoData = v continue } // 生成线性方程式 var a, b float64 { coordinateData := make([]utils.Coordinate, 0) tmpCoordinate1 := utils.Coordinate{ X: 1, Y: startEdbInfoData.Value, } coordinateData = append(coordinateData, tmpCoordinate1) tmpCoordinate2 := utils.Coordinate{ X: float64(betweenDay) + 1, Y: v.Value, } coordinateData = append(coordinateData, tmpCoordinate2) a, b = utils.GetLinearResult(coordinateData) if math.IsNaN(a) || math.IsNaN(b) { err = errors.New("线性方程公式生成失败") return } } // 生成对应的值 { for i := 1; i < betweenDay; i++ { tmpDataTime := startDataTime.AddDate(0, 0, i) aDecimal := decimal.NewFromFloat(a) xDecimal := decimal.NewFromInt(int64(i) + 1) bDecimal := decimal.NewFromFloat(b) val, _ := aDecimal.Mul(xDecimal).Add(bDecimal).Round(4).Float64() handleDataMap[tmpDataTime.Format(utils.FormatDate)] = val } } startEdbInfoData = v } return } // HandleDataByLinearRegression 插值法补充数据(线性方程式) func HandleDataByLinearRegression(edbInfoDataList []*data_manage.EdbDataList, handleDataMap map[string]float64) (err error) { return handleDataByLinearRegression(edbInfoDataList, handleDataMap) } // CallCalculateComputeCorrelation 调用计算拟合残差的相关系数 func CallCalculateComputeCorrelation(data *data_manage.EdbInfoCalculateBatchSaveReqByEdbLib, lang string) (val string, err error, errMsg string) { errMsg = "计算失败" // 调用指标库去更新 reqJson, err := json.Marshal(data) if err != nil { errMsg = "计算相关系数参数解析异常!" err = errors.New("参数解析失败,Err:" + err.Error()) return } respItem, err := CalculateComputeCorrelation(string(reqJson), lang) if err != nil { return } if respItem.Ret == 200 { val = respItem.Data } return } // HandleDataByLinearRegressionToList 插值法补充数据(线性方程式) func HandleDataByLinearRegressionToList(edbInfoDataList []*data_manage.EdbDataList, handleDataMap map[string]float64) (dataTimeList []string, valueList []float64, err error) { if len(edbInfoDataList) < 2 { return } var startEdbInfoData *data_manage.EdbDataList for _, v := range edbInfoDataList { handleDataMap[v.DataTime] = v.Value dataTimeList = append(dataTimeList, v.DataTime) // 第一个数据就给过滤了,给后面的试用 if startEdbInfoData == nil { startEdbInfoData = v //startEdbInfoData.DataTime = startEdbInfoData.DataTime[:5]+ "01-01" continue } // 获取两条数据之间相差的天数 startDataTime, _ := time.ParseInLocation(utils.FormatDate, startEdbInfoData.DataTime, time.Local) currDataTime, _ := time.ParseInLocation(utils.FormatDate, v.DataTime, time.Local) betweenHour := int(currDataTime.Sub(startDataTime).Hours()) betweenDay := betweenHour / 24 // 如果相差一天,那么过滤 if betweenDay <= 1 { startEdbInfoData = v continue } // 生成线性方程式 var a, b float64 { coordinateData := make([]utils.Coordinate, 0) tmpCoordinate1 := utils.Coordinate{ X: 1, Y: startEdbInfoData.Value, } coordinateData = append(coordinateData, tmpCoordinate1) tmpCoordinate2 := utils.Coordinate{ X: float64(betweenDay) + 1, Y: v.Value, } coordinateData = append(coordinateData, tmpCoordinate2) a, b = utils.GetLinearResult(coordinateData) if math.IsNaN(a) || math.IsNaN(b) { err = errors.New("线性方程公式生成失败") return } } // 生成对应的值 { for i := 1; i < betweenDay; i++ { tmpDataTime := startDataTime.AddDate(0, 0, i) aDecimal := decimal.NewFromFloat(a) xDecimal := decimal.NewFromInt(int64(i) + 1) bDecimal := decimal.NewFromFloat(b) val, _ := aDecimal.Mul(xDecimal).Add(bDecimal).Round(4).Float64() handleDataMap[tmpDataTime.Format(utils.FormatDate)] = val dataTimeList = append(dataTimeList, tmpDataTime.Format(utils.FormatDate)) valueList = append(valueList, val) } } startEdbInfoData = v } return } // HandleDataByLinearRegressionToList 保证生成365个数据点的线性插值法 func HandleDataByLinearRegressionToListV2(edbInfoDataList []*data_manage.EdbDataList, handleDataMap map[string]float64) (dataTimeList []string, valueList []float64, err error) { if len(edbInfoDataList) < 2 { return } // 确保至少有两天数据来生成线性方程 if len(edbInfoDataList) < 2 { err = errors.New("至少需要两天的数据来执行线性插值") return } // 对数据按日期排序,确保顺序正确 sort.Slice(edbInfoDataList, func(i, j int) bool { t1, _ := time.ParseInLocation(utils.FormatDate, edbInfoDataList[i].DataTime, time.Local) t2, _ := time.ParseInLocation(utils.FormatDate, edbInfoDataList[j].DataTime, time.Local) return t1.Before(t2) }) startEdbInfoData := edbInfoDataList[0] endEdbInfoData := edbInfoDataList[len(edbInfoDataList)-1] // 计算起始和结束日期间实际的天数 startDate, _ := time.ParseInLocation(utils.FormatDate, startEdbInfoData.DataTime, time.Local) endDate, _ := time.ParseInLocation(utils.FormatDate, endEdbInfoData.DataTime, time.Local) actualDays := endDate.Sub(startDate).Hours() / 24 // 生成365个数据点,首先处理已有数据 for _, v := range edbInfoDataList { handleDataMap[v.DataTime] = v.Value dataTimeList = append(dataTimeList, v.DataTime) valueList = append(valueList, v.Value) } // 如果已有数据跨越天数不足365天,则对缺失的日期进行线性插值 if actualDays < 365 { // 使用已有数据点生成线性方程(这里简化处理,实际可能需更细致处理边界情况) var a, b float64 coordinateData := []utils.Coordinate{ {X: 1, Y: startEdbInfoData.Value}, {X: float64(len(edbInfoDataList)), Y: endEdbInfoData.Value}, } a, b = utils.GetLinearResult(coordinateData) if math.IsNaN(a) || math.IsNaN(b) { err = errors.New("线性方程公式生成失败") return } // 对剩余日期进行插值 for i := 1; i < 365; i++ { day := startDate.AddDate(0, 0, i) if _, exists := handleDataMap[day.Format(utils.FormatDate)]; !exists { aDecimal := decimal.NewFromFloat(a) xDecimal := decimal.NewFromInt(int64(i) + 1) bDecimal := decimal.NewFromFloat(b) val, _ := aDecimal.Mul(xDecimal).Add(bDecimal).Round(4).Float64() handleDataMap[day.Format(utils.FormatDate)] = val dataTimeList = append(dataTimeList, day.Format(utils.FormatDate)) valueList = append(valueList, val) } } } return } // HandleDataByLinearRegressionToListV3 插值法补充数据(线性方程式)-直接补充指标起始日期间的所有数据 func HandleDataByLinearRegressionToListV3(edbInfoDataList []*data_manage.EdbDataList, handleDataMap map[string]float64) (newEdbInfoDataList []*data_manage.EdbDataList, dataTimeList []string, valueList []float64, err error) { if len(edbInfoDataList) < 2 { return } var startEdbInfoData *data_manage.EdbDataList for _, v := range edbInfoDataList { handleDataMap[v.DataTime] = v.Value newEdbInfoDataList = append(newEdbInfoDataList, v) dataTimeList = append(dataTimeList, v.DataTime) // 第一个数据就给过滤了,给后面的试用 if startEdbInfoData == nil { startEdbInfoData = v //startEdbInfoData.DataTime = startEdbInfoData.DataTime[:5]+ "01-01" continue } // 获取两条数据之间相差的天数 startDataTime, _ := time.ParseInLocation(utils.FormatDate, startEdbInfoData.DataTime, time.Local) currDataTime, _ := time.ParseInLocation(utils.FormatDate, v.DataTime, time.Local) betweenHour := int(currDataTime.Sub(startDataTime).Hours()) betweenDay := betweenHour / 24 // 如果相差一天,那么过滤 if betweenDay <= 1 { startEdbInfoData = v continue } // 生成线性方程式 var a, b float64 { coordinateData := make([]utils.Coordinate, 0) tmpCoordinate1 := utils.Coordinate{ X: 1, Y: startEdbInfoData.Value, } coordinateData = append(coordinateData, tmpCoordinate1) tmpCoordinate2 := utils.Coordinate{ X: float64(betweenDay) + 1, Y: v.Value, } coordinateData = append(coordinateData, tmpCoordinate2) a, b = utils.GetLinearResult(coordinateData) if math.IsNaN(a) || math.IsNaN(b) { err = errors.New("线性方程公式生成失败") return } } // 生成对应的值 { for i := 1; i < betweenDay; i++ { tmpDataTime := startDataTime.AddDate(0, 0, i) aDecimal := decimal.NewFromFloat(a) xDecimal := decimal.NewFromInt(int64(i) + 1) bDecimal := decimal.NewFromFloat(b) val, _ := aDecimal.Mul(xDecimal).Add(bDecimal).Round(4).Float64() handleDataMap[tmpDataTime.Format(utils.FormatDate)] = val dataTimeList = append(dataTimeList, tmpDataTime.Format(utils.FormatDate)) valueList = append(valueList, val) newEdbInfoDataList = append(newEdbInfoDataList, &data_manage.EdbDataList{ EdbDataId: v.EdbDataId, EdbInfoId: v.EdbInfoId, DataTime: tmpDataTime.Format(utils.FormatDate), DataTimestamp: tmpDataTime.UnixNano() / 1e6, Value: val, }) } } startEdbInfoData = v } return }