package data import ( "encoding/json" "errors" "eta/eta_api/models/data_manage" "eta/eta_api/utils" "fmt" "github.com/shopspring/decimal" "math" "regexp" "sort" "strings" "time" ) func CheckFormula(formula string) map[string]string { mathFormula := []string{"MAX", "MIN", "ABS", "ACOS", "ASIN", "CEIL", "MOD", "POW", "ROUND", "SIGN", "SIN", "TAN", "LOG10", "LOG2", "LOG", "LN", "EXP"} str := strings.ToUpper(formula) for _, v := range mathFormula { str = strings.Replace(str, v, "", -1) } str = strings.Replace(str, "(", "", -1) str = strings.Replace(str, ")", "", -1) byteMap := make(map[string]string) for i := 0; i < len(str); i++ { byteInt := str[i] if byteInt >= 65 && byteInt <= 90 { byteStr := string(byteInt) if _, ok := byteMap[byteStr]; !ok { byteMap[byteStr] = byteStr } } } return byteMap } type FormulaListItem struct { Formula string `json:"f"` Date string `json:"d"` } // CheckFormulaJson 检测计算公式json串是否异常 func CheckFormulaJson(formula string) (formulaSlice []string, err error) { list := make([]FormulaListItem, 0) err = json.Unmarshal([]byte(formula), &list) if err != nil { err = fmt.Errorf("公式串解析失败: json.Unmarshal Err: %v", err) return } formulaSlice = make([]string, 0) // 日期排序 for _, v := range list { formulaSlice = append(formulaSlice, v.Formula) } return } type CalculateItems struct { EdbInfoId int DataMap map[string]float64 } func GetFormulaMap() map[string]string { funMap := make(map[string]string) funMap["MAX"] = "[@@]" funMap["MIN"] = "[@!]" funMap["ABS"] = "[@#]" funMap["CEIL"] = "[@$]" funMap["COS"] = "[@%]" funMap["FLOOR"] = "[@^]" funMap["MOD"] = "[@&]" funMap["POW"] = "[@*]" funMap["ROUND"] = "[@(]" return funMap } // 处理整个数据 func handleDateSaveDataMap(dateList []string, realSaveDataMap, saveDataMap map[string]map[int]float64, edbInfoIdArr []*data_manage.EdbInfo, emptyType int) { var startDate, endDate string var startDateT, endDateT time.Time if emptyType == 2 || emptyType == 3 { for k, _ := range realSaveDataMap { if k > endDate { endDate = k } if k < startDate || startDate == "" { startDate = k } } startDateT, _ = time.ParseInLocation(utils.FormatDate, startDate, time.Local) endDateT, _ = time.ParseInLocation(utils.FormatDate, endDate, time.Local) } for _, date := range dateList { tmpDataMap := realSaveDataMap[date] for _, edbInfo := range edbInfoIdArr { tmpEdbInfoId := edbInfo.EdbInfoId // 当前指标id // 如果该日期不存在该指标数据,那么需要找寻前后日期的数据,进行填补 if _, ok := tmpDataMap[tmpEdbInfoId]; !ok { //day := 0 //switch edbInfo.Frequency { //case "周度": // day = 7 //case "旬度": // day = 15 //case "月度": // day = 30 //case "季度": // day = 90 //case "年度": // day = 365 //} // 需求池 255 指标运算文案修改,补数据遍历区间修改(2023-3-7 09:37:23修改) switch emptyType { case 0: handleDateDataMap(realSaveDataMap, saveDataMap, date, tmpEdbInfoId, 35) case 2: handleDateDataMapBefore(realSaveDataMap, saveDataMap, date, tmpEdbInfoId, startDateT, endDateT) case 3: handleDateDataMapAfter(realSaveDataMap, saveDataMap, date, tmpEdbInfoId, startDateT, endDateT) case 4: handleDateDataMapZero(saveDataMap, date, tmpEdbInfoId) } } } } } // handleDataMap 处理单个日期的数据 func handleDateDataMap(realSaveDataMap, saveDataMap map[string]map[int]float64, date string, edbInfoId, day int) { currDate, _ := time.ParseInLocation(utils.FormatDate, date, time.Local) // 后一天 nextDateDayStr := currDate.AddDate(0, 0, 1).Format(utils.FormatDate) // 前一天 preDateDayStr := currDate.AddDate(0, 0, -1).Format(utils.FormatDate) for i := 1; i <= day; i++ { // 下个日期的数据 { if i >= 1 { nextDateDayStr = currDate.AddDate(0, 0, i).Format(utils.FormatDate) } if findDataMap, hasFindDataMap := realSaveDataMap[nextDateDayStr]; hasFindDataMap { // 下一个日期有数据 if val, hasFindItem := findDataMap[edbInfoId]; hasFindItem { saveDataMap[date][edbInfoId] = val return } } } // 上个日期的数据 { if i >= 1 { preDateDayStr = currDate.AddDate(0, 0, -i).Format(utils.FormatDate) } if findDataMap, hasFindDataMap := realSaveDataMap[preDateDayStr]; hasFindDataMap { // 下一个日期有数据 if val, hasFindItem := findDataMap[edbInfoId]; hasFindItem { saveDataMap[date][edbInfoId] = val return } } } } } // handleDataByLinearRegression 插值法补充数据(线性方程式) func handleDataByLinearRegression(edbInfoDataList []*data_manage.EdbDataList, handleDataMap map[string]float64) (err error) { if len(edbInfoDataList) < 2 { return } var startEdbInfoData *data_manage.EdbDataList for _, v := range edbInfoDataList { handleDataMap[v.DataTime] = v.Value // 第一个数据就给过滤了,给后面的试用 if startEdbInfoData == nil { startEdbInfoData = v continue } // 获取两条数据之间相差的天数 startDataTime, _ := time.ParseInLocation(utils.FormatDate, startEdbInfoData.DataTime, time.Local) currDataTime, _ := time.ParseInLocation(utils.FormatDate, v.DataTime, time.Local) betweenHour := int(currDataTime.Sub(startDataTime).Hours()) betweenDay := betweenHour / 24 // 如果相差一天,那么过滤 if betweenDay <= 1 { startEdbInfoData = v continue } // 生成线性方程式 var a, b float64 { coordinateData := make([]utils.Coordinate, 0) tmpCoordinate1 := utils.Coordinate{ X: 1, Y: startEdbInfoData.Value, } coordinateData = append(coordinateData, tmpCoordinate1) tmpCoordinate2 := utils.Coordinate{ X: float64(betweenDay) + 1, Y: v.Value, } coordinateData = append(coordinateData, tmpCoordinate2) a, b = utils.GetLinearResult(coordinateData) if math.IsNaN(a) || math.IsNaN(b) { err = errors.New("线性方程公式生成失败") return } } // 生成对应的值 { for i := 1; i < betweenDay; i++ { tmpDataTime := startDataTime.AddDate(0, 0, i) aDecimal := decimal.NewFromFloat(a) xDecimal := decimal.NewFromInt(int64(i) + 1) bDecimal := decimal.NewFromFloat(b) val, _ := aDecimal.Mul(xDecimal).Add(bDecimal).Round(4).Float64() handleDataMap[tmpDataTime.Format(utils.FormatDate)] = val } } startEdbInfoData = v } return } // HandleDataByLinearRegression 插值法补充数据(线性方程式) func HandleDataByLinearRegression(edbInfoDataList []*data_manage.EdbDataList, handleDataMap map[string]float64) (err error) { return handleDataByLinearRegression(edbInfoDataList, handleDataMap) } // CallCalculateComputeCorrelation 调用计算拟合残差的相关系数 func CallCalculateComputeCorrelation(data *data_manage.EdbInfoCalculateBatchSaveReqByEdbLib, lang string) (val string, err error, errMsg string) { errMsg = "计算失败" // 调用指标库去更新 reqJson, err := json.Marshal(data) if err != nil { errMsg = "计算相关系数参数解析异常!" err = errors.New("参数解析失败,Err:" + err.Error()) return } respItem, err := CalculateComputeCorrelation(string(reqJson), lang) if err != nil { return } if respItem.Ret == 200 { val = respItem.Data } return } // handleDateDataMapBefore 前值填充:空值优先以最近的前值填充,没有前值时,用后值填充 func handleDateDataMapBefore(realSaveDataMap, saveDataMap map[string]map[int]float64, date string, edbInfoId int, startDateT, endDateT time.Time) { currDate, _ := time.ParseInLocation(utils.FormatDate, date, time.Local) // 后一天 nextDateDay := currDate // 前一天 preDateDay := currDate for i := 1; preDateDay.After(startDateT) || preDateDay == startDateT; i++ { // 上个日期的数据 { preDateDay = currDate.AddDate(0, 0, -i) preDateDayStr := preDateDay.Format(utils.FormatDate) if findDataMap, hasFindDataMap := realSaveDataMap[preDateDayStr]; hasFindDataMap { // 下一个日期有数据 if val, hasFindItem := findDataMap[edbInfoId]; hasFindItem { fmt.Println(fmt.Sprintf("date:%s, 无值,取%s的值%.4f", date, preDateDayStr, val)) saveDataMap[date][edbInfoId] = val return } } } } for i := 1; nextDateDay.Before(endDateT) || nextDateDay == endDateT; i++ { // 下个日期的数据 { nextDateDay = currDate.AddDate(0, 0, i) nextDateDayStr := nextDateDay.Format(utils.FormatDate) if findDataMap, hasFindDataMap := realSaveDataMap[nextDateDayStr]; hasFindDataMap { // 下一个日期有数据 if val, hasFindItem := findDataMap[edbInfoId]; hasFindItem { fmt.Println(fmt.Sprintf("date:%s, 无值,取%s的值%.4f", date, nextDateDayStr, val)) saveDataMap[date][edbInfoId] = val return } } } } return } // handleDateDataMapAfter 后值填充:空值优先以最近的后值填充,没有后值时,用前值填充 func handleDateDataMapAfter(realSaveDataMap, saveDataMap map[string]map[int]float64, date string, edbInfoId int, startDateT, endDateT time.Time) { currDate, _ := time.ParseInLocation(utils.FormatDate, date, time.Local) // 后一天 nextDateDay := currDate // 前一天 preDateDay := currDate for i := 1; nextDateDay.Before(endDateT) || nextDateDay == endDateT; i++ { // 下个日期的数据 { nextDateDay = currDate.AddDate(0, 0, i) nextDateDayStr := nextDateDay.Format(utils.FormatDate) if findDataMap, hasFindDataMap := realSaveDataMap[nextDateDayStr]; hasFindDataMap { // 下一个日期有数据 if val, hasFindItem := findDataMap[edbInfoId]; hasFindItem { fmt.Println(fmt.Sprintf("date:%s, 无值,取%s的值%.4f", date, nextDateDayStr, val)) saveDataMap[date][edbInfoId] = val return } } } } for i := 1; preDateDay.After(startDateT) || preDateDay == startDateT; i++ { // 上个日期的数据 { preDateDay = currDate.AddDate(0, 0, -i) preDateDayStr := preDateDay.Format(utils.FormatDate) if findDataMap, hasFindDataMap := realSaveDataMap[preDateDayStr]; hasFindDataMap { // 下一个日期有数据 if val, hasFindItem := findDataMap[edbInfoId]; hasFindItem { fmt.Println(fmt.Sprintf("date:%s, 无值,取%s的值%.4f", date, preDateDayStr, val)) saveDataMap[date][edbInfoId] = val return } } } } return } // handleDateDataMapZero 等于0 func handleDateDataMapZero(saveDataMap map[string]map[int]float64, date string, edbInfoId int) { saveDataMap[date][edbInfoId] = 0 return } func GetMaxMinEdbInfo(formula string) string { //formula := "A+min(A,B,max(A,C))" // todo 无法处理max里嵌套max或者min的情况 // 使用正则表达式匹配MAX和MIN函数及其参数 regex := regexp.MustCompile(`(?i)(MAX|MIN)\((.*?)\)`) matches := regex.FindAllStringSubmatch(formula, -1) // 遍历匹配结果,输出MAX和MIN函数及其参数 for _, match := range matches { if len(match) == 3 { parameter := strings.ToLower(match[0]) // 参数 formula = strings.ReplaceAll(formula, match[0], parameter) fmt.Printf("formula: %s\n", formula) } } formula = strings.ReplaceAll(formula, "max", "MAX") formula = strings.ReplaceAll(formula, "min", "MIN") return formula } // HandleDataByLinearRegressionToList 插值法补充数据(线性方程式) func HandleDataByLinearRegressionToList(edbInfoDataList []*data_manage.EdbDataList, handleDataMap map[string]float64) (dataTimeList []string, valueList []float64, err error) { if len(edbInfoDataList) < 2 { return } var startEdbInfoData *data_manage.EdbDataList for _, v := range edbInfoDataList { handleDataMap[v.DataTime] = v.Value dataTimeList = append(dataTimeList, v.DataTime) // 第一个数据就给过滤了,给后面的试用 if startEdbInfoData == nil { startEdbInfoData = v //startEdbInfoData.DataTime = startEdbInfoData.DataTime[:5]+ "01-01" continue } // 获取两条数据之间相差的天数 startDataTime, _ := time.ParseInLocation(utils.FormatDate, startEdbInfoData.DataTime, time.Local) currDataTime, _ := time.ParseInLocation(utils.FormatDate, v.DataTime, time.Local) betweenHour := int(currDataTime.Sub(startDataTime).Hours()) betweenDay := betweenHour / 24 // 如果相差一天,那么过滤 if betweenDay <= 1 { startEdbInfoData = v continue } // 生成线性方程式 var a, b float64 { coordinateData := make([]utils.Coordinate, 0) tmpCoordinate1 := utils.Coordinate{ X: 1, Y: startEdbInfoData.Value, } coordinateData = append(coordinateData, tmpCoordinate1) tmpCoordinate2 := utils.Coordinate{ X: float64(betweenDay) + 1, Y: v.Value, } coordinateData = append(coordinateData, tmpCoordinate2) a, b = utils.GetLinearResult(coordinateData) if math.IsNaN(a) || math.IsNaN(b) { err = errors.New("线性方程公式生成失败") return } } // 生成对应的值 { for i := 1; i < betweenDay; i++ { tmpDataTime := startDataTime.AddDate(0, 0, i) aDecimal := decimal.NewFromFloat(a) xDecimal := decimal.NewFromInt(int64(i) + 1) bDecimal := decimal.NewFromFloat(b) val, _ := aDecimal.Mul(xDecimal).Add(bDecimal).Round(4).Float64() handleDataMap[tmpDataTime.Format(utils.FormatDate)] = val dataTimeList = append(dataTimeList, tmpDataTime.Format(utils.FormatDate)) valueList = append(valueList, val) } } startEdbInfoData = v } return } // HandleDataByLinearRegressionToList 保证生成365个数据点的线性插值法 func HandleDataByLinearRegressionToListV2(edbInfoDataList []*data_manage.EdbDataList, handleDataMap map[string]float64) (dataTimeList []string, valueList []float64, err error) { if len(edbInfoDataList) < 2 { return } // 确保至少有两天数据来生成线性方程 if len(edbInfoDataList) < 2 { err = errors.New("至少需要两天的数据来执行线性插值") return } // 对数据按日期排序,确保顺序正确 sort.Slice(edbInfoDataList, func(i, j int) bool { t1, _ := time.ParseInLocation(utils.FormatDate, edbInfoDataList[i].DataTime, time.Local) t2, _ := time.ParseInLocation(utils.FormatDate, edbInfoDataList[j].DataTime, time.Local) return t1.Before(t2) }) startEdbInfoData := edbInfoDataList[0] endEdbInfoData := edbInfoDataList[len(edbInfoDataList)-1] // 计算起始和结束日期间实际的天数 startDate, _ := time.ParseInLocation(utils.FormatDate, startEdbInfoData.DataTime, time.Local) endDate, _ := time.ParseInLocation(utils.FormatDate, endEdbInfoData.DataTime, time.Local) actualDays := endDate.Sub(startDate).Hours() / 24 // 生成365个数据点,首先处理已有数据 for _, v := range edbInfoDataList { handleDataMap[v.DataTime] = v.Value dataTimeList = append(dataTimeList, v.DataTime) valueList = append(valueList, v.Value) } // 如果已有数据跨越天数不足365天,则对缺失的日期进行线性插值 if actualDays < 365 { // 使用已有数据点生成线性方程(这里简化处理,实际可能需更细致处理边界情况) var a, b float64 coordinateData := []utils.Coordinate{ {X: 1, Y: startEdbInfoData.Value}, {X: float64(len(edbInfoDataList)), Y: endEdbInfoData.Value}, } a, b = utils.GetLinearResult(coordinateData) if math.IsNaN(a) || math.IsNaN(b) { err = errors.New("线性方程公式生成失败") return } // 对剩余日期进行插值 for i := 1; i < 365; i++ { day := startDate.AddDate(0, 0, i) if _, exists := handleDataMap[day.Format(utils.FormatDate)]; !exists { aDecimal := decimal.NewFromFloat(a) xDecimal := decimal.NewFromInt(int64(i) + 1) bDecimal := decimal.NewFromFloat(b) val, _ := aDecimal.Mul(xDecimal).Add(bDecimal).Round(4).Float64() handleDataMap[day.Format(utils.FormatDate)] = val dataTimeList = append(dataTimeList, day.Format(utils.FormatDate)) valueList = append(valueList, val) } } } return } // HandleDataByLinearRegressionToListV3 插值法补充数据(线性方程式)-直接补充指标起始日期间的所有数据 func HandleDataByLinearRegressionToListV3(edbInfoDataList []*data_manage.EdbDataList, handleDataMap map[string]float64) (newEdbInfoDataList []*data_manage.EdbDataList, dataTimeList []string, valueList []float64, err error) { if len(edbInfoDataList) < 2 { return } var startEdbInfoData *data_manage.EdbDataList for _, v := range edbInfoDataList { handleDataMap[v.DataTime] = v.Value newEdbInfoDataList = append(newEdbInfoDataList, v) dataTimeList = append(dataTimeList, v.DataTime) // 第一个数据就给过滤了,给后面的试用 if startEdbInfoData == nil { startEdbInfoData = v //startEdbInfoData.DataTime = startEdbInfoData.DataTime[:5]+ "01-01" continue } // 获取两条数据之间相差的天数 startDataTime, _ := time.ParseInLocation(utils.FormatDate, startEdbInfoData.DataTime, time.Local) currDataTime, _ := time.ParseInLocation(utils.FormatDate, v.DataTime, time.Local) betweenHour := int(currDataTime.Sub(startDataTime).Hours()) betweenDay := betweenHour / 24 // 如果相差一天,那么过滤 if betweenDay <= 1 { startEdbInfoData = v continue } // 生成线性方程式 var a, b float64 { coordinateData := make([]utils.Coordinate, 0) tmpCoordinate1 := utils.Coordinate{ X: 1, Y: startEdbInfoData.Value, } coordinateData = append(coordinateData, tmpCoordinate1) tmpCoordinate2 := utils.Coordinate{ X: float64(betweenDay) + 1, Y: v.Value, } coordinateData = append(coordinateData, tmpCoordinate2) a, b = utils.GetLinearResult(coordinateData) if math.IsNaN(a) || math.IsNaN(b) { err = errors.New("线性方程公式生成失败") return } } // 生成对应的值 { for i := 1; i < betweenDay; i++ { tmpDataTime := startDataTime.AddDate(0, 0, i) aDecimal := decimal.NewFromFloat(a) xDecimal := decimal.NewFromInt(int64(i) + 1) bDecimal := decimal.NewFromFloat(b) val, _ := aDecimal.Mul(xDecimal).Add(bDecimal).Round(4).Float64() handleDataMap[tmpDataTime.Format(utils.FormatDate)] = val dataTimeList = append(dataTimeList, tmpDataTime.Format(utils.FormatDate)) valueList = append(valueList, val) newEdbInfoDataList = append(newEdbInfoDataList, &data_manage.EdbDataList{ EdbDataId: v.EdbDataId, EdbInfoId: v.EdbInfoId, DataTime: tmpDataTime.Format(utils.FormatDate), DataTimestamp: tmpDataTime.UnixNano() / 1e6, Value: val, }) } } startEdbInfoData = v } return }