Merge branch 'feature/eta1.2.0_edb_calculate' into debug

# Conflicts:
#	models/data_manage/edb_info.go

controllers/data_manage/predict_edb_info.go

@@ -1496,7 +1496,7 @@ func (this *PredictEdbInfoController) ChartDataList() {
 				return
 			}
 			// 获取计算数据
-			tmpDataList, err = data.CalculateByRuleByNine(formula, edbInfoList, edbInfoIdBytes)
+			tmpDataList, err = data.CalculateByRuleByNine(formula, edbInfoList, edbInfoIdBytes, v.EmptyType, v.MaxEmptyType)
 			if err != nil {
 				br.Msg = "计算失败"
 				br.ErrMsg = err.Error()
@@ -1663,7 +1663,7 @@ func (this *PredictEdbInfoController) PredictRuleCalculateByNine() {
 		return
 	}
 	// 获取计算数据
-	dataList, err := data.CalculateByRuleByNine(formula, edbInfoList, edbInfoIdBytes)
+	dataList, err := data.CalculateByRuleByNine(formula, edbInfoList, edbInfoIdBytes, req.EmptyType, req.MaxEmptyType)
 	if err != nil {
 		br.Msg = "数据计算失败"
 		br.ErrMsg = "数据计算失败:Err:" + err.Error()

models/data_manage/edb_info.go

@@ -55,6 +55,8 @@ type EdbInfo struct {
 	SubSourceName    string  `description:"子数据来源名称"`
 	IndicatorCode    string  `description:"指标代码"`
 	StockCode        string  `description:"证券代码"`
+	EmptyType        int     `description:"空值处理类型（0查找前后35天，1不计算，2前值填充，3后值填充，4等于0）"`
+	MaxEmptyType     int     `description:"MAX、MIN公式空值处理类型（1、等于0；2、跳过空值）"`
 }
 
 type EdbInfoFullClassify struct {

services/data/edb_info_calculate.go

@@ -9,6 +9,8 @@ import (
 	"github.com/shopspring/decimal"
 	"github.com/yidane/formula"
 	"math"
+	"regexp"
+	"sort"
 	"strconv"
 	"strings"
 	"time"
@@ -62,6 +64,31 @@ type FormulaListItem struct {
 	Date    string `json:"d"`
 }
 
+// HandleFormulaJson 处理计算公式json串是否异常
+func HandleFormulaJson(formula string, startDate string) (dateSlice []string, formulaMap map[string]string, err error) {
+	list := make([]FormulaListItem, 0)
+	err = json.Unmarshal([]byte(formula), &list)
+	if err != nil {
+		err = fmt.Errorf("公式串解析失败： json.Unmarshal Err: %v", err)
+		return
+	}
+	formulaMap = make(map[string]string)
+	dateSlice = make([]string, 0)
+	// 日期排序
+	for k, v := range list {
+		if k == 0 { // 首个日期均为起始日
+			v.Date = startDate
+		}
+		formulaMap[v.Date] = v.Formula
+		dateSlice = append(dateSlice, v.Date)
+	}
+	sort.Slice(dateSlice, func(i, j int) bool {
+		return dateSlice[i] > dateSlice[j]
+	})
+
+	return
+}
+
 // CheckFormulaJson 检测计算公式json串是否异常
 func CheckFormulaJson(formula string) (formulaSlice []string, err error) {
 	list := make([]FormulaListItem, 0)
@@ -83,91 +110,8 @@ type CalculateItems struct {
 	DataMap   map[string]float64
 }
 
-func Calculate(edbInfoIdArr []*data_manage.EdbInfo, edbInfoId int, edbCode, formulaStr string, edbInfoIdBytes []string) (err error) {
-	defer func() {
-		if err != nil {
-			utils.FileLog.Info("Calculate Err:%s" + err.Error())
-		}
-	}()
-	saveDataMap := make(map[string]map[int]float64)
-	for _, v := range edbInfoIdArr {
-		var condition string
-		var pars []interface{}
-		condition += " AND edb_info_id=? "
-		pars = append(pars, v.EdbInfoId)
-		dataList, err := data_manage.GetEdbDataListAll(condition, pars, v.Source, 1)
-		if err != nil {
-			return err
-		}
-		dataMap := make(map[string]float64)
-		for _, dv := range dataList {
-			if val, ok := saveDataMap[dv.DataTime]; ok {
-				if _, ok := val[v.EdbInfoId]; !ok {
-					val[v.EdbInfoId] = dv.Value
-				}
-			} else {
-				temp := make(map[int]float64)
-				temp[v.EdbInfoId] = dv.Value
-				saveDataMap[dv.DataTime] = temp
-			}
-		}
-		item := new(CalculateItems)
-		item.EdbInfoId = v.EdbInfoId
-		item.DataMap = dataMap
-	}
-	formulaMap := CheckFormula(formulaStr)
-	addSql := ` INSERT INTO edb_data_calculate(edb_info_id,edb_code,data_time,value,create_time,modify_time,status,data_timestamp) values `
-	nowStr := time.Now().Format(utils.FormatDateTime)
-	var isAdd bool
-	for sk, sv := range saveDataMap {
-		formulaStr = strings.ToUpper(formulaStr)
-		formulaFormStr := ReplaceFormula(edbInfoIdArr, sv, formulaMap, formulaStr, edbInfoIdBytes)
-		if formulaStr == "" {
-			return
-		}
-		if formulaFormStr != "" {
-			expression := formula.NewExpression(formulaFormStr)
-			calResult, err := expression.Evaluate()
-			if err != nil {
-				err = errors.New("计算失败：Err:" + err.Error() + ";formulaStr:" + formulaFormStr)
-				fmt.Println(err)
-				return err
-			}
-			calVal, err := calResult.Float64()
-			if err != nil {
-				err = errors.New("计算失败：获取计算值失败 Err:" + err.Error() + ";formulaStr:" + formulaFormStr)
-				fmt.Println(err)
-				return err
-			}
-
-			//需要存入的数据
-			{
-				dataTime, _ := time.Parse(utils.FormatDate, sk)
-				timestamp := dataTime.UnixNano() / 1e6
-				timeStr := fmt.Sprintf("%d", timestamp)
-				addSql += "("
-				addSql += strconv.Itoa(edbInfoId) + "," + "'" + edbCode + "'" + "," + "'" + sk + "'" + "," + utils.SubFloatToString(calVal, 4) + "," + "'" + nowStr + "'" +
-					"," + "'" + nowStr + "'" + "," + "1"
-				addSql += "," + "'" + timeStr + "'"
-				addSql += "),"
-				isAdd = true
-			}
-		} else {
-			fmt.Println("formulaFormStr is empty")
-		}
-	}
-	if isAdd {
-		addSql = strings.TrimRight(addSql, ",")
-		data_manage.AddEdbDataCalculateBySql(addSql)
-		if err != nil {
-			fmt.Println("AddEdbDataCalculate Err:" + err.Error())
-			return err
-		}
-	}
-	return
-}
-
 func ReplaceFormula(edbInfoIdArr []*data_manage.EdbInfo, valArr map[int]float64, formulaMap map[string]string, formulaStr string, edbInfoIdBytes []string) string {
+	// todo 处理min和max
 	funMap := GetFormulaMap()
 	for k, v := range funMap {
 		formulaStr = strings.Replace(formulaStr, k, v, -1)
@@ -569,7 +513,22 @@ func RefreshCalculate(edbInfoIdArr []*data_manage.EdbInfo, edbInfoId int, edbCod
 }
 
 // 处理整个数据
-func handleDateSaveDataMap(dateList []string, realSaveDataMap, saveDataMap map[string]map[int]float64, edbInfoIdArr []*data_manage.EdbInfo) {
+func handleDateSaveDataMap(dateList []string, realSaveDataMap, saveDataMap map[string]map[int]float64, edbInfoIdArr []*data_manage.EdbInfo, emptyType int) {
+	var startDate, endDate string
+	var startDateT, endDateT time.Time
+	if emptyType == 2 || emptyType == 3 {
+		for k, _ := range realSaveDataMap {
+			if k > endDate {
+				endDate = k
+			}
+			if k < startDate || startDate == "" {
+				startDate = k
+			}
+		}
+
+		startDateT, _ = time.ParseInLocation(utils.FormatDate, startDate, time.Local)
+		endDateT, _ = time.ParseInLocation(utils.FormatDate, endDate, time.Local)
+	}
 	for _, date := range dateList {
 		tmpDataMap := realSaveDataMap[date]
 		for _, edbInfo := range edbInfoIdArr {
@@ -590,8 +549,16 @@ func handleDateSaveDataMap(dateList []string, realSaveDataMap, saveDataMap map[s
 				//	day = 365
 				//}
 				// 需求池 255 指标运算文案修改，补数据遍历区间修改（2023-3-7 09:37:23修改）
-				day := 35
-				handleDateDataMap(realSaveDataMap, saveDataMap, date, tmpEdbInfoId, day)
+				switch emptyType {
+				case 0:
+					handleDateDataMap(realSaveDataMap, saveDataMap, date, tmpEdbInfoId, 35)
+				case 2:
+					handleDateDataMapBefore(realSaveDataMap, saveDataMap, date, tmpEdbInfoId, startDateT, endDateT)
+				case 3:
+					handleDateDataMapAfter(realSaveDataMap, saveDataMap, date, tmpEdbInfoId, startDateT, endDateT)
+				case 4:
+					handleDateDataMapZero(saveDataMap, date, tmpEdbInfoId)
+				}
 			}
 		}
 	}
@@ -732,3 +699,112 @@ func CallCalculateComputeCorrelation(data *data_manage.EdbInfoCalculateBatchSave
 
 	return
 }
+
+// handleDateDataMapBefore 前值填充：空值优先以最近的前值填充，没有前值时，用后值填充
+func handleDateDataMapBefore(realSaveDataMap, saveDataMap map[string]map[int]float64, date string, edbInfoId int, startDateT, endDateT time.Time) {
+	currDate, _ := time.ParseInLocation(utils.FormatDate, date, time.Local)
+
+	// 后一天
+	nextDateDay := currDate
+
+	// 前一天
+	preDateDay := currDate
+
+	for i := 1; preDateDay.After(startDateT) || preDateDay == startDateT; i++ {
+		// 上个日期的数据
+		{
+			preDateDay = currDate.AddDate(0, 0, -i)
+			preDateDayStr := preDateDay.Format(utils.FormatDate)
+			if findDataMap, hasFindDataMap := realSaveDataMap[preDateDayStr]; hasFindDataMap { // 下一个日期有数据
+				if val, hasFindItem := findDataMap[edbInfoId]; hasFindItem {
+					fmt.Println(fmt.Sprintf("date:%s, 无值，取%s的值%.4f", date, preDateDayStr, val))
+					saveDataMap[date][edbInfoId] = val
+					return
+				}
+			}
+		}
+	}
+
+	for i := 1; nextDateDay.Before(endDateT) || nextDateDay == endDateT; i++ {
+		// 下个日期的数据
+		{
+			nextDateDay = currDate.AddDate(0, 0, i)
+			nextDateDayStr := nextDateDay.Format(utils.FormatDate)
+			if findDataMap, hasFindDataMap := realSaveDataMap[nextDateDayStr]; hasFindDataMap { // 下一个日期有数据
+				if val, hasFindItem := findDataMap[edbInfoId]; hasFindItem {
+					fmt.Println(fmt.Sprintf("date:%s, 无值，取%s的值%.4f", date, nextDateDayStr, val))
+					saveDataMap[date][edbInfoId] = val
+					return
+				}
+			}
+		}
+	}
+	return
+}
+
+// handleDateDataMapAfter 后值填充：空值优先以最近的后值填充，没有后值时，用前值填充
+func handleDateDataMapAfter(realSaveDataMap, saveDataMap map[string]map[int]float64, date string, edbInfoId int, startDateT, endDateT time.Time) {
+	currDate, _ := time.ParseInLocation(utils.FormatDate, date, time.Local)
+
+	// 后一天
+	nextDateDay := currDate
+
+	// 前一天
+	preDateDay := currDate
+
+	for i := 1; nextDateDay.Before(endDateT) || nextDateDay == endDateT; i++ {
+		// 下个日期的数据
+		{
+			nextDateDay = currDate.AddDate(0, 0, i)
+			nextDateDayStr := nextDateDay.Format(utils.FormatDate)
+			if findDataMap, hasFindDataMap := realSaveDataMap[nextDateDayStr]; hasFindDataMap { // 下一个日期有数据
+				if val, hasFindItem := findDataMap[edbInfoId]; hasFindItem {
+					fmt.Println(fmt.Sprintf("date:%s, 无值，取%s的值%.4f", date, nextDateDayStr, val))
+					saveDataMap[date][edbInfoId] = val
+					return
+				}
+			}
+		}
+	}
+
+	for i := 1; preDateDay.After(startDateT) || preDateDay == startDateT; i++ {
+		// 上个日期的数据
+		{
+			preDateDay = currDate.AddDate(0, 0, -i)
+			preDateDayStr := preDateDay.Format(utils.FormatDate)
+			if findDataMap, hasFindDataMap := realSaveDataMap[preDateDayStr]; hasFindDataMap { // 下一个日期有数据
+				if val, hasFindItem := findDataMap[edbInfoId]; hasFindItem {
+					fmt.Println(fmt.Sprintf("date:%s, 无值，取%s的值%.4f", date, preDateDayStr, val))
+					saveDataMap[date][edbInfoId] = val
+					return
+				}
+			}
+		}
+	}
+	return
+}
+
+// handleDateDataMapZero 等于0
+func handleDateDataMapZero(saveDataMap map[string]map[int]float64, date string, edbInfoId int) {
+	saveDataMap[date][edbInfoId] = 0
+	return
+}
+
+func GetMaxMinEdbInfo(formula string) string {
+	//formula := "A+min(A,B,max(A,C))"
+	// todo 无法处理max里嵌套max或者min的情况
+	// 使用正则表达式匹配MAX和MIN函数及其参数
+	regex := regexp.MustCompile(`(?i)(MAX|MIN)\((.*?)\)`)
+	matches := regex.FindAllStringSubmatch(formula, -1)
+	// 遍历匹配结果，输出MAX和MIN函数及其参数
+	for _, match := range matches {
+		if len(match) == 3 {
+			parameter := strings.ToLower(match[0]) // 参数
+			formula = strings.ReplaceAll(formula, match[0], parameter)
+			fmt.Printf("formula: %s\n", formula)
+		}
+	}
+	formula = strings.ReplaceAll(formula, "max", "MAX")
+	formula = strings.ReplaceAll(formula, "min", "MIN")
+	return formula
+}

services/data/predict_edb_info.go

@@ -1175,11 +1175,13 @@ func GetCalculateByRuleByNineParams(req request.RuleConfig) (formula string, edb
 }
 
 // CalculateByRuleByNine 动态环差规则计算入库
-func CalculateByRuleByNine(formulaStr string, edbInfoList []*data_manage.EdbInfo, edbInfoIdBytes []string) (dataList []*data_manage.EdbDataList, err error) {
+func CalculateByRuleByNine(formulaStr string, edbInfoList []*data_manage.EdbInfo, edbInfoIdBytes []string, emptyType, maxEmptyType int) (dataList []*data_manage.EdbDataList, err error) {
 	realSaveDataMap := make(map[string]map[int]float64)
 	saveDataMap := make(map[string]map[int]float64)
 	dateList := make([]string, 0) //日期
 
+	// 最小的结束日期 , 最晚的数据开始日期
+	var minLatestDate, maxStartDate time.Time
 	formulaStr = strings.ToUpper(formulaStr)
 	// 获取关联指标数据
 	for edbInfoIndex, v := range edbInfoList {
@@ -1189,6 +1191,7 @@ func CalculateByRuleByNine(formulaStr string, edbInfoList []*data_manage.EdbInfo
 			return
 		}
 		dataMap := make(map[string]float64)
+		lenData := len(sourceDataList)
 		for _, dv := range sourceDataList {
 			// 实际数据
 			if val, ok := realSaveDataMap[dv.DataTime]; ok {
@@ -1216,25 +1219,72 @@ func CalculateByRuleByNine(formulaStr string, edbInfoList []*data_manage.EdbInfo
 			if edbInfoIndex == 0 {
 				dateList = append(dateList, dv.DataTime)
 			}
+
+			if lenData > 0 {
+				tmpLatestDate, _ := time.ParseInLocation(utils.FormatDate, sourceDataList[lenData-1].DataTime, time.Local)
+				if minLatestDate.IsZero() || minLatestDate.After(tmpLatestDate) {
+					minLatestDate = tmpLatestDate
+				}
+
+				tmpStartDate, _ := time.ParseInLocation(utils.FormatDate, sourceDataList[0].DataTime, time.Local)
+				if maxStartDate.IsZero() || maxStartDate.Before(tmpStartDate) {
+					maxStartDate = tmpStartDate
+				}
+			}
 		}
 		item := new(CalculateItems)
 		item.EdbInfoId = v.EdbInfoId
 		item.DataMap = dataMap
 	}
 
-	//数据处理，将日期内不全的数据做补全
-	handleDateSaveDataMap(dateList, realSaveDataMap, saveDataMap, edbInfoList)
-
+	// todo 数据处理，将日期内不全的数据做填补
+	handleDateSaveDataMap(dateList, realSaveDataMap, saveDataMap, edbInfoList, emptyType)
 	// 添加数据
 	dataList = make([]*data_manage.EdbDataList, 0)
 
 	// 计算规则
-	formulaMap := CheckFormula(formulaStr)
-
+	formulaDateSlice, formulaDateMap, err := HandleFormulaJson(formulaStr, maxStartDate.Format(utils.FormatDate))
+	if err != nil {
+		return
+	}
 	existDataMap := make(map[string]string)
-
+	// 判断是否特殊处理max和min函数
+	/*maxDealFlag := false
+	if emptyType == 4 && maxEmptyType == 2 {
+		maxDealFlag = true
+	}*/
 	for k, date := range dateList {
 		sv := saveDataMap[date]
+		// 当空值处理类型选择了不计算时，只要有一个指标在某个日期没有值(即空值)，则计算指标在该日期没有值
+		if emptyType == 1 {
+			if len(sv) != len(edbInfoList) {
+				continue
+			}
+		}
+		//fmt.Println(sk, sv)
+		// 根据时间范围，选择对应的公式
+		formulaMap := make(map[string]string)
+		formulaStr = ""
+		for _, fv := range formulaDateSlice {
+			if date >= fv {
+				if f, ok := formulaDateMap[fv]; ok {
+					formulaStr = f
+					formulaMap = CheckFormula(formulaStr)
+				}
+				break
+			}
+		}
+		if formulaStr == "" {
+			continue
+		}
+		/*svMax := make(map[int]float64)
+		if maxDealFlag {
+			// 特殊处理max和min函数，如果原本的值为空，则选择空值参与运算
+			if svMaxData, ok := realSaveDataMap[date]; ok {
+				svMax = svMaxData
+			}
+		}
+		*/
 		//fmt.Println(date, sv)
 
 		formulaFormStr := ReplaceFormula(edbInfoList, sv, formulaMap, formulaStr, edbInfoIdBytes)