Merge branch '11.4'

controllers/base_from_predict_calculate.go

@@ -245,7 +245,7 @@ func addPredictCalculate(br *models.BaseResponse, req models.EdbInfoCalculateSav
 	}
 
 	// 开始添加预测指标
-	latestDateStr, latestValue, err := models.AddPredictCalculateData(edbInfoList, int(edbInfoId), edbCode, req.CalculateFormula, edbInfoIdBytes)
+	latestDateStr, latestValue, err := models.AddPredictCalculateData(edbInfoList, edbInfo, edbCode, req.CalculateFormula, edbInfoIdBytes)
 	if err != nil {
 		br.Msg = "生成计算指标失败"
 		br.Msg = "生成计算指标失败,Calculate Err:" + err.Error()
@@ -449,7 +449,7 @@ func editPredictCalculate(br *models.BaseResponse, req models.EdbInfoCalculateSa
 	}
 
 	// 开始添加预测指标
-	latestDateStr, latestValue, err := models.AddPredictCalculateData(edbInfoList, edbInfo.EdbInfoId, edbInfo.EdbCode, req.CalculateFormula, edbInfoIdBytes)
+	latestDateStr, latestValue, err := models.AddPredictCalculateData(edbInfoList, edbInfo, edbInfo.EdbCode, req.CalculateFormula, edbInfoIdBytes)
 	if err != nil {
 		br.Msg = "生成计算指标失败"
 		br.Msg = "生成计算指标失败,Calculate Err:" + err.Error()

models/base_from_calculate.go

@@ -362,8 +362,10 @@ func RefreshAllCalculate(edbInfoIdArr []*EdbInfo, edbInfoId, source int, edbCode
 
 // refreshAllCalculate 刷新全部数据
 func refreshAllCalculate(to orm.TxOrmer, edbInfoIdArr []*EdbInfo, edbInfoId, source int, edbCode, formulaStr, startDate, endDate string, edbInfoIdBytes []string) (err error) {
+	realSaveDataMap := make(map[string]map[int]float64)
 	saveDataMap := make(map[string]map[int]float64)
-	for _, v := range edbInfoIdArr {
+	dateList := make([]string, 0)
+	for edbInfoIndex, v := range edbInfoIdArr {
 		var condition string
 		var pars []interface{}
 		condition += " AND edb_info_id=? "
@@ -381,23 +383,38 @@ func refreshAllCalculate(to orm.TxOrmer, edbInfoIdArr []*EdbInfo, edbInfoId, sou
 		if err != nil {
 			return err
 		}
-		dataMap := make(map[string]float64)
 		for _, dv := range dataList {
-			if val, ok := saveDataMap[dv.DataTime]; ok {
+			if val, ok := realSaveDataMap[dv.DataTime]; ok {
 				if _, ok := val[v.EdbInfoId]; !ok {
 					val[v.EdbInfoId] = dv.Value
 				}
 			} else {
 				temp := make(map[int]float64)
 				temp[v.EdbInfoId] = dv.Value
-				saveDataMap[dv.DataTime] = temp
+				realSaveDataMap[dv.DataTime] = temp
+			}
+
+			// saveDataMap
+			if val, ok := saveDataMap[dv.DataTime]; ok {
+				if _, ok := val[v.EdbInfoId]; !ok {
+					val[v.EdbInfoId] = dv.Value
+				}
+			} else {
+				temp2 := make(map[int]float64)
+				temp2[v.EdbInfoId] = dv.Value
+				saveDataMap[dv.DataTime] = temp2
+			}
+
+			// 以第一个指标的日期作为基准日期
+			if edbInfoIndex == 0 {
+				dateList = append(dateList, dv.DataTime)
 			}
 		}
-		item := new(CalculateItems)
-		item.EdbInfoId = v.EdbInfoId
-		item.DataMap = dataMap
 	}
 
+	//数据处理，将日期内不全的数据做补全
+	handleDateSaveDataMap(dateList, realSaveDataMap, saveDataMap, edbInfoIdArr)
+
 	formulaMap := services.CheckFormula(formulaStr)
 	addSql := ` INSERT INTO edb_data_calculate(edb_info_id,edb_code,data_time,value,create_time,modify_time,data_timestamp) values `
 	var isAdd bool
@@ -423,54 +440,54 @@ func refreshAllCalculate(to orm.TxOrmer, edbInfoIdArr []*EdbInfo, edbInfoId, sou
 		//fmt.Println(sk, sv)
 		formulaStr = strings.ToUpper(formulaStr)
 		formulaFormStr := ReplaceFormula(edbInfoIdArr, sv, formulaMap, formulaStr, edbInfoIdBytes)
-		if formulaFormStr != "" {
-			utils.FileLog.Info(fmt.Sprintf("formulaFormStr:%s", formulaFormStr))
-			expression := formula.NewExpression(formulaFormStr)
-			calResult, err := expression.Evaluate()
-			if err != nil {
-				// 分母为0的报错
-				if strings.Contains(err.Error(), "divide by zero") {
-					removeDateList = append(removeDateList, sk)
-					continue
-				}
-				err = errors.New("计算失败：Err:" + err.Error() + ";formulaStr:" + formulaFormStr)
-				fmt.Println(err)
-				return err
-			}
-			calVal, err := calResult.Float64()
-			if err != nil {
-				err = errors.New("计算失败：获取计算值失败 Err:" + err.Error() + ";formulaStr:" + formulaFormStr)
-				fmt.Println(err)
-				return err
+		//计算公式异常，那么就移除该指标
+		if formulaFormStr == `` {
+			removeDateList = append(removeDateList, sk)
+			continue
+		}
+		//utils.FileLog.Info(fmt.Sprintf("formulaFormStr:%s", formulaFormStr))
+		fmt.Println(fmt.Sprintf("formulaFormStr:%s", formulaFormStr))
+		expression := formula.NewExpression(formulaFormStr)
+		calResult, err := expression.Evaluate()
+		if err != nil {
+			// 分母为0的报错
+			if strings.Contains(err.Error(), "divide by zero") {
+				removeDateList = append(removeDateList, sk)
+				continue
 			}
+			err = errors.New("计算失败：Err:" + err.Error() + ";formulaStr:" + formulaFormStr)
+			fmt.Println(err)
+			return err
+		}
+		calVal, err := calResult.Float64()
+		if err != nil {
+			err = errors.New("计算失败：获取计算值失败 Err:" + err.Error() + ";formulaStr:" + formulaFormStr)
+			fmt.Println(err)
+			return err
+		}
 
-			saveValue := decimal.NewFromFloat(calVal).RoundCeil(4).String() //utils.SubFloatToString(calVal, 4)
-			if existVal, ok := dataMap[sk]; !ok {
-				dataTime, _ := time.Parse(utils.FormatDate, sk)
-				timestamp := dataTime.UnixNano() / 1e6
-				timeStr := fmt.Sprintf("%d", timestamp)
+		saveValue := decimal.NewFromFloat(calVal).RoundCeil(4).String() //utils.SubFloatToString(calVal, 4)
+		if existVal, ok := dataMap[sk]; !ok {
+			dataTime, _ := time.Parse(utils.FormatDate, sk)
+			timestamp := dataTime.UnixNano() / 1e6
+			timeStr := fmt.Sprintf("%d", timestamp)
 
-				if _, existOk := existDataMap[sk]; !existOk {
-					addSql += GetAddSql(edbInfoIdStr, edbCode, sk, timeStr, saveValue)
-					isAdd = true
-				}
-				existDataMap[sk] = sk
-			} else {
-				existValDecimal, err := decimal.NewFromString(existVal)
-				existStr := existValDecimal.String()
-				if existStr != saveValue {
-					sql := ` UPDATE %s SET value=?,modify_time=NOW() WHERE edb_info_id=? AND data_time=? `
-					sql = fmt.Sprintf(sql, dataTableName)
-					_, err = to.Raw(sql, saveValue, edbInfoId, sk).Exec()
-					if err != nil {
-						return err
-					}
-				}
+			if _, existOk := existDataMap[sk]; !existOk {
+				addSql += GetAddSql(edbInfoIdStr, edbCode, sk, timeStr, saveValue)
+				isAdd = true
 			}
+			existDataMap[sk] = sk
 		} else {
-			//计算公式异常，那么就移除该指标
-			removeDateList = append(removeDateList, sk)
-			continue
+			existValDecimal, err := decimal.NewFromString(existVal)
+			existStr := existValDecimal.String()
+			if existStr != saveValue {
+				sql := ` UPDATE %s SET value=?,modify_time=NOW() WHERE edb_info_id=? AND data_time=? `
+				sql = fmt.Sprintf(sql, dataTableName)
+				_, err = to.Raw(sql, saveValue, edbInfoId, sk).Exec()
+				if err != nil {
+					return err
+				}
+			}
 		}
 	}
 
@@ -609,3 +626,70 @@ func CheckFormula2(edbInfoArr []*EdbInfo, formulaMap map[string]string, formulaS
 	}
 	return
 }
+
+// 处理整个数据
+func handleDateSaveDataMap(dateList []string, realSaveDataMap, saveDataMap map[string]map[int]float64, edbInfoIdArr []*EdbInfo) {
+	for _, date := range dateList {
+		tmpDataMap := realSaveDataMap[date]
+		for _, edbInfo := range edbInfoIdArr {
+			tmpEdbInfoId := edbInfo.EdbInfoId // 当前指标id
+			// 如果该日期不存在该指标数据，那么需要找寻前后日期的数据，进行填补
+			if _, ok := tmpDataMap[tmpEdbInfoId]; !ok {
+				day := 0
+				switch edbInfo.Frequency {
+				case "周度":
+					day = 7
+				case "旬度":
+					day = 15
+				case "月度":
+					day = 30
+				case "季度":
+					day = 90
+				case "年度":
+					day = 365
+				}
+				handleDataMap(realSaveDataMap, saveDataMap, date, tmpEdbInfoId, day)
+			}
+		}
+	}
+}
+
+// handleDataMap 处理单个日期的数据
+func handleDataMap(realSaveDataMap, saveDataMap map[string]map[int]float64, date string, edbInfoId, day int) {
+	currDate, _ := time.ParseInLocation(utils.FormatDate, date, time.Local)
+
+	// 后一天
+	nextDateDayStr := currDate.AddDate(0, 0, 1).Format(utils.FormatDate)
+
+	// 前一天
+	preDateDayStr := currDate.AddDate(0, 0, -1).Format(utils.FormatDate)
+
+	for i := 1; i <= day; i++ {
+
+		// 下个日期的数据
+		{
+			if i >= 1 {
+				nextDateDayStr = currDate.AddDate(0, 0, i).Format(utils.FormatDate)
+			}
+			if findDataMap, hasFindDataMap := realSaveDataMap[nextDateDayStr]; hasFindDataMap { // 下一个日期有数据
+				if val, hasFindItem := findDataMap[edbInfoId]; hasFindItem {
+					saveDataMap[date][edbInfoId] = val
+					return
+				}
+			}
+		}
+
+		// 上个日期的数据
+		{
+			if i >= 1 {
+				preDateDayStr = currDate.AddDate(0, 0, -i).Format(utils.FormatDate)
+			}
+			if findDataMap, hasFindDataMap := realSaveDataMap[preDateDayStr]; hasFindDataMap { // 下一个日期有数据
+				if val, hasFindItem := findDataMap[edbInfoId]; hasFindItem {
+					saveDataMap[date][edbInfoId] = val
+					return
+				}
+			}
+		}
+	}
+}

models/base_predict_from_calculate.go

@@ -91,7 +91,7 @@ func EditPredictCalculate(edbInfo *EdbInfo, calculateMappingList []*EdbInfoCalcu
 }
 
 // AddPredictCalculateData 新增预测计算（运算）指标的数据
-func AddPredictCalculateData(edbInfoIdList []*EdbInfo, edbInfoId int, edbCode, formulaStr string, edbInfoIdBytes []string) (latestDateStr string, latestValue float64, err error) {
+func AddPredictCalculateDataBak(edbInfoIdList []*EdbInfo, edbInfoId int, edbCode, formulaStr string, edbInfoIdBytes []string) (latestDateStr string, latestValue float64, err error) {
 	o := orm.NewOrm()
 	defer func() {
 		if err != nil {
@@ -100,8 +100,11 @@ func AddPredictCalculateData(edbInfoIdList []*EdbInfo, edbInfoId int, edbCode, f
 	}()
 
 	var minLatestDate time.Time
+	oldSaveDataMap := make(map[string]map[int]float64)
 	saveDataMap := make(map[string]map[int]float64)
-	for _, v := range edbInfoIdList {
+	dateList := make([]string, 0)
+
+	for edbInfoIndex, v := range edbInfoIdList {
 		tmpLatestDate, tmpErr := time.ParseInLocation(utils.FormatDate, v.LatestDate, time.Local)
 		if tmpErr != nil {
 			err = errors.New(`最近的日期格式化失败;日期：` + v.LatestDate + `;err:` + tmpErr.Error())
@@ -120,6 +123,18 @@ func AddPredictCalculateData(edbInfoIdList []*EdbInfo, edbInfoId int, edbCode, f
 		}
 		dataMap := make(map[string]float64)
 		for _, dv := range dataList {
+			// 现有实际数据
+			if val, ok := oldSaveDataMap[dv.DataTime]; ok {
+				if _, ok := val[v.EdbInfoId]; !ok {
+					val[v.EdbInfoId] = dv.Value
+				}
+			} else {
+				temp := make(map[int]float64)
+				temp[v.EdbInfoId] = dv.Value
+				oldSaveDataMap[dv.DataTime] = temp
+			}
+
+			// 待处理的数据
 			if val, ok := saveDataMap[dv.DataTime]; ok {
 				if _, ok := val[v.EdbInfoId]; !ok {
 					val[v.EdbInfoId] = dv.Value
@@ -129,11 +144,20 @@ func AddPredictCalculateData(edbInfoIdList []*EdbInfo, edbInfoId int, edbCode, f
 				temp[v.EdbInfoId] = dv.Value
 				saveDataMap[dv.DataTime] = temp
 			}
+
+			// 以第一个指标的日期作为基准日期
+			if edbInfoIndex == 0 {
+				dateList = append(dateList, dv.DataTime)
+			}
 		}
 		item := new(PredictCalculateItems)
 		item.EdbInfoId = v.EdbInfoId
 		item.DataMap = dataMap
 	}
+
+	//数据处理，将日期内不全的数据做补全
+	handleDateSaveDataMap(dateList, oldSaveDataMap, saveDataMap, edbInfoIdList)
+
 	formulaMap := services.CheckFormula(formulaStr)
 	addSql := ` INSERT INTO edb_data_predict_calculate (edb_info_id,edb_code,data_time,value,create_time,modify_time,data_timestamp) values `
 	nowStr := time.Now().Format(utils.FormatDateTime)
@@ -193,6 +217,25 @@ func AddPredictCalculateData(edbInfoIdList []*EdbInfo, edbInfoId int, edbCode, f
 	return
 }
 
+func AddPredictCalculateData(edbInfoIdList []*EdbInfo, edbInfo *EdbInfo, edbCode, formulaStr string, edbInfoIdBytes []string) (latestDateStr string, latestValue float64, err error) {
+	o := orm.NewOrm()
+	to, err := o.Begin()
+	if err != nil {
+		return
+	}
+	defer func() {
+		if err != nil {
+			fmt.Println("AddPredictCalculateData Calculate,Err:" + err.Error())
+			_ = to.Rollback()
+		} else {
+			_ = to.Commit()
+		}
+	}()
+
+	latestDateStr, latestValue, err = refreshAllPredictCalculate(to, edbInfoIdList, edbInfo.EdbInfoId, edbInfo.Source, edbCode, formulaStr, "", edbInfoIdBytes)
+	return
+}
+
 // RefreshAllPredictCalculate 刷新预测计算指标的全部数据
 func RefreshAllPredictCalculate(edbInfoIdList []*EdbInfo, edbInfoId, source int, edbCode, formulaStr, startDate string, edbInfoIdBytes []string) (latestDateStr string, latestValue float64, err error) {
 	o := orm.NewOrm()
@@ -210,9 +253,21 @@ func RefreshAllPredictCalculate(edbInfoIdList []*EdbInfo, edbInfoId, source int,
 	}()
 	fmt.Println(startDate)
 
+	latestDateStr, latestValue, err = refreshAllPredictCalculate(to, edbInfoIdList, edbInfoId, source, edbCode, formulaStr, startDate, edbInfoIdBytes)
+
+	return
+}
+
+// refreshAllPredictCalculate 刷新预测计算指标的全部数据
+func refreshAllPredictCalculate(to orm.TxOrmer, edbInfoIdList []*EdbInfo, edbInfoId, source int, edbCode, formulaStr, startDate string, edbInfoIdBytes []string) (latestDateStr string, latestValue float64, err error) {
+	fmt.Println(startDate)
+
 	var minLatestDate time.Time
+	realSaveDataMap := make(map[string]map[int]float64)
 	saveDataMap := make(map[string]map[int]float64)
-	for _, v := range edbInfoIdList {
+	dateList := make([]string, 0)
+
+	for edbInfoIndex, v := range edbInfoIdList {
 		tmpLatestDate, tmpErr := time.ParseInLocation(utils.FormatDate, v.LatestDate, time.Local)
 		if tmpErr != nil {
 			err = errors.New(`最近的日期格式化失败;日期：` + v.LatestDate + `;err:` + tmpErr.Error())
@@ -229,7 +284,18 @@ func RefreshAllPredictCalculate(edbInfoIdList []*EdbInfo, edbInfoId, source int,
 			return
 		}
 		dataMap := make(map[string]float64)
-		for _, dv := range dataList {
+		for _, dv := range dataList { // 现有实际数据
+			if val, ok := realSaveDataMap[dv.DataTime]; ok {
+				if _, ok := val[v.EdbInfoId]; !ok {
+					val[v.EdbInfoId] = dv.Value
+				}
+			} else {
+				temp := make(map[int]float64)
+				temp[v.EdbInfoId] = dv.Value
+				realSaveDataMap[dv.DataTime] = temp
+			}
+
+			// 待处理的数据
 			if val, ok := saveDataMap[dv.DataTime]; ok {
 				if _, ok := val[v.EdbInfoId]; !ok {
 					val[v.EdbInfoId] = dv.Value
@@ -239,12 +305,20 @@ func RefreshAllPredictCalculate(edbInfoIdList []*EdbInfo, edbInfoId, source int,
 				temp[v.EdbInfoId] = dv.Value
 				saveDataMap[dv.DataTime] = temp
 			}
+
+			// 以第一个指标的日期作为基准日期
+			if edbInfoIndex == 0 {
+				dateList = append(dateList, dv.DataTime)
+			}
 		}
 		item := new(CalculateItems)
 		item.EdbInfoId = v.EdbInfoId
 		item.DataMap = dataMap
 	}
 
+	//数据处理，将日期内不全的数据做补全
+	handleDateSaveDataMap(dateList, realSaveDataMap, saveDataMap, edbInfoIdList)
+
 	formulaMap := services.CheckFormula(formulaStr)
 	addSql := ` INSERT INTO edb_data_predict_calculate (edb_info_id,edb_code,data_time,value,create_time,modify_time,data_timestamp) values `
 	var isAdd bool
@@ -257,7 +331,7 @@ func RefreshAllPredictCalculate(edbInfoIdList []*EdbInfo, edbInfoId, source int,
 	dataTableName := GetEdbDataTableName(source)
 	sql := `SELECT * FROM %s WHERE edb_info_id=? `
 	sql = fmt.Sprintf(sql, dataTableName)
-	_, err = o.Raw(sql, edbInfoId).QueryRows(&dataList)
+	_, err = to.Raw(sql, edbInfoId).QueryRows(&dataList)
 	if err != nil {
 		return
 	}
@@ -320,7 +394,7 @@ func RefreshAllPredictCalculate(edbInfoIdList []*EdbInfo, edbInfoId, source int,
 				if existStr != saveValue {
 					sql := ` UPDATE %s SET value=?,modify_time=NOW() WHERE edb_info_id=? AND data_time=? `
 					sql = fmt.Sprintf(sql, dataTableName)
-					_, err = o.Raw(sql, saveValue, edbInfoId, sk).Exec()
+					_, err = to.Raw(sql, saveValue, edbInfoId, sk).Exec()
 					if err != nil {
 						return
 					}
@@ -335,7 +409,7 @@ func RefreshAllPredictCalculate(edbInfoIdList []*EdbInfo, edbInfoId, source int,
 
 	if isAdd {
 		addSql = strings.TrimRight(addSql, ",")
-		_, err = o.Raw(addSql).Exec()
+		_, err = to.Raw(addSql).Exec()
 		if err != nil {
 			fmt.Println("RefreshAllCalculate add Err", err.Error())
 			return
@@ -349,7 +423,7 @@ func RefreshAllPredictCalculate(edbInfoIdList []*EdbInfo, edbInfoId, source int,
 		tableName := GetEdbDataTableName(source)
 		sql := fmt.Sprintf(` DELETE FROM %s WHERE edb_info_id = ? and data_time in (%s) `, tableName, removeDateStr)
 
-		_, err = o.Raw(sql, edbInfoId).Exec()
+		_, err = to.Raw(sql, edbInfoId).Exec()
 		if err != nil {
 			err = fmt.Errorf("删除计算失败的计算指标数据失败,Err:" + err.Error())
 			return