Merge remote-tracking branch 'origin/master' into eta/1.8.4

controllers/base_from_predict_calculate.go

@@ -141,7 +141,9 @@ func addPredictCalculate(br *models.BaseResponse, req models.EdbInfoCalculateSav
 	// 关联指标信息
 	edbInfoList := make([]*models.EdbInfo, 0)
 	calculateMappingList := make([]*models.EdbInfoCalculateMapping, 0)
+	edbInfoTag := make(map[string]int)
 	for k, v := range req.EdbInfoIdArr {
+		edbInfoTag[v.FromTag] = v.EdbInfoId
 		fromEdbInfo, err := models.GetEdbInfoById(v.EdbInfoId)
 		if err != nil {
 			if err.Error() == utils.ErrNoRow() {
@@ -214,6 +216,7 @@ func addPredictCalculate(br *models.BaseResponse, req models.EdbInfoCalculateSav
 		EdbType:          2,
 		EmptyType:        req.EmptyType,
 		MaxEmptyType:     req.MaxEmptyType,
+		Extra:            req.Extra,
 		EdbNameEn:        req.EdbName,
 		UnitEn:           req.Unit,
 	}
@@ -259,7 +262,7 @@ func addPredictCalculate(br *models.BaseResponse, req models.EdbInfoCalculateSav
 	}
 
 	// 开始添加预测指标
-	latestDateStr, latestValue, err := models.AddPredictCalculateData(edbInfoList, edbInfo, edbCode, req.CalculateFormula, edbInfoIdBytes)
+	latestDateStr, latestValue, err := models.AddPredictCalculateData(edbInfoList, edbInfoTag, edbInfo, edbCode, req.CalculateFormula, edbInfoIdBytes)
 	if err != nil {
 		br.Msg = "生成计算指标失败"
 		br.Msg = "生成计算指标失败,Calculate Err:" + err.Error()
@@ -391,7 +394,9 @@ func editPredictCalculate(br *models.BaseResponse, req models.EdbInfoCalculateSa
 	// 关联指标信息
 	edbInfoList := make([]*models.EdbInfo, 0)
 	calculateMappingList := make([]*models.EdbInfoCalculateMapping, 0)
+	edbInfoTag := make(map[string]int)
 	for k, v := range req.EdbInfoIdArr {
+		edbInfoTag[v.FromTag] = v.EdbInfoId
 		fromEdbInfo, err := models.GetEdbInfoById(v.EdbInfoId)
 		if err != nil {
 			if err.Error() == utils.ErrNoRow() {
@@ -473,6 +478,7 @@ func editPredictCalculate(br *models.BaseResponse, req models.EdbInfoCalculateSa
 	edbInfo.CalculateFormula = req.CalculateFormula
 	edbInfo.EmptyType = req.EmptyType
 	edbInfo.MaxEmptyType = req.MaxEmptyType
+	edbInfo.Extra = req.Extra
 	err = models.EditPredictCalculate(edbInfo, calculateMappingList)
 	if err != nil {
 		br.Msg = "修改计算指标失败"
@@ -481,7 +487,7 @@ func editPredictCalculate(br *models.BaseResponse, req models.EdbInfoCalculateSa
 	}
 
 	// 开始添加预测指标
-	latestDateStr, latestValue, err := models.AddPredictCalculateData(edbInfoList, edbInfo, edbInfo.EdbCode, req.CalculateFormula, edbInfoIdBytes)
+	latestDateStr, latestValue, err := models.AddPredictCalculateData(edbInfoList, edbInfoTag, edbInfo, edbInfo.EdbCode, req.CalculateFormula, edbInfoIdBytes)
 	if err != nil {
 		br.Msg = "生成计算指标失败"
 		br.Msg = "生成计算指标失败,Calculate Err:" + err.Error()
@@ -968,14 +974,16 @@ func (this *PredictCalculateController) Refresh() {
 		}
 		var formulaStr string
 		edbInfoList := make([]*models.EdbInfo, 0)
-
+		edbInfoTag := make(map[string]int)
 		for _, v := range calculateMap {
+			edbInfoTag[v.FromTag] = v.FromEdbInfoId
 			formulaStr += v.FromTag + ","
 			edbInfoIdBytes = append(edbInfoIdBytes, v.FromTag)
 			edbInfo, _ := models.GetEdbInfoById(v.FromEdbInfoId)
 			edbInfoList = append(edbInfoList, edbInfo)
 		}
-		latestDateStr, latestValue, err = models.RefreshAllPredictCalculate(edbInfoList, edbInfo.EdbInfoId, source, edbInfo.SubSource, edbInfo.EdbCode, edbInfo.CalculateFormula, startDate, edbInfoIdBytes, edbInfo.EmptyType, edbInfo.MaxEmptyType)
+
+		latestDateStr, latestValue, err = models.RefreshAllPredictCalculate(edbInfoList, edbInfoTag, edbInfo.EdbInfoId, source, edbInfo.SubSource, edbInfo.EdbCode, edbInfo.CalculateFormula, startDate, edbInfoIdBytes, edbInfo.EmptyType, edbInfo.MaxEmptyType, edbInfo.Extra)
 		if err != nil && err.Error() != utils.ErrNoRow() {
 			errMsg = "RefreshCalculate Err:" + err.Error()
 			break

models/base_predict_from_calculate.go

@@ -1,6 +1,7 @@
 package models
 
 import (
+	"encoding/json"
 	"errors"
 	"eta/eta_index_lib/utils"
 	"fmt"
@@ -59,7 +60,7 @@ func EditPredictCalculate(edbInfo *EdbInfo, calculateMappingList []*EdbInfoCalcu
 	}()
 
 	// 修改指标信息
-	_, err = to.Update(edbInfo, "EdbName", "Frequency", "Unit", "ClassifyId", "CalculateFormula", "EmptyType", "MaxEmptyType", "EdbNameEn", "UnitEn")
+	_, err = to.Update(edbInfo, "EdbName", "Frequency", "Unit", "ClassifyId", "CalculateFormula", "EmptyType", "MaxEmptyType", "Extra", "EdbNameEn", "UnitEn")
 	if err != nil {
 		return
 	}
@@ -223,7 +224,7 @@ func EditPredictCalculate(edbInfo *EdbInfo, calculateMappingList []*EdbInfoCalcu
 //	return
 //}
 
-func AddPredictCalculateData(edbInfoIdList []*EdbInfo, edbInfo *EdbInfo, edbCode, formulaStr string, edbInfoIdBytes []string) (latestDateStr string, latestValue float64, err error) {
+func AddPredictCalculateData(edbInfoIdList []*EdbInfo, edbInfoTag map[string]int, edbInfo *EdbInfo, edbCode, formulaStr string, edbInfoIdBytes []string) (latestDateStr string, latestValue float64, err error) {
 	o := orm.NewOrm()
 	to, err := o.Begin()
 	if err != nil {
@@ -238,12 +239,12 @@ func AddPredictCalculateData(edbInfoIdList []*EdbInfo, edbInfo *EdbInfo, edbCode
 		}
 	}()
 
-	latestDateStr, latestValue, err = refreshAllPredictCalculate(to, edbInfoIdList, edbInfo.EdbInfoId, edbInfo.Source, edbInfo.SubSource, edbCode, formulaStr, "", edbInfoIdBytes, edbInfo.EmptyType, edbInfo.MaxEmptyType)
+	latestDateStr, latestValue, err = refreshAllPredictCalculate(to, edbInfoIdList, edbInfoTag, edbInfo.EdbInfoId, edbInfo.Source, edbInfo.SubSource, edbCode, formulaStr, "", edbInfoIdBytes, edbInfo.EmptyType, edbInfo.MaxEmptyType, edbInfo.Extra)
 	return
 }
 
 // RefreshAllPredictCalculate 刷新预测计算指标的全部数据
-func RefreshAllPredictCalculate(edbInfoIdList []*EdbInfo, edbInfoId, source, subSource int, edbCode, formulaStr, startDate string, edbInfoIdBytes []string, emptyType, maxEmptyType int) (latestDateStr string, latestValue float64, err error) {
+func RefreshAllPredictCalculate(edbInfoIdList []*EdbInfo, edbInfoTag map[string]int, edbInfoId, source, subSource int, edbCode, formulaStr, startDate string, edbInfoIdBytes []string, emptyType, maxEmptyType int, extra string) (latestDateStr string, latestValue float64, err error) {
 	o := orm.NewOrm()
 	to, err := o.Begin()
 	if err != nil {
@@ -258,33 +259,44 @@ func RefreshAllPredictCalculate(edbInfoIdList []*EdbInfo, edbInfoId, source, sub
 		}
 	}()
 
-	latestDateStr, latestValue, err = refreshAllPredictCalculate(to, edbInfoIdList, edbInfoId, source, subSource, edbCode, formulaStr, startDate, edbInfoIdBytes, emptyType, maxEmptyType)
+	latestDateStr, latestValue, err = refreshAllPredictCalculate(to, edbInfoIdList, edbInfoTag, edbInfoId, source, subSource, edbCode, formulaStr, startDate, edbInfoIdBytes, emptyType, maxEmptyType, extra)
 
 	return
 }
 
 // refreshAllPredictCalculate 刷新预测计算指标的全部数据
-func refreshAllPredictCalculate(to orm.TxOrmer, edbInfoIdList []*EdbInfo, edbInfoId, source, subSource int, edbCode, formulaStr, startDate string, edbInfoIdBytes []string, emptyType, maxEmptyType int) (latestDateStr string, latestValue float64, err error) {
+func refreshAllPredictCalculate(to orm.TxOrmer, edbInfoIdList []*EdbInfo, edbInfoTag map[string]int, edbInfoId, source, subSource int, edbCode, formulaStr, startDate string, edbInfoIdBytes []string, emptyType, maxEmptyType int, extra string) (latestDateStr string, latestValue float64, err error) {
 	fmt.Println("startDate:", startDate)
-
 	// 最小的结束日期 , 最晚的数据开始日期
-	var minLatestDate, maxStartDate time.Time
+	var newLatestDate, minLatestDate, maxStartDate time.Time
 	dateList := make([]string, 0) // 第一个指标的日期数据
 
 	realSaveDataMap := make(map[string]map[int]float64)
 	saveDataMap := make(map[string]map[int]float64)
+	dateMap := make(map[string]struct{}) // 最终的日期数据
+
+	dateTagConfig := ""
+	if extra != "" {
+		var dateConfig CalculateEdbExtra
+		err = json.Unmarshal([]byte(extra), &dateConfig)
+		if err != nil {
+			err = fmt.Errorf("refreshAllCalculate,extra解析失败,Err:%s", err.Error())
+			return
+		}
+		dateTagConfig = dateConfig.DateTag
+	}
 
 	for edbInfoIndex, v := range edbInfoIdList {
 		// 单独存储max、min函数里的指标的数据
-		/*tmpLatestDate, tmpErr := time.ParseInLocation(utils.FormatDate, v.LatestDate, time.Local)
+		tmpNewLatestDate, tmpErr := time.ParseInLocation(utils.FormatDate, v.LatestDate, time.Local)
 		if tmpErr != nil {
-			err = errors.New(`最近的日期格式化失败;日期：` + v.EndDate + `;err:` + tmpErr.Error())
+			err = errors.New(`最近的日期格式化失败;日期：` + v.LatestDate + `;err:` + tmpErr.Error())
 			return
 		}
-		if minLatestDate.IsZero() || tmpLatestDate.Before(minLatestDate) {
-			minLatestDate = tmpLatestDate
+		if newLatestDate.IsZero() || tmpNewLatestDate.Before(newLatestDate) {
+			newLatestDate = tmpNewLatestDate
 		}
-		tmpStartDate, tmpErr := time.ParseInLocation(utils.FormatDate, v.StartDate, time.Local)
+		/*tmpStartDate, tmpErr := time.ParseInLocation(utils.FormatDate, v.StartDate, time.Local)
 		if tmpErr != nil {
 			err = errors.New(`最近的日期格式化失败;日期：` + v.StartDate + `;err:` + tmpErr.Error())
 			return
@@ -321,7 +333,31 @@ func refreshAllPredictCalculate(to orm.TxOrmer, edbInfoIdList []*EdbInfo, edbInf
 				temp[v.EdbInfoId] = dv.Value
 				saveDataMap[dv.DataTime] = temp
 			}
-			if edbInfoIndex == 0 {
+
+			if dateTagConfig == "all" {
+				if _, ok := dateMap[dv.DataTime]; !ok {
+					dateList = append(dateList, dv.DataTime)
+					dateMap[dv.DataTime] = struct{}{}
+				}
+			} else if dateTagConfig == "" { // 默认取第一个指标的时间序列
+				if edbInfoIndex == 0 {
+					if _, ok := dateMap[dv.DataTime]; !ok {
+						dateList = append(dateList, dv.DataTime)
+						dateMap[dv.DataTime] = struct{}{}
+					}
+				}
+			} else {
+				if eId, ok := edbInfoTag[dateTagConfig]; ok {
+					if v.EdbInfoId == eId {
+						if _, ok1 := dateMap[dv.DataTime]; !ok1 {
+							dateList = append(dateList, dv.DataTime)
+							dateMap[dv.DataTime] = struct{}{}
+						}
+					}
+				}
+			}
+
+			/*if edbInfoIndex == 0 {
 				dateList = append(dateList, dv.DataTime)
 				tmpDate, _ := time.ParseInLocation(utils.FormatDate, dv.DataTime, time.Local)
 				if minLatestDate.IsZero() || tmpDate.After(minLatestDate) {
@@ -330,10 +366,19 @@ func refreshAllPredictCalculate(to orm.TxOrmer, edbInfoIdList []*EdbInfo, edbInf
 				if maxStartDate.IsZero() || tmpDate.Before(maxStartDate) {
 					maxStartDate = tmpDate
 				}
-			}
+			}*/
+		}
+	}
+	// 处理最大日期和最小日期
+	for _, v := range dateList {
+		tmpDate, _ := time.ParseInLocation(utils.FormatDate, v, time.Local)
+		if minLatestDate.IsZero() || tmpDate.After(minLatestDate) {
+			minLatestDate = tmpDate
+		}
+		if maxStartDate.IsZero() || tmpDate.Before(maxStartDate) {
+			maxStartDate = tmpDate
 		}
 	}
-
 	//数据处理，将日期内不全的数据做填补
 	HandleDateSaveDataMap(dateList, maxStartDate, minLatestDate, realSaveDataMap, saveDataMap, edbInfoIdList, emptyType)
 
@@ -343,9 +388,6 @@ func refreshAllPredictCalculate(to orm.TxOrmer, edbInfoIdList []*EdbInfo, edbInf
 	}
 	addSql := ` INSERT INTO edb_data_predict_calculate (edb_info_id,edb_code,data_time,value,create_time,modify_time,data_timestamp) values `
 	var isAdd bool
-	if !minLatestDate.IsZero() {
-		latestDateStr = minLatestDate.Format(utils.FormatDate)
-	}
 
 	//获取指标所有数据
 	dataList := make([]*EdbData, 0)
@@ -371,6 +413,9 @@ func refreshAllPredictCalculate(to orm.TxOrmer, edbInfoIdList []*EdbInfo, edbInf
 	}
 
 	for sk, sv := range saveDataMap {
+		if _, ok := dateMap[sk]; !ok { //不在最终的日期序列里面不计算
+			continue
+		}
 		// 当空值处理类型选择了不计算时，只要有一个指标在某个日期没有值(即空值)，则计算指标在该日期没有值
 		if emptyType == 1 {
 			if len(sv) != len(edbInfoIdList) {
@@ -433,10 +478,6 @@ func refreshAllPredictCalculate(to orm.TxOrmer, edbInfoIdList []*EdbInfo, edbInf
 				removeDateList = append(removeDateList, sk)
 				continue
 			}
-			// 最近实际日期的实际数据
-			if latestDateStr == sk {
-				latestValue, _ = decimal.NewFromFloat(calVal).Truncate(4).Float64() //保留4位小数
-			}
 			saveValue := decimal.NewFromFloat(calVal).RoundCeil(4).String() //utils.SubFloatToString(calVal, 4)
 			if existVal, ok := dataMap[sk]; !ok {
 				dataTime, _ := time.ParseInLocation(utils.FormatDate, sk, time.Local)
@@ -493,5 +534,36 @@ func refreshAllPredictCalculate(to orm.TxOrmer, edbInfoIdList []*EdbInfo, edbInf
 			return
 		}
 	}
+
+	//查询预测日期
+	if !newLatestDate.IsZero() {
+		latestDateStr = newLatestDate.Format(utils.FormatDate)
+		sql = fmt.Sprintf(` SELECT * FROM %s WHERE edb_info_id = ? and data_time <= ? ORDER BY data_time desc limit 1`, dataTableName)
+		var latestDate EdbData
+		e := to.Raw(sql, edbInfoId, latestDateStr).QueryRow(&latestDate)
+		if e != nil {
+			if e.Error() == utils.ErrNoRow() {
+				//首个日期的值
+				sql = fmt.Sprintf(` SELECT * FROM %s WHERE edb_info_id = ? ORDER BY data_time ASC limit 1`, dataTableName)
+				e = to.Raw(sql, edbInfoId).QueryRow(&latestDate)
+				if e != nil {
+					err = fmt.Errorf("查询最新日期失败,Err:" + e.Error())
+					return
+				}
+				floatValue, _ := strconv.ParseFloat(latestDate.Value, 64)
+				latestDateStr = latestDate.DataTime
+				latestValue = floatValue
+			} else {
+				err = fmt.Errorf("查询最新日期失败,Err:" + e.Error())
+				return
+			}
+
+		} else {
+			floatValue, _ := strconv.ParseFloat(latestDate.Value, 64)
+			latestDateStr = latestDate.DataTime
+			latestValue = floatValue
+		}
+	}
+
 	return
 }

utils/base_from_calculate.go

@@ -71,6 +71,7 @@ func HandleFormulaJson(formula string, endDate time.Time) (dateSlice []string, f
 		formulaMap[v.Date] = v.Formula
 		dateSlice = append(dateSlice, v.Date)
 	}
+	// 得到的日期序列是从大到小，还是从小到大？
 	sort.Slice(dateSlice, func(i, j int) bool {
 		return dateSlice[i] < dateSlice[j]
 	})

utils/logs.go

@@ -124,8 +124,8 @@ func initMgoLog() {
 		logPath = "./etalogs"
 	}
 
-	mongoLogPath := logPath + "/mongo"
-	mongoLogFile := "mongo.log"
+	mongoLogPath := logPath + "/mongologs"
+	mongoLogFile := "mongolog.log"
 
 	os.MkdirAll(mongoLogPath, os.ModePerm)
 	logFileName := path.Join(mongoLogPath, mongoLogFile)