Merge branch 'master' into feature/eta2.5.9_api_stat

logic/base_edb_info.go

@@ -41,6 +41,15 @@ func RefreshBaseEdbInfo(edbInfo *models.EdbInfo, startDate string) (isHandling b
 	case utils.DATA_SOURCE_RADISH_RESEARCH:
 		ob := new(models.BaseFromRadishResearch)
 		err = ob.Refresh(edbInfo.EdbInfoId, edbInfo.EdbCode, startDate)
+	case utils.DATA_SOURCE_YONYI:
+		ob := new(models.BaseFromYongyi)
+		err = ob.Refresh(edbInfo.EdbInfoId, edbInfo.EdbCode, startDate)
+	case utils.DATA_SOURCE_GPR_RISK:
+		ob := new(models.BaseFromGprRisk)
+		err = ob.Refresh(edbInfo.EdbInfoId, edbInfo.EdbCode, startDate)
+	case utils.DATA_SOURCE_PuRang:
+		ob := new(models.BaseFromPurang)
+		err = ob.Refresh(edbInfo.EdbInfoId, edbInfo.EdbCode, startDate)
 	default:
 		return
 	}

models/base_from_yongyi.go

@@ -382,7 +382,7 @@ func (y *BaseFromYongyiClassify) GetByClassifyName(classifyName string) (item *B
 
 func (y *BaseFromYongyiClassify) GetParentClassify() (items []*BaseFromYongyiClassify, err error) {
 	//o := orm.NewOrm()
-	sql := ` SELECT * FROM base_from_yongyi_classify WHERE classify_name in ("日度", "月度", "周度", "年度", "旬度", "半年度", "年度") and parent_id=0 `
+	sql := ` SELECT * FROM base_from_yongyi_classify WHERE classify_name in ('日度', '月度', '周度', '年度', '旬度', '半年度', '年度') and parent_id=0 `
 	//_, err = o.Raw(sql).QueryRows(&items)
 	err = global.DEFAULT_DB.Raw(sql).Find(&items).Error
 	return

models/calculate_residual_analysis_config_mapping.go

@@ -11,32 +11,31 @@ type CalculateResidualAnalysisConfigMapping struct {
 	CalculateResidualAnalysisConfigId        int       `gorm:"column:calculate_residual_analysis_config_id" description:"残差分析配置id"`
 	EdbInfoId                                int64     `gorm:"column:edb_info_id" description:"指标id"`
 	ResidualType                             int       `gorm:"column:residual_type" description:"残差类型: 1-映射残差 2-拟合残差"`
-	IndexType                                int       `gorm:"column:index_type)" description:"指标类型：1-映射指标 2-残差指标 3-因变量指标 4-自变量指标"`
+	IndexType                                int       `gorm:"column:index_type" description:"指标类型：1-映射指标 2-残差指标 3-因变量指标 4-自变量指标"`
 	CreateTime                               time.Time `gorm:"column:create_time" description:"创建时间"`
 	ModifyTime                               time.Time `gorm:"column:modify_time" description:"修改时间"`
 }
 
 // GetConfigMappingListByConditionNotBase 查询非基础指标的配置映射
 func GetConfigMappingListByConditionNotBase(edbInfoId int) (configMapping []CalculateResidualAnalysisConfigMapping, err error) {
-	//sql := `
-	//SELECT
-	//	*
-	//FROM
-	//	calculate_residual_analysis_config_mapping
-	//WHERE
-	//	calculate_residual_analysis_config_id IN ( SELECT calculate_residual_analysis_config_id FROM calculate_residual_analysis_config_mapping WHERE edb_info_id = ? )
-	//	// AND index_type != 3
-	//	// AND index_type != 4`
 	sql := `
-	SELECT *
-		FROM calculate_residual_analysis_config_mapping m
-		WHERE EXISTS (
-			SELECT 1
-			FROM calculate_residual_analysis_config_mapping subq
-			WHERE subq.calculate_residual_analysis_config_id = m.calculate_residual_analysis_config_id
-			  AND subq.edb_info_id = ? AND subq.index_type != 3
- 			AND subq.index_type != 4
-		)`
+	SELECT
+		*
+	FROM
+		calculate_residual_analysis_config_mapping
+	WHERE
+		calculate_residual_analysis_config_id IN ( SELECT calculate_residual_analysis_config_id FROM calculate_residual_analysis_config_mapping WHERE edb_info_id = ?  AND index_type != 3
+		 AND index_type != 4)
+		`
+	//sql :=
+	//SELECT *
+	//	FROM calculate_residual_analysis_config_mapping m
+	//	WHERE calculate_residual_analysis_config_id in  (
+	//		SELECT calculate_residual_analysis_config_id
+	//		FROM calculate_residual_analysis_config_mapping subq
+	//		WHERE  subq.edb_info_id = ? AND subq.index_type != 3
+	//		AND subq.index_type != 4
+	//	)`
 
 	err = global.DEFAULT_DB.Raw(sql, edbInfoId).Find(&configMapping).Error
 	if err != nil {

models/edb_data_base.go

@@ -163,6 +163,7 @@ func GetFinalLastByTo(to *gorm.DB, edbInfoId, source, subSource int, latestDate
 	if err != nil && err.Error() != utils.ErrNoRow() {
 		return
 	}
+	finalLast.DataTime = utils.GormDateStrToDateStr(finalLast.DataTime)
 	return
 }
 

models/edb_data_residual_analysis.go

@@ -5,6 +5,7 @@ import (
 	"eta/eta_index_lib/global"
 	"eta/eta_index_lib/utils"
 	"fmt"
+	"github.com/shopspring/decimal"
 	"gorm.io/gorm"
 	"math"
 	"sort"
@@ -198,46 +199,55 @@ func refreshAllCalculateResidualAnalysis(edbInfoId, source, subSource, formulaIn
 		if err != nil {
 			return err
 		}
+		for _, edbList := range mappingEdbList {
+			if edbList.EdbInfoId == 0 {
+				for _, edbData := range edbList.DataList {
+					value, _ := strconv.ParseFloat(edbData.Value, 64)
+					edbDataResidualAnalysisList = append(edbDataResidualAnalysisList, edbDataResidualAnalysis{
+						EdbInfoId:     int(mapping.EdbInfoId),
+						EdbCode:       edbInfo.EdbCode,
+						DataTime:      edbData.DataTime,
+						Value:         value,
+						CreateTime:    time.Now(),
+						ModifyTime:    time.Now(),
+						DataTimeStamp: edbData.DataTimestamp,
+					})
+				}
 
-		for _, edbData := range mappingEdbList[0].DataList {
-			value, _ := strconv.ParseFloat(edbData.Value, 64)
-			edbDataResidualAnalysisList = append(edbDataResidualAnalysisList, edbDataResidualAnalysis{
-				EdbInfoId:     int(mapping.EdbInfoId),
-				EdbCode:       edbInfo.EdbCode,
-				DataTime:      edbData.DataTime,
-				Value:         value,
-				CreateTime:    time.Now(),
-				ModifyTime:    time.Now(),
-				DataTimeStamp: edbData.DataTimestamp,
-			})
+				err = AddResidualAnalysisData(edbDataResidualAnalysisList)
+				if err != nil {
+					return err
+				}
+				break
+			}
 		}
 
-		err = AddResidualAnalysisData(edbDataResidualAnalysisList)
-		if err != nil {
-			return err
-		}
 	} else if mapping.IndexType == 2 {
 		edbInfo, err := GetEdbInfoById(int(mapping.EdbInfoId))
 		if err != nil {
 			return err
 		}
+		for _, edbList := range residualEdbList {
+			if edbList.EdbInfoId == 0 {
+				for _, edbData := range edbList.DataList {
+					value, _ := strconv.ParseFloat(edbData.Value, 64)
+					edbDataResidualAnalysisList = append(edbDataResidualAnalysisList, edbDataResidualAnalysis{
+						EdbInfoId:     int(mapping.EdbInfoId),
+						EdbCode:       edbInfo.EdbCode,
+						DataTime:      edbData.DataTime,
+						Value:         value,
+						CreateTime:    time.Now(),
+						ModifyTime:    time.Now(),
+						DataTimeStamp: edbData.DataTimestamp,
+					})
+				}
 
-		for _, edbData := range residualEdbList[0].DataList {
-			value, _ := strconv.ParseFloat(edbData.Value, 64)
-			edbDataResidualAnalysisList = append(edbDataResidualAnalysisList, edbDataResidualAnalysis{
-				EdbInfoId:     int(mapping.EdbInfoId),
-				EdbCode:       edbInfo.EdbCode,
-				DataTime:      edbData.DataTime,
-				Value:         value,
-				CreateTime:    time.Now(),
-				ModifyTime:    time.Now(),
-				DataTimeStamp: edbData.DataTimestamp,
-			})
-		}
-
-		err = AddResidualAnalysisData(edbDataResidualAnalysisList)
-		if err != nil {
-			return err
+				err = AddResidualAnalysisData(edbDataResidualAnalysisList)
+				if err != nil {
+					return err
+				}
+				break
+			}
 		}
 	}
 
@@ -257,9 +267,10 @@ func fillResidualChartInfo(config ResidualAnalysisConfig, req *EdbInfo, edbInfoM
 	dataAList := edbInfoA.DataList
 	edbData := make([]*EdbDataList, len(dataAList))
 	for i, data := range dataAList {
-		f, _ := strconv.ParseFloat(data.Value, 64)
+		//f, _ := strconv.ParseFloat(data.Value, 64)
+		f, _ := decimal.NewFromString(data.Value)
 		edbData[i] = &EdbDataList{
-			Value:         f,
+			Value:         f.InexactFloat64(),
 			DataTimestamp: data.DataTimestamp,
 			DataTime:      data.DataTime,
 			EdbInfoId:     data.EdbInfoId,
@@ -380,7 +391,12 @@ func fillMappingChartInfo(config ResidualAnalysisConfig, req *EdbInfo, edbInfoMa
 	}
 
 	dataList := edbInfoMappingB.DataList
+	sort.Sort(ByDataTime(dataList))
+	for _, indexData := range dataList {
+		fmt.Printf("%s--%s\n", indexData.Value, indexData.DataTime)
+	}
 
+	fmt.Println("========================================================================================================")
 	// 指标B数据补充
 	// 新建一个切片来保存补充的数据
 	var replenishDataList []*EdbData
@@ -414,7 +430,13 @@ func fillMappingChartInfo(config ResidualAnalysisConfig, req *EdbInfo, edbInfoMa
 
 	// 排序
 	sort.Sort(ByDataTime(dataList))
+	// 排序
+	sort.Sort(ByDataTime(dataList))
+	for _, indexData := range dataList {
+		fmt.Printf("%s--%s\n", indexData.Value, indexData.DataTime)
+	}
 
+	fmt.Println("========================================================================================================")
 	// 拟合残差 计算a,b
 	var coordinateList []utils.Coordinate
 	var replenishADataList []*EdbDataList
@@ -514,7 +536,9 @@ func fillMappingChartInfo(config ResidualAnalysisConfig, req *EdbInfo, edbInfoMa
 
 	// 填充映射指标值 使得时间长度一致
 	dataList = FillDataBList(dataList, edbInfoMappingA)
-
+	for _, indexData := range dataList {
+		fmt.Printf("%s--%s\n", indexData.Value, indexData.DataTime)
+	}
 	// 根据指标A的时间key，在B的映射指标中筛选出对应的值
 	var dataBList []*EdbDataList
 
@@ -528,9 +552,10 @@ func fillMappingChartInfo(config ResidualAnalysisConfig, req *EdbInfo, edbInfoMa
 				indexDataCopy := *indexData
 
 				// 计算指标B映射值
-				f, _ := strconv.ParseFloat(indexData.Value, 64)
-				indexDataCopy.Value = fmt.Sprintf("%f", math.Round((a*f+b)*10000)/10000)
-
+				//f, _ := strconv.ParseFloat(indexData.Value, 64)
+				f, _ := decimal.NewFromString(indexData.Value)
+				indexDataCopy.Value = fmt.Sprintf("%f", math.Round((a*f.InexactFloat64()+b)*10000)/10000)
+				//fmt.Printf("%s--%s%s\n", indexData.Value, indexDataCopy.Value, indexData.DataTime)
 				// 比较最大值
 				if indexData.Value > indexMax {
 					indexMax = indexData.Value
@@ -580,19 +605,22 @@ func fillMappingChartInfo(config ResidualAnalysisConfig, req *EdbInfo, edbInfoMa
 func FillDataBList(dataList []*EdbData, edbInfoMappingA *EdbInfoList) []*EdbData {
 	dataAList := edbInfoMappingA.DataList
 
-	for utils.StringToTime(dataList[len(dataList)-1].DataTime).Before(utils.StringToTime(dataAList[len(dataAList)-1].DataTime)) {
-		// 使用A的时间填充时间差
-		timeDiff := utils.GetNextDayN(dataList[len(dataList)-1].DataTime, 1)
+	if len(dataList) > 0 && len(dataAList) > 0 {
+		fmt.Println("len(dataList):", len(dataList), "len(dataAList):", len(dataAList))
+		for utils.StringToTime(dataList[len(dataList)-1].DataTime).Before(utils.StringToTime(dataAList[len(dataAList)-1].DataTime)) {
+			// 使用A的时间填充时间差
+			timeDiff := utils.GetNextDayN(dataList[len(dataList)-1].DataTime, 1)
+
+			// 创建新的数据点并填充 前值填充
+			newDataPoint := &EdbData{
+				DataTime:      timeDiff,
+				Value:         dataList[len(dataList)-1].Value,
+				DataTimestamp: utils.StringToTime(timeDiff).UnixMilli(),
+			}
 
-		// 创建新的数据点并填充 前值填充
-		newDataPoint := &EdbData{
-			DataTime:      timeDiff,
-			Value:         dataList[len(dataList)-1].Value,
-			DataTimestamp: utils.StringToTime(timeDiff).UnixMilli(),
+			// 将新数据点添加到dataList末尾
+			dataList = append(dataList, newDataPoint)
 		}
-
-		// 将新数据点添加到dataList末尾
-		dataList = append(dataList, newDataPoint)
 	}
 
 	return dataList
@@ -624,7 +652,7 @@ func fillOriginalChart(config ResidualAnalysisConfig, req *EdbInfo, mappingList
 					return nil, nil, nil, fmt.Errorf("领先值不能小于0")
 				} else if config.LeadValue > 0 {
 					edbInfoMapping.EdbName = v.EdbName + "（领先" + strconv.Itoa(config.LeadValue) + config.LeadFrequency + "）"
-					for _, indexData := range dataList {
+					for _, indexData := range data {
 						switch config.LeadFrequency {
 						case "天":
 							indexData.DataTime = utils.GetNextDayN(indexData.DataTime, config.LeadValue)

models/predict_edb_data_calculate_tcz.go

@@ -455,7 +455,7 @@ func RefreshAllPredictCalculateTcz(edbInfoId, source, subSource int, fromEdbInfo
 			} else {
 				if fromEdbInfo.Frequency == "月度" { //向上和向下，各找一个月
 					for i := 0; i <= 35; i++ {
-						nextDateDay := preDate.AddDate(0, 0, 1)
+						nextDateDay := preDate.AddDate(0, 0, i)
 						nextDateDayStr := nextDateDay.Format(utils.FormatDate)
 						if findItem, ok := dataMap[nextDateDayStr]; ok { //上一年同期->下一个月找到
 							timestamp := currentDate.UnixNano() / 1e6
@@ -690,5 +690,36 @@ func RefreshAllPredictCalculateTcz(edbInfoId, source, subSource int, fromEdbInfo
 			return
 		}
 	}
+
+	// 取出最新值(小于源指标最新日期的最大日期所对应的值)
+	{
+		edb, e := GetEdbInfoById(edbInfoId)
+		if e != nil {
+			err = fmt.Errorf("获取刷新后的指标失败, %v", e)
+			return
+		}
+		list, e := GetPredictEdbDataListAll(edb, 0)
+		if e != nil {
+			err = fmt.Errorf("获取刷新后的指标数据失败, %v", e)
+			return
+		}
+		dateData := make(map[time.Time]float64)
+		for _, v := range list {
+			d, _ := time.ParseInLocation(utils.FormatDate, v.DataTime, time.Local)
+			if !d.IsZero() {
+				dateData[d] = v.Value
+			}
+		}
+		t, _ := time.ParseInLocation(utils.FormatDate, fromEdbInfo.LatestDate, time.Local)
+		if !t.IsZero() {
+			var maxDate time.Time
+			for k := range dateData {
+				if k.After(maxDate) && !k.After(t) {
+					maxDate = k
+				}
+			}
+			latestValue = dateData[maxDate]
+		}
+	}
 	return
 }

services/base_from_yongyi.go

@@ -298,7 +298,7 @@ func handleYongyiIndex(req *models.HandleYongyiExcelData, terminalCode string, c
 		}
 
 		// 已经加入到指标库的话，那么就去更新ETA指标库吧
-		if edbInfo != nil {
+		if edbInfo != nil && edbInfo.EdbInfoId > 0 {
 			go logic.RefreshBaseEdbInfo(edbInfo, ``)
 		}
 	}