Merge remote-tracking branch 'origin/master' into dm

# Conflicts:
#	models/calculate_residual_analysis_config.go
#	models/calculate_residual_analysis_config_mapping.go
#	models/edb_data_residual_analysis.go
#	utils/constants.go

logic/predict_edb.go

@@ -172,7 +172,7 @@ func AddPredictEdbInfo(sourceEdbInfoId, classifyId int, edbName, dataDateType st
 			return
 		}
 
-		if v.RuleType == 16 && endDateType != 1 {
+		if v.RuleType == 16 && endDateType == 1 {
 			errMsg = "年度值倒推不支持截止期数"
 			err = errors.New(errMsg)
 			return
@@ -577,7 +577,7 @@ func EditPredictEdbInfo(edbInfoId, classifyId int, edbName, dataDateType string,
 			err = errors.New(errMsg)
 			return
 		}
-		if v.RuleType == 16 && endDateType != 1 {
+		if v.RuleType == 16 && endDateType == 1 {
 			errMsg = "年度值倒推不支持截止期数"
 			err = errors.New(errMsg)
 			return

models/calculate_residual_analysis_config.go

@@ -46,8 +46,7 @@ func (m *ResidualAnalysisConfig) AfterFind(db *gorm.DB) (err error) {
 func GetResidualAnalysisConfigById(edbInfoId int) (residualAnalysisConfig CalculateResidualAnalysisConfig, err error) {
 	sql := `SELECT t2.* FROM calculate_residual_analysis_config_mapping t1 
 			join calculate_residual_analysis_config t2 on t1.calculate_residual_analysis_config_id = t2.calculate_residual_analysis_config_id
-			WHERE t1.edb_info_id = ?`
+			WHERE t1.edb_info_id = ? and index_type!=3 and index_type !=4`
 	err = global.DEFAULT_DB.Raw(sql, edbInfoId).First(&residualAnalysisConfig).Error
-
 	return
 }

models/calculate_residual_analysis_config_mapping.go

@@ -18,15 +18,26 @@ type CalculateResidualAnalysisConfigMapping struct {
 
 // GetConfigMappingListByConditionNotBase 查询非基础指标的配置映射
 func GetConfigMappingListByConditionNotBase(edbInfoId int) (configMapping []CalculateResidualAnalysisConfigMapping, err error) {
+	//sql := `
+	//SELECT
+	//	*
+	//FROM
+	//	calculate_residual_analysis_config_mapping
+	//WHERE
+	//	calculate_residual_analysis_config_id IN ( SELECT calculate_residual_analysis_config_id FROM calculate_residual_analysis_config_mapping WHERE edb_info_id = ? )
+	//	// AND index_type != 3
+	//	// AND index_type != 4`
 	sql := `
-	SELECT
-  		* 
-	FROM
-  		calculate_residual_analysis_config_mapping 
-	WHERE
-  		calculate_residual_analysis_config_id IN ( SELECT calculate_residual_analysis_config_id FROM calculate_residual_analysis_config_mapping WHERE edb_info_id = ? ) 
-  		AND index_type != 3 
-  		AND index_type != 4`
+	SELECT *
+		FROM calculate_residual_analysis_config_mapping m
+		WHERE EXISTS (
+			SELECT 1
+			FROM calculate_residual_analysis_config_mapping subq
+			WHERE subq.calculate_residual_analysis_config_id = m.calculate_residual_analysis_config_id
+			  AND subq.edb_info_id = ? AND subq.index_type != 3
+ 			AND subq.index_type != 4
+		)`
+
 	err = global.DEFAULT_DB.Raw(sql, edbInfoId).Find(&configMapping).Error
 	if err != nil {
 		return nil, fmt.Errorf("查询数据时出错: %v", err)

models/edb_data_residual_analysis.go

@@ -20,7 +20,7 @@ type edbDataResidualAnalysis struct {
 	Value         float64   `gorm:"column:value;type:double;comment:数据值;default:NULL;"`               // 数据值
 	CreateTime    time.Time `gorm:"column:create_time;type:datetime;comment:创建时间;default:NULL;" `     // 创建时间
 	ModifyTime    time.Time `gorm:"column:modify_time;type:datetime;comment:修改时间;default:NULL;"`      // 修改时间
-	DataTimestamp int64     `gorm:"column:data_timestamp;type:bigint(20);comment:数据日期时间戳;default:0;"` // 数据日期时间戳
+	DataTimeStamp int64     `gorm:"column:data_timestamp;type:bigint(20);comment:数据日期时间戳;default:0;"` // 数据日期时间戳
 }
 
 // AfterFind 在该模型上设置钩子函数，把日期转成正确的string，所以查询函数只能用Find函数，First或者Scan是不会触发该函数的来获取数据
@@ -64,19 +64,31 @@ func RefreshAllCalculateResidualAnalysis(edbInfoId, source, subSource, formulaIn
 			_ = to.Commit()
 		}
 	}()*/
+	//只清空残差分析数据自身指标的数据
 	configMapping, err := GetConfigMappingListByConditionNotBase(edbInfoId)
-	if err != nil {
-		return err
-	}
-
-	var edbInfoIdList []int64
-	for _, v := range configMapping {
-		edbInfoIdList = append(edbInfoIdList, v.EdbInfoId)
-	}
+	//if err != nil {
+	//	return err
+	//}
+	//
+	//var edbInfoIdList []int64
+	//for _, v := range configMapping {
+	//	edbInfoIdList = append(edbInfoIdList, v.EdbInfoId)
+	//}
 
 	//清空原有数据
-	sql := ` DELETE FROM edb_data_residual_analysis WHERE edb_info_id in (` + utils.GetOrmInReplace(len(edbInfoIdList)) + `) `
-	err = to.Exec(sql, edbInfoIdList).Error
+	//sql := ` DELETE FROM edb_data_residual_analysis WHERE edb_info_id in (` + utils.GetOrmInReplace(len(edbInfoIdList)) + `) `
+	//var params []interface{}
+	//for _, i := range edbInfoIdList {
+	//	params = append(params, i)
+	//}
+	var params []interface{}
+	sql := ` DELETE FROM edb_data_residual_analysis WHERE edb_info_id = ?`
+	params = append(params, edbInfoId)
+	//var params []interface{}
+	//for _, i := range edbInfoIdList {
+	//	params = append(params, i)
+	//}
+	err = to.Exec(sql, params...).Error
 	if err != nil {
 		return
 	}
@@ -90,13 +102,23 @@ func RefreshAllCalculateResidualAnalysis(edbInfoId, source, subSource, formulaIn
 // refreshAllCalculateResidualAnalysis 刷新所有残差分析
 func refreshAllCalculateResidualAnalysis(edbInfoId, source, subSource, formulaInt, moveType int, fromEdbInfo *EdbInfo, edbCode, startDate, endDate, moveFrequency string, configMapping []CalculateResidualAnalysisConfigMapping) (err error) {
 	fmt.Println("refreshAllCalculateResidualAnalysis startDate:", startDate)
-
+	configMap := make(map[int64]CalculateResidualAnalysisConfigMapping, len(configMapping))
+	for _, v := range configMapping {
+		configMap[v.EdbInfoId] = v
+	}
 	calculateMappingList, err := GetCalculateMappingListByEdbInfoId(edbInfoId)
 	if err != nil {
 		return err
 	}
-	edbInfoIdA := calculateMappingList[0].FromEdbInfoId
-	edbInfoIdB := calculateMappingList[1].FromEdbInfoId
+	//这里顺序和之前的不一致导致的计算错误
+	var edbInfoIdA, edbInfoIdB int
+	if configMap[int64(calculateMappingList[0].FromEdbInfoId)].IndexType == 3 {
+		edbInfoIdA = calculateMappingList[0].FromEdbInfoId
+		edbInfoIdB = calculateMappingList[1].FromEdbInfoId
+	} else {
+		edbInfoIdA = calculateMappingList[1].FromEdbInfoId
+		edbInfoIdB = calculateMappingList[0].FromEdbInfoId
+	}
 
 	mappingList, err := GetEdbInfoListByIds([]int{edbInfoIdA, edbInfoIdB})
 	if err != nil {
@@ -134,10 +156,10 @@ func refreshAllCalculateResidualAnalysis(edbInfoId, source, subSource, formulaIn
 
 	// 时间处理
 	switch config.DateType {
-	case 0:
+	case -1:
 		startDate = config.StartDate
 		endDate = config.EndDate
-	case 1:
+	case 0:
 		startDate = config.StartDate
 		endDate = ""
 	default:
@@ -168,54 +190,54 @@ func refreshAllCalculateResidualAnalysis(edbInfoId, source, subSource, formulaIn
 	residualEdbList, _, err := fillResidualChartInfo(config, fromEdbInfo, edbInfoMappingA, edbInfoMappingB, mappingEdbList)
 
 	// 映射指标 与 残差指标 同步刷新
-	for _, mapping := range configMapping {
-		var edbDataResidualAnalysisList []edbDataResidualAnalysis
-		if mapping.IndexType == 1 {
-			edbInfo, err := GetEdbInfoById(int(mapping.EdbInfoId))
-			if err != nil {
-				return err
-			}
+	//for _, mapping := range configMapping {
+	mapping := configMap[int64(edbInfoId)]
+	var edbDataResidualAnalysisList []edbDataResidualAnalysis
+	if mapping.IndexType == 1 {
+		edbInfo, err := GetEdbInfoById(int(mapping.EdbInfoId))
+		if err != nil {
+			return err
+		}
 
-			for _, edbData := range mappingEdbList[1].DataList {
-				value, _ := strconv.ParseFloat(edbData.Value, 64)
-				edbDataResidualAnalysisList = append(edbDataResidualAnalysisList, edbDataResidualAnalysis{
-					EdbInfoId:     int(mapping.EdbInfoId),
-					EdbCode:       edbInfo.EdbCode,
-					DataTime:      edbData.DataTime,
-					Value:         value,
-					CreateTime:    time.Now(),
-					ModifyTime:    time.Now(),
-					DataTimestamp: edbData.DataTimestamp,
-				})
-			}
+		for _, edbData := range mappingEdbList[0].DataList {
+			value, _ := strconv.ParseFloat(edbData.Value, 64)
+			edbDataResidualAnalysisList = append(edbDataResidualAnalysisList, edbDataResidualAnalysis{
+				EdbInfoId:     int(mapping.EdbInfoId),
+				EdbCode:       edbInfo.EdbCode,
+				DataTime:      edbData.DataTime,
+				Value:         value,
+				CreateTime:    time.Now(),
+				ModifyTime:    time.Now(),
+				DataTimeStamp: edbData.DataTimestamp,
+			})
+		}
 
-			err = AddResidualAnalysisData(edbDataResidualAnalysisList)
-			if err != nil {
-				return err
-			}
-		} else if mapping.IndexType == 2 {
-			edbInfo, err := GetEdbInfoById(int(mapping.EdbInfoId))
-			if err != nil {
-				return err
-			}
+		err = AddResidualAnalysisData(edbDataResidualAnalysisList)
+		if err != nil {
+			return err
+		}
+	} else if mapping.IndexType == 2 {
+		edbInfo, err := GetEdbInfoById(int(mapping.EdbInfoId))
+		if err != nil {
+			return err
+		}
 
-			for _, edbData := range residualEdbList[1].DataList {
-				value, _ := strconv.ParseFloat(edbData.Value, 64)
-				edbDataResidualAnalysisList = append(edbDataResidualAnalysisList, edbDataResidualAnalysis{
-					EdbInfoId:     int(mapping.EdbInfoId),
-					EdbCode:       edbInfo.EdbCode,
-					DataTime:      edbData.DataTime,
-					Value:         value,
-					CreateTime:    time.Now(),
-					ModifyTime:    time.Now(),
-					DataTimestamp: edbData.DataTimestamp,
-				})
-			}
+		for _, edbData := range residualEdbList[0].DataList {
+			value, _ := strconv.ParseFloat(edbData.Value, 64)
+			edbDataResidualAnalysisList = append(edbDataResidualAnalysisList, edbDataResidualAnalysis{
+				EdbInfoId:     int(mapping.EdbInfoId),
+				EdbCode:       edbInfo.EdbCode,
+				DataTime:      edbData.DataTime,
+				Value:         value,
+				CreateTime:    time.Now(),
+				ModifyTime:    time.Now(),
+				DataTimeStamp: edbData.DataTimestamp,
+			})
+		}
 
-			err = AddResidualAnalysisData(edbDataResidualAnalysisList)
-			if err != nil {
-				return err
-			}
+		err = AddResidualAnalysisData(edbDataResidualAnalysisList)
+		if err != nil {
+			return err
 		}
 	}
 
@@ -325,10 +347,20 @@ func fillResidualChartInfo(config ResidualAnalysisConfig, req *EdbInfo, edbInfoM
 			}
 		}
 	}
-
+	//reverse(mappingEdb)
 	return mappingEdb, R2, nil
 }
 
+// 双指针法
+func reverse(slice []EdbInfoList) {
+	left, right := 0, len(slice)-1
+	for left < right {
+		// 交换元素
+		slice[left], slice[right] = slice[right], slice[left]
+		left++
+		right--
+	}
+}
 func fillMappingChartInfo(config ResidualAnalysisConfig, req *EdbInfo, edbInfoMappingA *EdbInfoList, edbInfoMappingB *EdbInfoList, originalEdbList []EdbInfoList, indexADataMap map[string]*EdbData, startDate string, endDate string, fullADataList []*EdbDataList, fullBDataList []*EdbDataList) ([]EdbInfoList, float64, float64, float64, error) {
 	// 计算公式：Y=aX+b，Y为映射后的指标，X为自变量指标
 	// 正序：a=(L2-L1)/(R2-R1)	b=L2-R2*a
@@ -587,7 +619,7 @@ func fillOriginalChart(config ResidualAnalysisConfig, req *EdbInfo, mappingList
 
 		if v.EdbInfoId == edbInfoMappingB.EdbInfoId {
 			// 领先指标 dataList进行数据处理
-			if config.IndexType == 1 {
+			if config.IndexType == 2 {
 				if config.LeadValue < 0 {
 					return nil, nil, nil, fmt.Errorf("领先值不能小于0")
 				} else if config.LeadValue > 0 {

models/predict_edb_info_rule.go

@@ -1656,6 +1656,10 @@ func GetChartPredictEdbInfoDataListByRuleAnnualValueInversion(edbInfoId int, con
 		//兼容历史数据
 		yearList = append(yearList, annualValueInversionConf.Year)
 	}
+	if len(yearList) == 0 {
+		err = errors.New("同比年份不能为空")
+		return
+	}
 	// 每年截止到当前日期的累计值
 	dateTotalMap := make(map[time.Time]float64)
 
@@ -1753,6 +1757,10 @@ func GetChartPredictEdbInfoDataListByRuleAnnualValueInversion(edbInfoId int, con
 			}
 		}
 	}
+	if sum == 0 {
+		err = errors.New("同比年份的累计值为0")
+		return
+	}
 	//fmt.Printf("同比年份的余额%.4f\n", sum)
 	avg = sum / float64(len(yearList))
 	//fmt.Printf("同比年份的余额%.4f\n", avg)

services/edb_data_calculate_stl.go

@@ -1,10 +1,12 @@
 package services
 
 import (
+	"database/sql"
 	"encoding/json"
 	"eta/eta_index_lib/models"
 	"eta/eta_index_lib/utils"
 	"fmt"
+	"github.com/shopspring/decimal"
 	"os"
 	"os/exec"
 	"path/filepath"
@@ -166,7 +168,7 @@ func RefreshStlData(edbInfoId int) (msg string, err error) {
 		return
 	}
 
-	trendChart, seasonalChart, residualChart, err := ParseStlExcel(saveFilePath)
+	trendChart, seasonalChart, residualChart, nonTrendChartInfo, err := ParseStlExcel(saveFilePath)
 	if err != nil {
 		msg = "解析Excel失败"
 		return
@@ -177,7 +179,7 @@ func RefreshStlData(edbInfoId int) (msg string, err error) {
 		msg = "获取指标信息失败"
 		return
 	}
-	err = SyncUpdateRelationEdbInfo(edbInfo, stlConfig, trendChart, seasonalChart, residualChart)
+	err = SyncUpdateRelationEdbInfo(edbInfo, stlConfig, trendChart, seasonalChart, residualChart, nonTrendChartInfo)
 	if err != nil {
 		msg = "更新关联指标失败"
 		return
@@ -186,7 +188,7 @@ func RefreshStlData(edbInfoId int) (msg string, err error) {
 	return
 }
 
-func SyncUpdateRelationEdbInfo(edbInfo *models.EdbInfo, config EdbStlConfig, trendData, seasonalData, residualData ChartEdbInfo) (err error) {
+func SyncUpdateRelationEdbInfo(edbInfo *models.EdbInfo, config EdbStlConfig, trendData, seasonalData, residualData, nonTrendChartInfo ChartEdbInfo) (err error) {
 	configId, err := models.GetCalculateStlConfigMappingIdByEdbInfoId(edbInfo.EdbInfoId)
 	if err != nil {
 		return
@@ -201,15 +203,18 @@ func SyncUpdateRelationEdbInfo(edbInfo *models.EdbInfo, config EdbStlConfig, tre
 			continue
 		}
 		switch v.StlEdbType {
-		case 1:
+		case utils.StlTypeTrend:
 			// 趋势指标
 			er = UpdateStlEdbData(edbInfo, config, edbInfo.EdbCode, trendData)
-		case 2:
+		case utils.StlTypeSeasonal:
 			// 季节性指标
 			er = UpdateStlEdbData(edbInfo, config, edbInfo.EdbCode, seasonalData)
-		case 3:
+		case utils.StlTypeResidual:
 			// 残差指标
 			er = UpdateStlEdbData(edbInfo, config, edbInfo.EdbCode, residualData)
+		case utils.StlTypeNonTrend:
+			// 季节+残差指标
+			er = UpdateStlEdbData(edbInfo, config, edbInfo.EdbCode, nonTrendChartInfo)
 		default:
 			utils.FileLog.Info("未知的stlEdbType类型, mapping:%v", v)
 			continue
@@ -298,7 +303,7 @@ func SaveToExcel(data []*models.EdbInfoSearchData, filePath string) (err error)
 	return
 }
 
-func ParseStlExcel(excelPath string) (TrendChart, SeasonalChart, ResidualChart ChartEdbInfo, err error) {
+func ParseStlExcel(excelPath string) (TrendChart, SeasonalChart, ResidualChart, nonTrendChartInfo ChartEdbInfo, err error) {
 	file, err := xlsx.OpenFile(excelPath)
 	if err != nil {
 		return
@@ -394,6 +399,68 @@ func ParseStlExcel(excelPath string) (TrendChart, SeasonalChart, ResidualChart C
 			ResidualChart.MinData = MinData
 		}
 	}
+
+	// 数据处理
+	dateList := make([]string, 0)
+	residualDateMap := make(map[string]*EdbData)
+	for _, item := range ResidualChart.DataList {
+		if _, ok := residualDateMap[item.DataTime]; ok {
+			continue
+		}
+		residualDateMap[item.DataTime] = item
+		dateList = append(dateList, item.DataTime)
+	}
+	seasonalDateMap := make(map[string]*EdbData)
+	for _, item := range SeasonalChart.DataList {
+		if _, ok := seasonalDateMap[item.DataTime]; ok {
+			continue
+		}
+		seasonalDateMap[item.DataTime] = item
+	}
+
+	// 季节性项+残差项
+	{
+		dataList := make([]*EdbData, 0)
+		var minValue, maxValue sql.NullFloat64
+		for _, date := range dateList {
+			tmpResidual, ok := residualDateMap[date]
+			if !ok {
+				continue
+			}
+			tmpSeasonal, ok := seasonalDateMap[date]
+			if !ok {
+				continue
+			}
+
+			tmpValue := tmpResidual.Value + tmpSeasonal.Value
+			tmpValue, _ = decimal.NewFromFloat(tmpValue).Round(4).Float64()
+
+			dataList = append(dataList, &EdbData{
+				DataTime:      date,
+				DataTimestamp: 0,
+				Value:         tmpValue,
+			})
+			// 如果没有设置最小值，或者设置的最小值比当前值还大，则需要更新最小值
+			if !minValue.Valid || minValue.Float64 > tmpValue {
+				err = minValue.Scan(tmpValue)
+				if err != nil {
+					return
+				}
+			}
+			// 如果没有设置最大值，或者设置的最大值比当前值还小，则需要更新最大值
+			if !maxValue.Valid || maxValue.Float64 < tmpValue {
+				err = maxValue.Scan(tmpValue)
+				if err != nil {
+					return
+				}
+			}
+		}
+
+		nonTrendChartInfo.DataList = dataList
+		nonTrendChartInfo.MinData = minValue.Float64
+		nonTrendChartInfo.MaxData = maxValue.Float64
+	}
+
 	return
 }
 

utils/constants.go

@@ -374,3 +374,11 @@ const (
 	DbNameAI          = "eta_ai"
 	DbNameWeekly      = "weekly_report"
 )
+
+// STL模型类型
+const (
+	StlTypeTrend    = 1 // 趋势
+	StlTypeSeasonal = 2 // 季节性指标
+	StlTypeResidual = 3 // 残差性指标
+	StlTypeNonTrend = 4 // 非趋势性指标
+)