Merge branch 'pool/337' into debug

# Conflicts:
#	controllers/data_manage/excel/excel_info.go
#	models/data_manage/excel/excel_info.go
#	models/data_manage/excel/referenced_excel_config.go
#	models/db.go
#	routers/router.go
#	services/data/stl/stl.go
#	services/data/trade_analysis/trade_analysis_data.go
#	services/excel_info.go
#	utils/constants.go

models/data_manage/excel/referenced_excel_config.go

@@ -37,6 +37,14 @@ func AddReferencedExcelConfig(items []*ReferencedExcelConfig) (err error) {
 	return
 }
 
+// getByCode
+func GetReferencedExcelConfigByUniqueCode(uniqueCode string) (item ReferencedExcelConfig, err error) {
+	o := orm.NewOrmUsingDB("data")
+	sql := ` SELECT * FROM referenced_excel_config WHERE referenced_excel_unique_code = ? `
+	err = o.Raw(sql, uniqueCode).QueryRow(&item)
+	return
+}
+
 // getByCode
 func GetReferencedExcelConfig(referencedId, fromScene int, uniqueCode, uuid string) (item ReferencedExcelConfig, err error) {
 	o := orm.NewOrmUsingDB("data")

models/data_manage/stl/response/stl.go

@@ -5,6 +5,7 @@ type StlPreviewResp struct {
 	TrendChartInfo    ChartEdbInfo
 	SeasonalChartInfo ChartEdbInfo
 	ResidualChartInfo ChartEdbInfo
+	NonTrendChartInfo ChartEdbInfo
 	EvaluationResult  EvaluationResult
 }
 

services/data/stl/stl.go

@@ -1,6 +1,7 @@
 package stl
 
 import (
+	"database/sql"
 	"encoding/json"
 	"errors"
 	"eta/eta_api/models/data_manage"
@@ -34,6 +35,7 @@ const (
 var EDB_DATA_CALCULATE_STL_TREND_CACHE = `eta:stl_decompose:trend:config_id:`
 var EDB_DATA_CALCULATE_STL_SEASONAL_CACHE = `eta:stl_decompose:seasonal:config_id:`
 var EDB_DATA_CALCULATE_STL_RESIDUAL_CACHE = `eta:stl_decompose:residual:config_id:`
+var EDB_DATA_CALCULATE_STL_NonTrend_CACHE = `eta:stl_decompose:non_trend:config_id:`
 
 func GenerateStlEdbData(req *request.StlConfigReq, adminId int) (resp *response.StlPreviewResp, msg string, err error) {
 	config, err := stl.GetCalculateStlConfigById(req.CalculateStlConfigId)
@@ -156,7 +158,7 @@ func GenerateStlEdbData(req *request.StlConfigReq, adminId int) (resp *response.
 		msg = "保存数据到Excel失败"
 		return
 	}
-	defer os.Remove(loadFilePath)
+	//defer os.Remove(loadFilePath)
 
 	saveFilePath := exPath + "/" + strconv.Itoa(adminId) + "_" + time.Now().Format(utils.FormatDateTimeUnSpace) + "_res" + ".xlsx"
 	result, err := execStlPythonCode(loadFilePath, saveFilePath, confReq.Period, confReq.Seasonal, confReq.Trend, confReq.TrendDeg, confReq.SeasonalDeg, confReq.LowPassDeg, confReq.Fraction, confReq.Robust)
@@ -164,7 +166,7 @@ func GenerateStlEdbData(req *request.StlConfigReq, adminId int) (resp *response.
 		msg = "计算失败，请重新选择指标和参数后计算"
 		return
 	}
-	trendChart, seasonalChart, residualChart, err := ParseStlExcel(saveFilePath)
+	trendChart, seasonalChart, residualChart, nonTrendChartInfo, err := ParseStlExcel(saveFilePath)
 	if err != nil {
 		msg = "解析Excel失败"
 		return
@@ -203,6 +205,16 @@ func GenerateStlEdbData(req *request.StlConfigReq, adminId int) (resp *response.
 	resp.ResidualChartInfo.Title = residualName
 	resp.ResidualChartInfo.Frequency = edbInfo.Frequency
 	resp.ResidualChartInfo.Unit = edbInfo.Unit
+
+	// 季节性项+残差项
+	resp.NonTrendChartInfo.DataList = nonTrendChartInfo.DataList
+	resp.NonTrendChartInfo.MaxData = nonTrendChartInfo.MaxData
+	resp.NonTrendChartInfo.MinData = nonTrendChartInfo.MinData
+	resp.NonTrendChartInfo.ClassifyId = edbInfo.ClassifyId
+	resp.NonTrendChartInfo.Title = edbInfo.EdbName + "Non-Trend"
+	resp.NonTrendChartInfo.Frequency = edbInfo.Frequency
+	resp.NonTrendChartInfo.Unit = edbInfo.Unit
+
 	resp.EvaluationResult.Mean = strconv.FormatFloat(result.ResidualMean, 'f', 4, 64)
 	resp.EvaluationResult.Std = strconv.FormatFloat(result.ResidualVar, 'f', 4, 64)
 	resp.EvaluationResult.AdfPValue = strconv.FormatFloat(result.AdfPValue, 'f', -1, 64)
@@ -216,15 +228,18 @@ func GenerateStlEdbData(req *request.StlConfigReq, adminId int) (resp *response.
 	var relationEdbInfoId []int
 	for _, mapping := range confMapping {
 		switch mapping.StlEdbType {
-		case 1:
+		case utils.StlTypeTrend:
 			resp.TrendChartInfo.EdbInfoId = mapping.EdbInfoId
 			relationEdbInfoId = append(relationEdbInfoId, mapping.EdbInfoId)
-		case 2:
+		case utils.StlTypeSeasonal:
 			resp.SeasonalChartInfo.EdbInfoId = mapping.EdbInfoId
 			relationEdbInfoId = append(relationEdbInfoId, mapping.EdbInfoId)
-		case 3:
+		case utils.StlTypeResidual:
 			resp.ResidualChartInfo.EdbInfoId = mapping.EdbInfoId
 			relationEdbInfoId = append(relationEdbInfoId, mapping.EdbInfoId)
+		case utils.StlTypeNonTrend:
+			resp.NonTrendChartInfo.EdbInfoId = mapping.EdbInfoId
+			relationEdbInfoId = append(relationEdbInfoId, mapping.EdbInfoId)
 		}
 	}
 	relationEdbInfo, err := data_manage.GetEdbInfoByIdList(relationEdbInfoId)
@@ -249,12 +264,18 @@ func GenerateStlEdbData(req *request.StlConfigReq, adminId int) (resp *response.
 			resp.ResidualChartInfo.ClassifyId = info.ClassifyId
 			resp.ResidualChartInfo.Frequency = info.Frequency
 			resp.ResidualChartInfo.Unit = info.Unit
+		case resp.NonTrendChartInfo.EdbInfoId:
+			resp.NonTrendChartInfo.Title = info.EdbName
+			resp.NonTrendChartInfo.ClassifyId = info.ClassifyId
+			resp.NonTrendChartInfo.Frequency = info.Frequency
+			resp.NonTrendChartInfo.Unit = info.Unit
 		}
 	}
 
 	bTrend, _ := json.Marshal(trendChart.DataList)
 	bSeasonal, _ := json.Marshal(seasonalChart.DataList)
 	bResidual, _ := json.Marshal(residualChart.DataList)
+	bNonTrend, _ := json.Marshal(nonTrendChartInfo.DataList)
 	err = utils.Rc.Put(EDB_DATA_CALCULATE_STL_TREND_CACHE+strconv.Itoa(config.CalculateStlConfigId), bTrend, time.Hour*2)
 	if err != nil {
 		msg = "计算失败，请重新计算"
@@ -265,7 +286,11 @@ func GenerateStlEdbData(req *request.StlConfigReq, adminId int) (resp *response.
 		msg = "计算失败，请重新计算"
 		return
 	}
-	utils.Rc.Put(EDB_DATA_CALCULATE_STL_RESIDUAL_CACHE+strconv.Itoa(config.CalculateStlConfigId), bResidual, time.Hour*2)
+	err = utils.Rc.Put(EDB_DATA_CALCULATE_STL_RESIDUAL_CACHE+strconv.Itoa(config.CalculateStlConfigId), bResidual, time.Hour*2)
+	if err != nil {
+		msg = "计算失败，请重新计算"
+	}
+	err = utils.Rc.Put(EDB_DATA_CALCULATE_STL_NonTrend_CACHE+strconv.Itoa(config.CalculateStlConfigId), bNonTrend, time.Hour*2)
 	if err != nil {
 		msg = "计算失败，请重新计算"
 	}
@@ -292,7 +317,7 @@ func CheckOsPathAndMake(path string) (err error) {
 	return
 }
 
-func ParseStlExcel(excelPath string) (TrendChart, SeasonalChart, ResidualChart response.ChartEdbInfo, err error) {
+func ParseStlExcel(excelPath string) (TrendChart, SeasonalChart, ResidualChart, nonTrendChartInfo response.ChartEdbInfo, err error) {
 	file, err := xlsx.OpenFile(excelPath)
 	if err != nil {
 		return
@@ -391,6 +416,68 @@ func ParseStlExcel(excelPath string) (TrendChart, SeasonalChart, ResidualChart r
 			ResidualChart.MinData = MinData
 		}
 	}
+
+	// 数据处理
+	dateList := make([]string, 0)
+	residualDateMap := make(map[string]*response.EdbData)
+	for _, item := range ResidualChart.DataList {
+		if _, ok := residualDateMap[item.DataTime]; ok {
+			continue
+		}
+		residualDateMap[item.DataTime] = item
+		dateList = append(dateList, item.DataTime)
+	}
+	seasonalDateMap := make(map[string]*response.EdbData)
+	for _, item := range SeasonalChart.DataList {
+		if _, ok := seasonalDateMap[item.DataTime]; ok {
+			continue
+		}
+		seasonalDateMap[item.DataTime] = item
+	}
+
+	// 季节性项+残差项
+	{
+		dataList := make([]*response.EdbData, 0)
+		var minValue, maxValue sql.NullFloat64
+		for _, date := range dateList {
+			tmpResidual, ok := residualDateMap[date]
+			if !ok {
+				continue
+			}
+			tmpSeasonal, ok := seasonalDateMap[date]
+			if !ok {
+				continue
+			}
+
+			tmpValue := tmpResidual.Value + tmpSeasonal.Value
+			tmpValue, _ = decimal.NewFromFloat(tmpValue).Round(4).Float64()
+
+			dataList = append(dataList, &response.EdbData{
+				DataTime:      date,
+				DataTimestamp: 0,
+				Value:         tmpValue,
+			})
+			// 如果没有设置最小值，或者设置的最小值比当前值还大，则需要更新最小值
+			if !minValue.Valid || minValue.Float64 > tmpValue {
+				err = minValue.Scan(tmpValue)
+				if err != nil {
+					return
+				}
+			}
+			// 如果没有设置最大值，或者设置的最大值比当前值还小，则需要更新最大值
+			if !maxValue.Valid || maxValue.Float64 < tmpValue {
+				err = maxValue.Scan(tmpValue)
+				if err != nil {
+					return
+				}
+			}
+		}
+
+		nonTrendChartInfo.DataList = dataList
+		nonTrendChartInfo.MinData = minValue.Float64
+		nonTrendChartInfo.MaxData = maxValue.Float64
+	}
+
 	return
 }
 
@@ -826,7 +913,7 @@ func SaveStlEdbInfo(req *request.SaveStlEdbInfoReq, adminId int, adminRealName,
 	}
 	var edbInfoData []*response.EdbData
 	switch req.StlEdbType {
-	case 1:
+	case utils.StlTypeTrend:
 		// 趋势指标
 		if ok := utils.Rc.IsExist(EDB_DATA_CALCULATE_STL_TREND_CACHE + strconv.Itoa(req.CalculateStlConfigId)); !ok {
 			msg = "计算已过期，请重新计算"
@@ -844,7 +931,7 @@ func SaveStlEdbInfo(req *request.SaveStlEdbInfoReq, adminId int, adminRealName,
 			err = fmt.Errorf("json解析失败，Err:" + er.Error())
 			return
 		}
-	case 2:
+	case utils.StlTypeSeasonal:
 		// 季节性指标
 		if ok := utils.Rc.IsExist(EDB_DATA_CALCULATE_STL_SEASONAL_CACHE + strconv.Itoa(req.CalculateStlConfigId)); !ok {
 			msg = "计算已过期，请重新计算"
@@ -862,7 +949,7 @@ func SaveStlEdbInfo(req *request.SaveStlEdbInfoReq, adminId int, adminRealName,
 			err = fmt.Errorf("json解析失败，Err:" + er.Error())
 			return
 		}
-	case 3:
+	case utils.StlTypeResidual:
 		// 残差性指标
 		if ok := utils.Rc.IsExist(EDB_DATA_CALCULATE_STL_RESIDUAL_CACHE + strconv.Itoa(req.CalculateStlConfigId)); !ok {
 			msg = "计算已过期，请重新计算"
@@ -880,6 +967,24 @@ func SaveStlEdbInfo(req *request.SaveStlEdbInfoReq, adminId int, adminRealName,
 			err = fmt.Errorf("json解析失败，Err:" + er.Error())
 			return
 		}
+	case utils.StlTypeNonTrend:
+		// 残差性指标
+		if ok := utils.Rc.IsExist(EDB_DATA_CALCULATE_STL_NonTrend_CACHE + strconv.Itoa(req.CalculateStlConfigId)); !ok {
+			msg = "计算已过期，请重新计算"
+			err = fmt.Errorf("not found")
+			return
+		}
+		nonTrendData, er := utils.Rc.RedisBytes(EDB_DATA_CALCULATE_STL_NonTrend_CACHE + strconv.Itoa(req.CalculateStlConfigId))
+		if er != nil {
+			msg = "获取失败"
+			err = fmt.Errorf("获取redis数据失败，Err:" + er.Error())
+			return
+		}
+		if er := json.Unmarshal(nonTrendData, &edbInfoData); er != nil {
+			msg = "获取失败"
+			err = fmt.Errorf("json解析失败，Err:" + er.Error())
+			return
+		}
 	default:
 		msg = "获取失败"
 		err = fmt.Errorf("未知的计算类型")
@@ -1190,7 +1295,7 @@ func SyncUpdateRelationEdbInfo(configId int, excludeId int) (err error) {
 		}
 		var edbInfoData []*response.EdbData
 		switch v.StlEdbType {
-		case 1:
+		case utils.StlTypeTrend:
 			// 趋势指标
 			if ok := utils.Rc.IsExist(EDB_DATA_CALCULATE_STL_TREND_CACHE + strconv.Itoa(v.CalculateStlConfigId)); !ok {
 				utils.FileLog.Info(EDB_DATA_CALCULATE_STL_TREND_CACHE + strconv.Itoa(v.CalculateStlConfigId) + "指标数据不存在")
@@ -1205,7 +1310,7 @@ func SyncUpdateRelationEdbInfo(configId int, excludeId int) (err error) {
 				utils.FileLog.Info("redis获取解析, body:%s,err:%s", string(trendData), er.Error())
 				continue
 			}
-		case 2:
+		case utils.StlTypeSeasonal:
 			// 季节性指标
 			if ok := utils.Rc.IsExist(EDB_DATA_CALCULATE_STL_SEASONAL_CACHE + strconv.Itoa(v.CalculateStlConfigId)); !ok {
 				utils.FileLog.Info(EDB_DATA_CALCULATE_STL_SEASONAL_CACHE + strconv.Itoa(v.CalculateStlConfigId) + "指标数据不存在")
@@ -1220,7 +1325,7 @@ func SyncUpdateRelationEdbInfo(configId int, excludeId int) (err error) {
 				utils.FileLog.Info("redis数据解析失败, body:%s,err:%s", string(seasonalData), er.Error())
 				continue
 			}
-		case 3:
+		case utils.StlTypeResidual:
 			// 残差性指标
 			if ok := utils.Rc.IsExist(EDB_DATA_CALCULATE_STL_RESIDUAL_CACHE + strconv.Itoa(v.CalculateStlConfigId)); !ok {
 				utils.FileLog.Info(EDB_DATA_CALCULATE_STL_RESIDUAL_CACHE + strconv.Itoa(v.CalculateStlConfigId) + "指标数据不存在")
@@ -1235,6 +1340,21 @@ func SyncUpdateRelationEdbInfo(configId int, excludeId int) (err error) {
 				utils.FileLog.Info("redis数据解析失败, body:%s,err:%s", string(residualData), er.Error())
 				continue
 			}
+		case utils.StlTypeNonTrend:
+			// 残差性指标
+			if ok := utils.Rc.IsExist(EDB_DATA_CALCULATE_STL_NonTrend_CACHE + strconv.Itoa(v.CalculateStlConfigId)); !ok {
+				utils.FileLog.Info(EDB_DATA_CALCULATE_STL_NonTrend_CACHE + strconv.Itoa(v.CalculateStlConfigId) + "指标数据不存在")
+				continue
+			}
+			nonTrendData, er := utils.Rc.RedisBytes(EDB_DATA_CALCULATE_STL_NonTrend_CACHE + strconv.Itoa(v.CalculateStlConfigId))
+			if er != nil {
+				utils.FileLog.Info(EDB_DATA_CALCULATE_STL_NonTrend_CACHE + strconv.Itoa(v.CalculateStlConfigId) + "redis获取失败,err:" + er.Error())
+				continue
+			}
+			if er := json.Unmarshal(nonTrendData, &edbInfoData); er != nil {
+				utils.FileLog.Info("redis数据解析失败, body:%s,err:%s", string(nonTrendData), er.Error())
+				continue
+			}
 		default:
 			utils.FileLog.Info("未知的stlEdbType类型, mapping:%v", v)
 			continue

services/data/trade_analysis/trade_analysis_data.go

@@ -67,7 +67,7 @@ func FormatCompanyTradeData2EdbMappings(companyTradeData []*tradeAnalysisModel.C
 				endTime, _ = time.ParseInLocation(utils.FormatDate, ed, time.Local)
 			}
 			if endTime.IsZero() {
-				endTime = time.Now()
+				endTime = time.Now().Local()
 			}
 		}
 

utils/constants.go

@@ -264,7 +264,7 @@ const (
 	CACHE_EXCEL_REFRESH = "CACHE_EXCEL_REFRESH" // 表格刷新
 
 	CACHE_DATA_SOURCE_ES_HANDLE = "eta:data_source_es:handle" // 数据源es处理队列
-	CACHE_CHART_AUTH      = "chart:auth:"     //图表数据授权
+	CACHE_CHART_AUTH            = "chart:auth:"               //图表数据授权
 )
 
 // 模板消息推送类型
@@ -538,6 +538,14 @@ const (
 	TableReferencedByEnPPT    = 5 // 英文PPT 与表格的关系
 )
 
+// STL模型类型
+const (
+	StlTypeTrend    = 1 // 趋势
+	StlTypeSeasonal = 2 // 季节性指标
+	StlTypeResidual = 3 // 残差性指标
+	StlTypeNonTrend = 4 // 非趋势性指标
+)
+
 // 基础数据初始化日期
 var (
 	BASE_START_DATE         = `1900-01-01`                                          //基础数据开始日期