fix:stl计算生成图

controllers/data_manage/stl/stl.go

@@ -1,8 +1,12 @@
 package stl
 
 import (
+	"encoding/json"
 	"eta/eta_api/controllers"
 	"eta/eta_api/models"
+	"eta/eta_api/services/data/stl"
+
+	"eta/eta_api/models/data_manage/stl/request"
 )
 
 type STLController struct {
@@ -27,5 +31,19 @@ func (c *STLController) Preview() {
 		br.Ret = 408
 		return
 	}
+	var req *request.STLReq
+	if err := json.Unmarshal(c.Ctx.Input.RequestBody, &req); err != nil {
+		br.Msg = "请求参数错误"
+		br.ErrMsg = err.Error()
+		br.Ret = 400
+		return
+	}
+	if e := json.Unmarshal(c.Ctx.Input.RequestBody, &req); e != nil {
+		br.Msg = "参数解析异常！"
+		br.ErrMsg = "参数解析失败，Err:" + e.Error()
+		return
+	}
+
+	stl.GenerateStlEdbData(req)
 
 }

models/data_manage/stl/request/stl.go

@@ -1,5 +1,17 @@
 package request
 
 type STLReq struct {
-	Data string `json:"data"`
+	EdbInfoId     int    `description:"指标ID"`
+	DataRangeType int    `description:"数据时间类型：1-全部时间，2-最近N年，3-区间设置，4-区间设置(至今)"`
+	StartDate     string `description:"开始日期"`
+	EndDate       string `description:"结束日期"`
+	LastNYear     int    `description:"最近N年"`
+	Period        int    `description:"数据的周期，根据频率设置"`
+	Seasonal      int    `description:"季节性成分窗口大小，一般为period+1,可以设置为大于period的正奇数"`
+	Trend         int    `description:"趋势成分窗口大小，一般为period+1,可以设置为大于period的正奇数"`
+	Fraction      int    `description:"趋势项的平滑系数，默认为0.2，区间为[0-1]"`
+	Robust        bool   `description:"是否使用稳健方法: true(使用) false(不使用)  "`
+	TrendDeg      int    `description:"分解中趋势多项式次数，默认为1，不超过5的正整数"`
+	SeasonalDeg   int    `description:"分解中季节性多项次数，默认为1，不超过5的正整数"`
+	LowPassDeg    int    `description:"分解中低通滤波器次数，默认为1，不超过5的正整数"`
 }

models/data_manage/stl/response/stl.go

@@ -0,0 +1,33 @@
+package response
+
+type StlPreviewResp struct {
+	OriginEdbInfo     ChartEdbInfo
+	TrendChartInfo    ChartEdbInfo
+	SeasonalChartInfo ChartEdbInfo
+	ResidualChartInfo ChartEdbInfo
+	EvaluationResult  EvaluationResult
+}
+
+type ChartEdbInfo struct {
+	Title        string
+	Unit         string
+	UnitEn       string
+	Frequency    string
+	FrequencyEn  string
+	DataTime     string
+	ClassifyId   string
+	ClassifyPath string
+	DataList     []*EdbData
+}
+
+type EvaluationResult struct {
+	Mean           string `description:"均值"`
+	Std            string `description:"标准差"`
+	AdfPValue      string `description:"ADF检验p值"`
+	LjungBoxPValue string `description:"Ljung-Box检验p值"`
+}
+
+type EdbData struct {
+	Value    string
+	DataTime string
+}

services/data/stl/stl.go

@@ -1 +1,257 @@
 package stl
+
+import (
+	"encoding/json"
+	"eta/eta_api/models/data_manage"
+	"eta/eta_api/models/data_manage/stl/request"
+	"eta/eta_api/models/data_manage/stl/response"
+	"eta/eta_api/utils"
+	"fmt"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"strings"
+	"time"
+
+	"github.com/tealeg/xlsx"
+)
+
+const (
+	ALL_DATE = iota + 1
+	LAST_N_YEARS
+	RANGE_DATE
+	RANGE_DATE_TO_NOW
+)
+
+func GenerateStlEdbData(req *request.STLReq) (resp *response.StlPreviewResp, msg string, err error) {
+	edbInfo, err := data_manage.GetEdbInfoById(req.EdbInfoId)
+	if err != nil {
+		if err.Error() == utils.ErrNoRow() {
+			msg = "指标不存在"
+			return
+		}
+		msg = "获取指标信息失败"
+		return
+	}
+	var condition string
+	var pars []interface{}
+	switch req.DataRangeType {
+	case ALL_DATE:
+	case LAST_N_YEARS:
+		condition += " AND data_time >=?"
+		year := time.Now().Year()
+		lastDate := time.Date(year-req.LastNYear, 1, 1, 0, 0, 0, 0, time.Local)
+		pars = append(pars, lastDate)
+	case RANGE_DATE:
+		condition = " AND data_time >=? AND data_time <=?"
+		pars = append(pars, req.StartDate, req.EndDate)
+	case RANGE_DATE_TO_NOW:
+		condition = " AND data_time >=?"
+		pars = append(pars, req.StartDate)
+	}
+	condition += " AND edb_code =?"
+	pars = append(pars, edbInfo.EdbCode)
+
+	edbData, err := data_manage.GetAllEdbDataListByCondition(condition, pars, edbInfo.Source, edbInfo.SubSource)
+	if err != nil {
+		msg = "获取指标数据失败"
+		return
+	}
+	dir, _ := os.Executable()
+	exPath := filepath.Dir(dir)
+	loadFilePath := exPath + "/" + time.Now().Format(utils.FormatDateTimeUnSpace) + ".xlsx"
+	err = SaveToExcel(edbData, loadFilePath)
+	if err != nil {
+		msg = "保存数据到Excel失败"
+		return
+	}
+	defer os.Remove(loadFilePath)
+
+	saveFilePath := exPath + "/" + time.Now().Format(utils.FormatDateTimeUnSpace) + "_res" + ".xlsx"
+	result, err := execStlPythonCode(loadFilePath, saveFilePath, req.Period, req.Seasonal, req.Trend, req.Fraction, req.TrendDeg, req.SeasonalDeg, req.LowPassDeg, req.Robust)
+	if err != nil {
+		msg = "计算失败，请重新选择指标和参数后计算"
+	}
+	trendChart, seasonalChart, residualChart, err := ParseStlExcel(saveFilePath)
+	if err != nil {
+		msg = "解析Excel失败"
+		return
+	}
+	resp.EvaluationResult.Mean = result.ResidualMean
+	resp.EvaluationResult.Std = result.ResidualVar
+	resp.EvaluationResult.AdfPValue = result.AdfPValue
+	resp.EvaluationResult.LjungBoxPValue = result.LbTestPValue
+
+	return
+}
+
+func ParseStlExcel(excelPath string) (TrendChart, SeasonalChart, ResidualChart response.ChartEdbInfo, err error) {
+	file, err := xlsx.OpenFile(excelPath)
+	if err != nil {
+		return
+	}
+	defer os.Remove(excelPath)
+	for _, sheet := range file.Sheets {
+		switch sheet.Name {
+		case "季节":
+			for i, row := range sheet.Rows {
+				if i == 0 {
+					continue
+				}
+				date := row.Cells[0].String()
+				date = strings.Split(date, " ")[0]
+				value := row.Cells[1].String()
+				SeasonalChart.DataList = append(SeasonalChart.DataList, &response.EdbData{DataTime: date, Value: value})
+			}
+		case "趋势":
+			for i, row := range sheet.Rows {
+				if i == 0 {
+					continue
+				}
+				date := row.Cells[0].String()
+				date = strings.Split(date, " ")[0]
+				value := row.Cells[1].String()
+				TrendChart.DataList = append(TrendChart.DataList, &response.EdbData{DataTime: date, Value: value})
+			}
+		case "残差":
+			for i, row := range sheet.Rows {
+				if i == 0 {
+					continue
+				}
+				date := row.Cells[0].String()
+				date = strings.Split(date, " ")[0]
+				value := row.Cells[1].String()
+				ResidualChart.DataList = append(ResidualChart.DataList, &response.EdbData{DataTime: date, Value: value})
+			}
+		}
+	}
+	return
+}
+
+func SaveToExcel(data []*data_manage.EdbData, filePath string) (err error) {
+	xlsxFile := xlsx.NewFile()
+	sheetNew, err := xlsxFile.AddSheet("Tmp")
+	if err != nil {
+		return
+	}
+	titleRow := sheetNew.AddRow()
+	titleRow.AddCell().SetString("日期")
+	titleRow.AddCell().SetString("值")
+
+	for i, d := range data {
+		row := sheetNew.Row(i + 1)
+		row.AddCell().SetString(d.DataTime)
+		row.AddCell().SetFloat(d.Value)
+	}
+	err = xlsxFile.Save(filePath)
+	if err != nil {
+		return
+	}
+	return
+}
+
+type STLResult struct {
+	ResidualMean string `json:"residual_mean"`
+	ResidualVar  string `json:"residual_var"`
+	AdfPValue    string `json:"adf_p_value"`
+	LbTestPValue string `json:"lb_test_p_value"`
+	LbTestStat   string `json:"lb_test_stat"`
+}
+
+func execStlPythonCode(path, toPath string, period, seasonal, trend, fraction, trendDeg, seasonalDeg, lowPassDeg int, robust bool) (stlResult *STLResult, err error) {
+	pythonCode := `
+import pandas as pd
+from statsmodels.tsa.seasonal import STL
+from statsmodels.nonparametric.smoothers_lowess import lowess
+from statsmodels.tsa.stattools import adfuller
+from statsmodels.stats.diagnostic import acorr_ljungbox
+import numpy as np
+import json
+
+file_path = r"%s"
+df = pd.read_excel(file_path, parse_dates=['日期'])
+df.set_index('日期', inplace=True)
+
+
+period = %d
+seasonal = %d
+trend = %d
+fraction = %d
+seasonal_deg = %d
+trend_deg = %d
+low_pass_deg = %d
+robust = %s
+
+stl = STL(
+    df['值'],
+    period=period,
+    seasonal=seasonal,
+    trend=trend,
+    low_pass=None,
+    seasonal_deg=seasonal_deg,
+    trend_deg=trend_deg,
+    low_pass_deg=low_pass_deg,
+    seasonal_jump=1,
+    trend_jump=1,
+    low_pass_jump=1,
+    robust=robust
+)
+result = stl.fit()
+
+smoothed = lowess(df['值'], df.index, frac=fraction)
+
+trend_lowess = smoothed[:, 1]
+
+# 季节图
+seasonal_component = result.seasonal
+# 趋势图
+trend_lowess_series = pd.Series(trend_lowess, index=df.index)
+# 残差图
+residual_component = df['值'] - trend_lowess - seasonal_component
+
+# 计算打印残差的均值
+residual_mean = np.mean(residual_component)
+# 计算打印残差的方差
+residual_var = np.std(residual_component)
+# 计算打印残差的ADF检验结果, 输出p-value
+adf_result = adfuller(residual_component)
+# 根据p-value判断是否平稳
+lb_test = acorr_ljungbox(residual_component, lags=period, return_df=True)
+
+output_file = r"%s"
+
+with pd.ExcelWriter(output_file) as writer:
+    # 保存季节图
+    pd.Series(seasonal_component, index=df.index, name='值').to_frame().reset_index().rename(columns={'index': '日期'}).to_excel(writer, sheet_name='季节', index=False)
+    # 保存趋势图
+    trend_lowess_series.to_frame(name='值').reset_index().rename(columns={'index': '日期'}).to_excel(writer, sheet_name='趋势', index=False)
+    # 保存残差图
+    pd.Series(residual_component, index=df.index, name='值').to_frame().reset_index().rename(columns={'index': '日期'}).to_excel(writer, sheet_name='残差', index=False)
+
+output =  json.dumps({
+    'residual_mean': residual_mean,
+    'residual_var': residual_var,
+    'adf_p_value': adf_result[1],
+    'lb_test_p_value': lb_test['lb_pvalue'].values[0],
+    'lb_test_stat': lb_test['lb_stat'].values[0]
+})
+
+print(output)
+	`
+	robustStr := "True"
+	if !robust {
+		robustStr = "False"
+	}
+
+	pythonCode = fmt.Sprintf(pythonCode, path, period, seasonal, trend, fraction, seasonalDeg, trendDeg, lowPassDeg, robustStr, toPath)
+	cmd := exec.Command("python3", "-c", pythonCode)
+	output, err := cmd.CombinedOutput()
+	if err != nil {
+		return
+	}
+	defer cmd.Process.Kill()
+	if err = json.Unmarshal(output, &stlResult); err != nil {
+		return
+	}
+	return
+}