|
|
@@ -1 +1,257 @@
|
|
|
package stl
|
|
|
+
|
|
|
+import (
|
|
|
+ "encoding/json"
|
|
|
+ "eta/eta_api/models/data_manage"
|
|
|
+ "eta/eta_api/models/data_manage/stl/request"
|
|
|
+ "eta/eta_api/models/data_manage/stl/response"
|
|
|
+ "eta/eta_api/utils"
|
|
|
+ "fmt"
|
|
|
+ "os"
|
|
|
+ "os/exec"
|
|
|
+ "path/filepath"
|
|
|
+ "strings"
|
|
|
+ "time"
|
|
|
+
|
|
|
+ "github.com/tealeg/xlsx"
|
|
|
+)
|
|
|
+
|
|
|
+const (
|
|
|
+ ALL_DATE = iota + 1
|
|
|
+ LAST_N_YEARS
|
|
|
+ RANGE_DATE
|
|
|
+ RANGE_DATE_TO_NOW
|
|
|
+)
|
|
|
+
|
|
|
+func GenerateStlEdbData(req *request.STLReq) (resp *response.StlPreviewResp, msg string, err error) {
|
|
|
+ edbInfo, err := data_manage.GetEdbInfoById(req.EdbInfoId)
|
|
|
+ if err != nil {
|
|
|
+ if err.Error() == utils.ErrNoRow() {
|
|
|
+ msg = "指标不存在"
|
|
|
+ return
|
|
|
+ }
|
|
|
+ msg = "获取指标信息失败"
|
|
|
+ return
|
|
|
+ }
|
|
|
+ var condition string
|
|
|
+ var pars []interface{}
|
|
|
+ switch req.DataRangeType {
|
|
|
+ case ALL_DATE:
|
|
|
+ case LAST_N_YEARS:
|
|
|
+ condition += " AND data_time >=?"
|
|
|
+ year := time.Now().Year()
|
|
|
+ lastDate := time.Date(year-req.LastNYear, 1, 1, 0, 0, 0, 0, time.Local)
|
|
|
+ pars = append(pars, lastDate)
|
|
|
+ case RANGE_DATE:
|
|
|
+ condition = " AND data_time >=? AND data_time <=?"
|
|
|
+ pars = append(pars, req.StartDate, req.EndDate)
|
|
|
+ case RANGE_DATE_TO_NOW:
|
|
|
+ condition = " AND data_time >=?"
|
|
|
+ pars = append(pars, req.StartDate)
|
|
|
+ }
|
|
|
+ condition += " AND edb_code =?"
|
|
|
+ pars = append(pars, edbInfo.EdbCode)
|
|
|
+
|
|
|
+ edbData, err := data_manage.GetAllEdbDataListByCondition(condition, pars, edbInfo.Source, edbInfo.SubSource)
|
|
|
+ if err != nil {
|
|
|
+ msg = "获取指标数据失败"
|
|
|
+ return
|
|
|
+ }
|
|
|
+ dir, _ := os.Executable()
|
|
|
+ exPath := filepath.Dir(dir)
|
|
|
+ loadFilePath := exPath + "/" + time.Now().Format(utils.FormatDateTimeUnSpace) + ".xlsx"
|
|
|
+ err = SaveToExcel(edbData, loadFilePath)
|
|
|
+ if err != nil {
|
|
|
+ msg = "保存数据到Excel失败"
|
|
|
+ return
|
|
|
+ }
|
|
|
+ defer os.Remove(loadFilePath)
|
|
|
+
|
|
|
+ saveFilePath := exPath + "/" + time.Now().Format(utils.FormatDateTimeUnSpace) + "_res" + ".xlsx"
|
|
|
+ result, err := execStlPythonCode(loadFilePath, saveFilePath, req.Period, req.Seasonal, req.Trend, req.Fraction, req.TrendDeg, req.SeasonalDeg, req.LowPassDeg, req.Robust)
|
|
|
+ if err != nil {
|
|
|
+ msg = "计算失败，请重新选择指标和参数后计算"
|
|
|
+ }
|
|
|
+ trendChart, seasonalChart, residualChart, err := ParseStlExcel(saveFilePath)
|
|
|
+ if err != nil {
|
|
|
+ msg = "解析Excel失败"
|
|
|
+ return
|
|
|
+ }
|
|
|
+ resp.EvaluationResult.Mean = result.ResidualMean
|
|
|
+ resp.EvaluationResult.Std = result.ResidualVar
|
|
|
+ resp.EvaluationResult.AdfPValue = result.AdfPValue
|
|
|
+ resp.EvaluationResult.LjungBoxPValue = result.LbTestPValue
|
|
|
+
|
|
|
+ return
|
|
|
+}
|
|
|
+
|
|
|
+func ParseStlExcel(excelPath string) (TrendChart, SeasonalChart, ResidualChart response.ChartEdbInfo, err error) {
|
|
|
+ file, err := xlsx.OpenFile(excelPath)
|
|
|
+ if err != nil {
|
|
|
+ return
|
|
|
+ }
|
|
|
+ defer os.Remove(excelPath)
|
|
|
+ for _, sheet := range file.Sheets {
|
|
|
+ switch sheet.Name {
|
|
|
+ case "季节":
|
|
|
+ for i, row := range sheet.Rows {
|
|
|
+ if i == 0 {
|
|
|
+ continue
|
|
|
+ }
|
|
|
+ date := row.Cells[0].String()
|
|
|
+ date = strings.Split(date, " ")[0]
|
|
|
+ value := row.Cells[1].String()
|
|
|
+ SeasonalChart.DataList = append(SeasonalChart.DataList, &response.EdbData{DataTime: date, Value: value})
|
|
|
+ }
|
|
|
+ case "趋势":
|
|
|
+ for i, row := range sheet.Rows {
|
|
|
+ if i == 0 {
|
|
|
+ continue
|
|
|
+ }
|
|
|
+ date := row.Cells[0].String()
|
|
|
+ date = strings.Split(date, " ")[0]
|
|
|
+ value := row.Cells[1].String()
|
|
|
+ TrendChart.DataList = append(TrendChart.DataList, &response.EdbData{DataTime: date, Value: value})
|
|
|
+ }
|
|
|
+ case "残差":
|
|
|
+ for i, row := range sheet.Rows {
|
|
|
+ if i == 0 {
|
|
|
+ continue
|
|
|
+ }
|
|
|
+ date := row.Cells[0].String()
|
|
|
+ date = strings.Split(date, " ")[0]
|
|
|
+ value := row.Cells[1].String()
|
|
|
+ ResidualChart.DataList = append(ResidualChart.DataList, &response.EdbData{DataTime: date, Value: value})
|
|
|
+ }
|
|
|
+ }
|
|
|
+ }
|
|
|
+ return
|
|
|
+}
|
|
|
+
|
|
|
+func SaveToExcel(data []*data_manage.EdbData, filePath string) (err error) {
|
|
|
+ xlsxFile := xlsx.NewFile()
|
|
|
+ sheetNew, err := xlsxFile.AddSheet("Tmp")
|
|
|
+ if err != nil {
|
|
|
+ return
|
|
|
+ }
|
|
|
+ titleRow := sheetNew.AddRow()
|
|
|
+ titleRow.AddCell().SetString("日期")
|
|
|
+ titleRow.AddCell().SetString("值")
|
|
|
+
|
|
|
+ for i, d := range data {
|
|
|
+ row := sheetNew.Row(i + 1)
|
|
|
+ row.AddCell().SetString(d.DataTime)
|
|
|
+ row.AddCell().SetFloat(d.Value)
|
|
|
+ }
|
|
|
+ err = xlsxFile.Save(filePath)
|
|
|
+ if err != nil {
|
|
|
+ return
|
|
|
+ }
|
|
|
+ return
|
|
|
+}
|
|
|
+
|
|
|
+type STLResult struct {
|
|
|
+ ResidualMean string `json:"residual_mean"`
|
|
|
+ ResidualVar string `json:"residual_var"`
|
|
|
+ AdfPValue string `json:"adf_p_value"`
|
|
|
+ LbTestPValue string `json:"lb_test_p_value"`
|
|
|
+ LbTestStat string `json:"lb_test_stat"`
|
|
|
+}
|
|
|
+
|
|
|
+func execStlPythonCode(path, toPath string, period, seasonal, trend, fraction, trendDeg, seasonalDeg, lowPassDeg int, robust bool) (stlResult *STLResult, err error) {
|
|
|
+ pythonCode := `
|
|
|
+import pandas as pd
|
|
|
+from statsmodels.tsa.seasonal import STL
|
|
|
+from statsmodels.nonparametric.smoothers_lowess import lowess
|
|
|
+from statsmodels.tsa.stattools import adfuller
|
|
|
+from statsmodels.stats.diagnostic import acorr_ljungbox
|
|
|
+import numpy as np
|
|
|
+import json
|
|
|
+
|
|
|
+file_path = r"%s"
|
|
|
+df = pd.read_excel(file_path, parse_dates=['日期'])
|
|
|
+df.set_index('日期', inplace=True)
|
|
|
+
|
|
|
+
|
|
|
+period = %d
|
|
|
+seasonal = %d
|
|
|
+trend = %d
|
|
|
+fraction = %d
|
|
|
+seasonal_deg = %d
|
|
|
+trend_deg = %d
|
|
|
+low_pass_deg = %d
|
|
|
+robust = %s
|
|
|
+
|
|
|
+stl = STL(
|
|
|
+ df['值'],
|
|
|
+ period=period,
|
|
|
+ seasonal=seasonal,
|
|
|
+ trend=trend,
|
|
|
+ low_pass=None,
|
|
|
+ seasonal_deg=seasonal_deg,
|
|
|
+ trend_deg=trend_deg,
|
|
|
+ low_pass_deg=low_pass_deg,
|
|
|
+ seasonal_jump=1,
|
|
|
+ trend_jump=1,
|
|
|
+ low_pass_jump=1,
|
|
|
+ robust=robust
|
|
|
+)
|
|
|
+result = stl.fit()
|
|
|
+
|
|
|
+smoothed = lowess(df['值'], df.index, frac=fraction)
|
|
|
+
|
|
|
+trend_lowess = smoothed[:, 1]
|
|
|
+
|
|
|
+# 季节图
|
|
|
+seasonal_component = result.seasonal
|
|
|
+# 趋势图
|
|
|
+trend_lowess_series = pd.Series(trend_lowess, index=df.index)
|
|
|
+# 残差图
|
|
|
+residual_component = df['值'] - trend_lowess - seasonal_component
|
|
|
+
|
|
|
+# 计算打印残差的均值
|
|
|
+residual_mean = np.mean(residual_component)
|
|
|
+# 计算打印残差的方差
|
|
|
+residual_var = np.std(residual_component)
|
|
|
+# 计算打印残差的ADF检验结果, 输出p-value
|
|
|
+adf_result = adfuller(residual_component)
|
|
|
+# 根据p-value判断是否平稳
|
|
|
+lb_test = acorr_ljungbox(residual_component, lags=period, return_df=True)
|
|
|
+
|
|
|
+output_file = r"%s"
|
|
|
+
|
|
|
+with pd.ExcelWriter(output_file) as writer:
|
|
|
+ # 保存季节图
|
|
|
+ pd.Series(seasonal_component, index=df.index, name='值').to_frame().reset_index().rename(columns={'index': '日期'}).to_excel(writer, sheet_name='季节', index=False)
|
|
|
+ # 保存趋势图
|
|
|
+ trend_lowess_series.to_frame(name='值').reset_index().rename(columns={'index': '日期'}).to_excel(writer, sheet_name='趋势', index=False)
|
|
|
+ # 保存残差图
|
|
|
+ pd.Series(residual_component, index=df.index, name='值').to_frame().reset_index().rename(columns={'index': '日期'}).to_excel(writer, sheet_name='残差', index=False)
|
|
|
+
|
|
|
+output = json.dumps({
|
|
|
+ 'residual_mean': residual_mean,
|
|
|
+ 'residual_var': residual_var,
|
|
|
+ 'adf_p_value': adf_result[1],
|
|
|
+ 'lb_test_p_value': lb_test['lb_pvalue'].values[0],
|
|
|
+ 'lb_test_stat': lb_test['lb_stat'].values[0]
|
|
|
+})
|
|
|
+
|
|
|
+print(output)
|
|
|
+ `
|
|
|
+ robustStr := "True"
|
|
|
+ if !robust {
|
|
|
+ robustStr = "False"
|
|
|
+ }
|
|
|
+
|
|
|
+ pythonCode = fmt.Sprintf(pythonCode, path, period, seasonal, trend, fraction, seasonalDeg, trendDeg, lowPassDeg, robustStr, toPath)
|
|
|
+ cmd := exec.Command("python3", "-c", pythonCode)
|
|
|
+ output, err := cmd.CombinedOutput()
|
|
|
+ if err != nil {
|
|
|
+ return
|
|
|
+ }
|
|
|
+ defer cmd.Process.Kill()
|
|
|
+ if err = json.Unmarshal(output, &stlResult); err != nil {
|
|
|
+ return
|
|
|
+ }
|
|
|
+ return
|
|
|
+}
|
zqbao