package correlation import ( "encoding/json" "errors" "fmt" "github.com/shopspring/decimal" "hongze/hongze_yb/models/response/chart_info" "hongze/hongze_yb/models/tables/chart_edb_mapping" "hongze/hongze_yb/models/tables/chart_info_correlation" "hongze/hongze_yb/models/tables/edb_data" edbDataModel "hongze/hongze_yb/models/tables/edb_data" "hongze/hongze_yb/models/tables/factor_edb_series" "hongze/hongze_yb/models/tables/factor_edb_series_chart_mapping" "hongze/hongze_yb/services/alarm_msg" "hongze/hongze_yb/services/chart" "hongze/hongze_yb/utils" "math" "strings" "time" ) // HandleDataByLinearRegression 线性方程插值法补全数据 func HandleDataByLinearRegression(originList []*edb_data.EdbDataList, handleDataMap map[string]float64) (newList []*edb_data.EdbDataList, err error) { if len(originList) < 2 { return } var startEdbInfoData *edb_data.EdbDataList for _, v := range originList { handleDataMap[v.DataTime] = v.Value // 第一个数据就给过滤了,给后面的试用 if startEdbInfoData == nil { startEdbInfoData = v newList = append(newList, &edb_data.EdbDataList{ DataTime: v.DataTime, Value: v.Value, }) continue } // 获取两条数据之间相差的天数 startDataTime, _ := time.ParseInLocation(utils.FormatDate, startEdbInfoData.DataTime, time.Local) currDataTime, _ := time.ParseInLocation(utils.FormatDate, v.DataTime, time.Local) betweenHour := int(currDataTime.Sub(startDataTime).Hours()) betweenDay := betweenHour / 24 // 如果相差一天,那么过滤 if betweenDay <= 1 { startEdbInfoData = v newList = append(newList, &edb_data.EdbDataList{ DataTime: v.DataTime, Value: v.Value, }) continue } // 生成线性方程式 var a, b float64 { coordinateData := make([]utils.Coordinate, 0) tmpCoordinate1 := utils.Coordinate{ X: 1, Y: startEdbInfoData.Value, } coordinateData = append(coordinateData, tmpCoordinate1) tmpCoordinate2 := utils.Coordinate{ X: float64(betweenDay) + 1, Y: v.Value, } coordinateData = append(coordinateData, tmpCoordinate2) a, b = utils.GetLinearResult(coordinateData) if math.IsNaN(a) || math.IsNaN(b) { err = fmt.Errorf("线性方程公式生成失败") return } } // 生成对应的值 { for i := 1; i < betweenDay; i++ { tmpDataTime := startDataTime.AddDate(0, 0, i) aDecimal := decimal.NewFromFloat(a) xDecimal := decimal.NewFromInt(int64(i) + 1) bDecimal := decimal.NewFromFloat(b) val, _ := aDecimal.Mul(xDecimal).Add(bDecimal).Round(4).Float64() handleDataMap[tmpDataTime.Format(utils.FormatDate)] = val newList = append(newList, &edb_data.EdbDataList{ DataTime: tmpDataTime.Format(utils.FormatDate), Value: val, }) } } // 最后将自己赋值 newList = append(newList, &edb_data.EdbDataList{ EdbDataId: v.EdbDataId, DataTime: v.DataTime, Value: v.Value, }) startEdbInfoData = v } return } // MoveDataDaysToNewDataList 平移指标数据生成新的数据序列 func MoveDataDaysToNewDataList(dataList []*edb_data.EdbDataList, moveDay int) (newDataList []edb_data.EdbDataList, dateDataMap map[string]float64) { dateMap := make(map[time.Time]float64) var minDate, maxDate time.Time dateDataMap = make(map[string]float64) for _, v := range dataList { currDate, _ := time.ParseInLocation(utils.FormatDate, v.DataTime, time.Local) if minDate.IsZero() || currDate.Before(minDate) { minDate = currDate } if maxDate.IsZero() || currDate.After(maxDate) { maxDate = currDate } dateMap[currDate] = v.Value } // 处理领先、滞后数据 newDateMap := make(map[time.Time]float64) for currDate, value := range dateMap { newDate := currDate.AddDate(0, 0, moveDay) newDateMap[newDate] = value } minDate = minDate.AddDate(0, 0, moveDay) maxDate = maxDate.AddDate(0, 0, moveDay) // 获取日期相差日 dayNum := utils.GetTimeSubDay(minDate, maxDate) for i := 0; i <= dayNum; i++ { currDate := minDate.AddDate(0, 0, i) tmpValue, ok := newDateMap[currDate] if !ok { //找不到数据,那么就用前面的数据吧 if len(newDataList)-1 < 0 { tmpValue = 0 } else { tmpValue = newDataList[len(newDataList)-1].Value } } tmpData := edb_data.EdbDataList{ DataTime: currDate.Format(utils.FormatDate), Value: tmpValue, } dateDataMap[tmpData.DataTime] = tmpData.Value newDataList = append(newDataList, tmpData) } return } // GetChartEdbInfoFormat 相关性图表-获取指标信息 func GetChartEdbInfoFormat(chartInfoId int, edbInfoMappingA, edbInfoMappingB *chart_edb_mapping.ChartEdbInfoMapping) (edbList []*chart_edb_mapping.ChartEdbInfoMappingList, err error) { edbList = make([]*chart_edb_mapping.ChartEdbInfoMappingList, 0) if edbInfoMappingA == nil || edbInfoMappingB == nil { err = fmt.Errorf("指标信息有误") return } edbInfoMappingA.FrequencyEn = chart.GetFrequencyEn(edbInfoMappingA.Frequency) if edbInfoMappingA.Unit == `无` { edbInfoMappingA.Unit = `` } if edbInfoMappingB.Unit == `无` { edbInfoMappingB.Unit = `` } if chartInfoId <= 0 { edbInfoMappingA.IsAxis = 1 edbInfoMappingA.LeadValue = 0 edbInfoMappingA.LeadUnit = "" edbInfoMappingA.ChartEdbMappingId = 0 edbInfoMappingA.ChartInfoId = 0 edbInfoMappingA.IsOrder = false edbInfoMappingA.EdbInfoType = 1 edbInfoMappingA.ChartStyle = "" edbInfoMappingA.ChartColor = "" edbInfoMappingA.ChartWidth = 0 edbInfoMappingB.IsAxis = 1 edbInfoMappingB.LeadValue = 0 edbInfoMappingB.LeadUnit = "" edbInfoMappingB.ChartEdbMappingId = 0 edbInfoMappingB.ChartInfoId = 0 edbInfoMappingB.IsOrder = false edbInfoMappingB.EdbInfoType = 1 edbInfoMappingB.ChartStyle = "" edbInfoMappingB.ChartColor = "" edbInfoMappingB.ChartWidth = 0 } else { edbInfoMappingA.LeadUnitEn = chart.GetLeadUnitEn(edbInfoMappingA.LeadUnit) edbInfoMappingB.LeadUnitEn = chart.GetLeadUnitEn(edbInfoMappingB.LeadUnit) } aList := new(chart_edb_mapping.ChartEdbInfoMappingList) aList.ChartEdbInfoMapping = *edbInfoMappingA bList := new(chart_edb_mapping.ChartEdbInfoMappingList) bList.ChartEdbInfoMapping = *edbInfoMappingB edbList = append(edbList, aList, bList) return } // GetChartDataByEdbInfo 相关性图表-根据指标信息获取x轴和y轴 func GetChartDataByEdbInfo(edbInfoMappingA, edbInfoMappingB *chart_edb_mapping.ChartEdbInfoMapping, leadValue int, leadUnit, startDate, endDate string) (xEdbIdValue []int, yDataList []chart_info.YData, err error) { xData := make([]int, 0) yData := make([]float64, 0) if leadValue == 0 { xData = append(xData, 0) } if leadValue > 0 { leadMin := 0 - leadValue xLen := 2*leadValue + 1 for i := 0; i < xLen; i++ { n := leadMin + i xData = append(xData, n) } } // 计算窗口,不包含第一天 startDateTime, _ := time.ParseInLocation(utils.FormatDate, startDate, time.Local) startDate = startDateTime.AddDate(0, 0, 1).Format(utils.FormatDate) //// 2023-03-02 时间序列始终以指标B为基准, 始终是A进行平移 //baseEdbInfo := edbInfoMappingB //changeEdbInfo := edbInfoMappingA // 2023-03-17 时间序列始终以指标A为基准, 始终是B进行平移 baseEdbInfo := edbInfoMappingA changeEdbInfo := edbInfoMappingB // 获取时间基准指标在时间区间内的值 aDataList := make([]*edb_data.EdbDataList, 0) switch baseEdbInfo.EdbInfoCategoryType { case 0: aDataList, err = edb_data.GetEdbDataList(baseEdbInfo.Source, baseEdbInfo.SubSource, baseEdbInfo.EdbInfoId, startDate, endDate) case 1: _, aDataList, _, _, err, _ = chart.GetPredictDataListByPredictEdbInfoId(baseEdbInfo.EdbInfoId, startDate, endDate, false) default: err = errors.New("指标base类型异常") return } // 获取变频指标所有日期的值, 插值法完善数据 bDataList := make([]*edb_data.EdbDataList, 0) switch changeEdbInfo.EdbInfoCategoryType { case 0: bDataList, err = edb_data.GetEdbDataList(changeEdbInfo.Source, changeEdbInfo.SubSource, changeEdbInfo.EdbInfoId, "", "") case 1: _, bDataList, _, _, err, _ = chart.GetPredictDataListByPredictEdbInfoId(changeEdbInfo.EdbInfoId, "", "", false) default: err = errors.New("指标change类型异常") return } //changeDataMap := make(map[string]float64) //newChangeDataList, e := HandleDataByLinearRegression(bDataList, changeDataMap) //if e != nil { // err = fmt.Errorf("获取变频指标插值法Map失败, Err: %s", e.Error()) // return //} // 2023-03-17 时间序列始终以指标A为基准, 始终是B进行平移 baseDataList := make([]*edb_data.EdbDataList, 0) baseDataMap := make(map[string]float64) changeDataList := make([]*edb_data.EdbDataList, 0) changeDataMap := make(map[string]float64) // 先把低频指标升频为高频 { frequencyIntMap := map[string]int{ "日度": 1, "周度": 2, "旬度": 3, "月度": 4, "季度": 5, "年度": 6, } // 如果A指标是高频,那么就需要对B指标进行升频 if frequencyIntMap[edbInfoMappingA.Frequency] < frequencyIntMap[edbInfoMappingB.Frequency] { tmpNewChangeDataList, e := HandleDataByLinearRegression(bDataList, changeDataMap) if e != nil { err = fmt.Errorf("获取变频指标插值法Map失败, Err: %s", e.Error()) return } changeDataList = tmpNewChangeDataList baseDataList = aDataList for _, v := range baseDataList { baseDataMap[v.DataTime] = v.Value } } else if frequencyIntMap[edbInfoMappingA.Frequency] > frequencyIntMap[edbInfoMappingB.Frequency] { // 如果B指标是高频,那么就需要对A指标进行升频 tmpNewChangeDataList, e := HandleDataByLinearRegression(aDataList, baseDataMap) if e != nil { err = fmt.Errorf("获取变频指标插值法Map失败, Err: %s", e.Error()) return } baseDataList = tmpNewChangeDataList changeDataList = bDataList for _, v := range changeDataList { changeDataMap[v.DataTime] = v.Value } } else { baseDataList = aDataList for _, v := range baseDataList { baseDataMap[v.DataTime] = v.Value } changeDataList = bDataList for _, v := range changeDataList { changeDataMap[v.DataTime] = v.Value } } } // 计算不领先也不滞后时的相关系数 baseCalculateData := make([]float64, 0) baseDataTimeArr := make([]string, 0) for i := range baseDataList { baseDataTimeArr = append(baseDataTimeArr, baseDataList[i].DataTime) baseCalculateData = append(baseCalculateData, baseDataList[i].Value) } //zeroBaseData := make([]float64, 0) //zeroCalculateData := make([]float64, 0) //for i := range baseDataTimeArr { // tmpBaseVal, ok1 := baseDataMap[baseDataTimeArr[i]] // tmpCalculateVal, ok2 := changeDataMap[baseDataTimeArr[i]] // if ok1 && ok2 { // zeroBaseData = append(zeroBaseData, tmpBaseVal) // zeroCalculateData = append(zeroCalculateData, tmpCalculateVal) // } //} //if len(zeroBaseData) != len(zeroCalculateData) { // err = fmt.Errorf("相关系数两组序列元素数不一致, %d-%d", len(baseCalculateData), len(zeroCalculateData)) // return //} //zeroRatio := utils.CalculateCorrelationByIntArr(zeroBaseData, zeroCalculateData) //if leadValue == 0 { // yData = append(yData, zeroRatio) //} // 计算领先/滞后N期 if leadValue > 0 { // 平移变频指标领先/滞后的日期(单位天) moveUnitDays := utils.FrequencyDaysMap[leadUnit] for i := range xData { //if xData[i] == 0 { // yData = append(yData, zeroRatio) // continue //} xCalculateData := make([]float64, 0) yCalculateData := make([]float64, 0) // 平移指定天数 mDays := int(moveUnitDays) * xData[i] _, dMap := MoveDataDaysToNewDataList(changeDataList, mDays) // 取出对应的基准日期的值 for i2 := range baseDataTimeArr { if yVal, ok := dMap[baseDataTimeArr[i2]]; ok { xCalculateData = append(xCalculateData, baseCalculateData[i2]) yCalculateData = append(yCalculateData, yVal) } } if len(yCalculateData) <= 0 { //err = fmt.Errorf("领先滞后相关系数两组序列元素数不一致, %d-%d", len(baseCalculateData), len(yCalculateData)) //return // 领先滞后后,没有可以计算的数据了 continue } // 公式计算出领先/滞后频度对应点的相关性系数 ratio := utils.CalculateCorrelationByIntArr(xCalculateData, yCalculateData) yData = append(yData, ratio) } } xEdbIdValue = xData yDataList = make([]chart_info.YData, 0) yDate := "0000-00-00" yDataList = append(yDataList, chart_info.YData{ Date: yDate, Value: yData, }) return } // RollingCorrelationChartDataResp 滚动相关性图的数据 type RollingCorrelationChartDataResp struct { MaxData float64 MinData float64 LatestDate string `description:"真实数据的最后日期"` EdbInfoCategoryType int ChartColor string ChartStyle string PredictChartColor string ChartType int ChartWidth int EdbName string EdbNameEn string Unit string UnitEn string IsAxis int DataList []edbDataModel.EdbDataList } // GetRollingCorrelationChartDataByEdbInfo 滚动相关性计算 func GetRollingCorrelationChartDataByEdbInfo(edbInfoMappingA, edbInfoMappingB *chart_edb_mapping.ChartEdbInfoMapping, leadValue int, leadUnit string, calculateValue int, calculateUnit string, startDate, endDate, chartName, chartNameEn string) (dataResp RollingCorrelationChartDataResp, err error) { dataResp = RollingCorrelationChartDataResp{ DataList: make([]edbDataModel.EdbDataList, 0), MaxData: 0, MinData: 0, ChartColor: "#00f", ChartStyle: `spline`, PredictChartColor: `#00f`, ChartType: 0, ChartWidth: 3, EdbName: chartName, EdbNameEn: chartNameEn, IsAxis: 1, } dataList := make([]edbDataModel.EdbDataList, 0) // 计算窗口,不包含第一天 startDateTime, _ := time.ParseInLocation(utils.FormatDate, startDate, time.Local) startDate = startDateTime.AddDate(0, 0, 1).Format(utils.FormatDate) baseEdbInfo := edbInfoMappingA changeEdbInfo := edbInfoMappingB // 获取时间基准指标在时间区间内的值 aDataList := make([]*edb_data.EdbDataList, 0) switch baseEdbInfo.EdbInfoCategoryType { case 0: aDataList, err = edb_data.GetEdbDataList(baseEdbInfo.Source, baseEdbInfo.SubSource, baseEdbInfo.EdbInfoId, startDate, endDate) case 1: _, aDataList, _, _, err, _ = chart.GetPredictDataListByPredictEdbInfoId(baseEdbInfo.EdbInfoId, startDate, endDate, false) default: err = errors.New("指标base类型异常") return } // 获取变频指标所有日期的值, 插值法完善数据 bDataList := make([]*edb_data.EdbDataList, 0) switch changeEdbInfo.EdbInfoCategoryType { case 0: bDataList, err = edb_data.GetEdbDataList(changeEdbInfo.Source, changeEdbInfo.SubSource, changeEdbInfo.EdbInfoId, "", "") case 1: _, bDataList, _, _, err, _ = chart.GetPredictDataListByPredictEdbInfoId(changeEdbInfo.EdbInfoId, "", "", false) default: err = errors.New("指标change类型异常") return } // 数据平移变频指标领先/滞后的日期(单位天) // 2023-03-17 时间序列始终以指标A为基准, 始终是B进行平移 //baseDataList := make([]*edb_data.EdbDataList, 0) baseDataMap := make(map[string]float64) changeDataList := make([]*edb_data.EdbDataList, 0) changeDataMap := make(map[string]float64) // A指标不管三七二十一,先变个频再说 { _, e := HandleDataByLinearRegression(aDataList, baseDataMap) if e != nil { err = fmt.Errorf("获取变频指标插值法Map失败, Err: %s", e.Error()) return } //baseDataList = tmpNewChangeDataList } // B指标不管三七二十一,先变个频再说 { tmpNewChangeDataList, e := HandleDataByLinearRegression(bDataList, changeDataMap) if e != nil { err = fmt.Errorf("获取变频指标插值法Map失败, Err: %s", e.Error()) return } changeDataList = tmpNewChangeDataList // 平移下日期 moveUnitDays := utils.FrequencyDaysMap[leadUnit] _, changeDataMap = MoveDataDaysToNewDataList(changeDataList, leadValue*moveUnitDays) } // 计算计算时,需要多少个日期内数据 calculateDay := utils.FrequencyDaysMap[calculateUnit] * calculateValue // 计算 每个日期的相关性值 { startDateTime, _ := time.ParseInLocation(utils.FormatDate, startDate, time.Local) if endDate == `` { // 因为时间格式是:2023-07-19T00:00:00+08:00;所以需要分开 endDateList := strings.Split(baseEdbInfo.EndDate, "T") endDate = endDateList[0] } endDateTime, _ := time.ParseInLocation(utils.FormatDate, endDate, time.Local) endDateTime = endDateTime.AddDate(0, 0, -(calculateDay - 1)) // 是否开始第一条数据 var isStart, isNotFirst bool for currDay := startDateTime; !currDay.After(endDateTime); currDay = currDay.AddDate(0, 0, 1) { yCalculateData := make([]float64, 0) baseCalculateData := make([]float64, 0) // 取出对应的基准日期的值 for i := 0; i < calculateDay; i++ { iDay := currDay.AddDate(0, 0, i).Format(utils.FormatDate) tmpBaseValue, ok1 := baseDataMap[iDay] tmpChangeValue, ok2 := changeDataMap[iDay] if ok1 && ok2 { baseCalculateData = append(baseCalculateData, tmpBaseValue) yCalculateData = append(yCalculateData, tmpChangeValue) } else { continue } } // 公式计算出领先/滞后频度对应点的相关性系数 var ratio float64 if len(baseCalculateData) > 0 { ratio = utils.CalculateCorrelationByIntArr(baseCalculateData, yCalculateData) } else { // 没有数据的话,那就不返回 continue } // 过滤前面都是0的数据 { if ratio != 0 { isStart = true } if !isStart { continue } } dataTime := currDay.AddDate(0, 0, calculateDay-1) dataList = append(dataList, edbDataModel.EdbDataList{ //EdbDataId: 0, EdbInfoId: 0, DataTime: dataTime.Format(utils.FormatDate), DataTimestamp: dataTime.UnixNano() / 1e6, Value: ratio, }) if !isNotFirst { dataResp.MinData = ratio dataResp.MaxData = ratio isNotFirst = true } if dataResp.MinData > ratio { dataResp.MinData = ratio } if dataResp.MaxData < ratio { dataResp.MaxData = ratio } } dataResp.DataList = dataList } return } // ChartInfoRefresh 图表刷新 func ChartInfoRefresh(chartInfoId int) (err error) { var errMsg string defer func() { if err != nil { //fmt.Println(err.Error()) go alarm_msg.SendAlarmMsg("CorrelationChartInfoRefresh: "+errMsg, 3) } }() correlationChart := new(chart_info_correlation.ChartInfoCorrelation) if err = correlationChart.GetItemById(chartInfoId); err != nil { errMsg = "获取相关性图表失败, Err: " + err.Error() return } // 批量刷新ETA指标 err, errMsg = chart.EdbInfoRefreshAllFromBase([]int{correlationChart.EdbInfoIdFirst, correlationChart.EdbInfoIdSecond}, false) if err != nil { return } // 重新生成数据并更新 edbInfoMappingA, err := chart_edb_mapping.GetChartEdbMappingByEdbInfoId(correlationChart.EdbInfoIdFirst) if err != nil { errMsg = "获取相关性图表, A指标mapping信息失败, Err:" + err.Error() return } edbInfoMappingB, err := chart_edb_mapping.GetChartEdbMappingByEdbInfoId(correlationChart.EdbInfoIdSecond) if err != nil { errMsg = "获取相关性图表, B指标mapping信息失败, Err:" + err.Error() return } periodData, correlationData, err := GetChartDataByEdbInfo(edbInfoMappingA, edbInfoMappingB, correlationChart.LeadValue, correlationChart.LeadUnit, correlationChart.StartDate.Format(utils.FormatDate), correlationChart.EndDate.Format(utils.FormatDate)) if err != nil { errMsg = "获取相关性图表, 图表计算值失败, Err:" + err.Error() return } periodDataByte, err := json.Marshal(periodData) if err != nil { errMsg = "相关性图表, X轴信息有误, Err:" + err.Error() return } correlationDataByte, err := json.Marshal(correlationData[0].Value) if err != nil { errMsg = "相关性图表, Y轴信息有误, Err:" + err.Error() return } correlationChart.PeriodData = string(periodDataByte) correlationChart.CorrelationData = string(correlationDataByte) correlationChart.ModifyTime = time.Now().Local() correlationUpdateCols := []string{"PeriodData", "CorrelationData", "ModifyTime"} if err = correlationChart.Update(correlationUpdateCols); err != nil { errMsg = "更新相关性图表失败, Err:" + err.Error() return } return } // GetFactorChartDataByChartId 获取多因子相关性图表数据 func GetFactorChartDataByChartId(chartInfoId int, extraConfig string) (xEdbIdValue []int, yDataList []chart_info.YData, err error) { if chartInfoId <= 0 { return } // 指标对应的图例 extra := new(chart_info_correlation.CorrelationChartInfoExtraConfig) if extraConfig != "" { if e := json.Unmarshal([]byte(extraConfig), extra); e != nil { err = fmt.Errorf("解析图表额外配置失败, err: %v", e) return } } legends := make(map[string]*chart_info_correlation.CorrelationChartLegend) if extra != nil { for _, v := range extra.LegendConfig { s := fmt.Sprintf("%d-%d", v.SeriesId, v.EdbInfoId) legends[s] = v } } // 获取图表引用到的系列指标 chartMappings, e := factor_edb_series_chart_mapping.GetChartUsedFactorSeriesEdb(chartInfoId) if e != nil { err = fmt.Errorf("获取图表引用系列指标失败, err: %v", e) return } // 取出计算结果 yDataList = make([]chart_info.YData, 0) yDate := "0000-00-00" for k, m := range chartMappings { var values []factor_edb_series.FactorEdbSeriesCorrelationMatrixValues if m.CalculateData != "" { e = json.Unmarshal([]byte(m.CalculateData), &values) if e != nil { err = fmt.Errorf("系列指标计算数据有误, err: %v", e) return } } var y []float64 for _, v := range values { if k == 0 { xEdbIdValue = append(xEdbIdValue, v.XData) } y = append(y, v.YData) } var yData chart_info.YData yData.Date = yDate yData.Value = y yData.SeriesEdb.SeriesId = m.FactorEdbSeriesId yData.SeriesEdb.EdbInfoId = m.EdbInfoId // 图例 s := fmt.Sprintf("%d-%d", m.FactorEdbSeriesId, m.EdbInfoId) legend := legends[s] if legend != nil { yData.Name = legend.LegendName yData.Color = legend.Color } yDataList = append(yDataList, yData) } return }