|
|
@@ -0,0 +1,385 @@
|
|
|
+package correlation
|
|
|
+
|
|
|
+import (
|
|
|
+ "encoding/json"
|
|
|
+ "errors"
|
|
|
+ "fmt"
|
|
|
+ "github.com/shopspring/decimal"
|
|
|
+ "hongze/hongze_yb/models/response/chart_info"
|
|
|
+ "hongze/hongze_yb/models/tables/chart_edb_mapping"
|
|
|
+ "hongze/hongze_yb/models/tables/chart_info_correlation"
|
|
|
+ "hongze/hongze_yb/models/tables/edb_data"
|
|
|
+ "hongze/hongze_yb/services/alarm_msg"
|
|
|
+ "hongze/hongze_yb/services/chart"
|
|
|
+ "hongze/hongze_yb/utils"
|
|
|
+ "math"
|
|
|
+ "time"
|
|
|
+)
|
|
|
+
|
|
|
+// HandleDataByLinearRegression 线性方程插值法补全数据
|
|
|
+func HandleDataByLinearRegression(originList []*edb_data.EdbDataList, handleDataMap map[string]float64) (newList []*edb_data.EdbDataList, err error) {
|
|
|
+ if len(originList) < 2 {
|
|
|
+ return
|
|
|
+ }
|
|
|
+
|
|
|
+ var startEdbInfoData *edb_data.EdbDataList
|
|
|
+ for _, v := range originList {
|
|
|
+ handleDataMap[v.DataTime] = v.Value
|
|
|
+
|
|
|
+ // 第一个数据就给过滤了,给后面的试用
|
|
|
+ if startEdbInfoData == nil {
|
|
|
+ startEdbInfoData = v
|
|
|
+ newList = append(newList, &edb_data.EdbDataList{
|
|
|
+ DataTime: v.DataTime,
|
|
|
+ Value: v.Value,
|
|
|
+ })
|
|
|
+ continue
|
|
|
+ }
|
|
|
+
|
|
|
+ // 获取两条数据之间相差的天数
|
|
|
+ startDataTime, _ := time.ParseInLocation(utils.FormatDate, startEdbInfoData.DataTime, time.Local)
|
|
|
+ currDataTime, _ := time.ParseInLocation(utils.FormatDate, v.DataTime, time.Local)
|
|
|
+ betweenHour := int(currDataTime.Sub(startDataTime).Hours())
|
|
|
+ betweenDay := betweenHour / 24
|
|
|
+
|
|
|
+ // 如果相差一天,那么过滤
|
|
|
+ if betweenDay <= 1 {
|
|
|
+ startEdbInfoData = v
|
|
|
+ newList = append(newList, &edb_data.EdbDataList{
|
|
|
+ DataTime: v.DataTime,
|
|
|
+ Value: v.Value,
|
|
|
+ })
|
|
|
+ continue
|
|
|
+ }
|
|
|
+
|
|
|
+ // 生成线性方程式
|
|
|
+ var a, b float64
|
|
|
+ {
|
|
|
+ coordinateData := make([]utils.Coordinate, 0)
|
|
|
+ tmpCoordinate1 := utils.Coordinate{
|
|
|
+ X: 1,
|
|
|
+ Y: startEdbInfoData.Value,
|
|
|
+ }
|
|
|
+ coordinateData = append(coordinateData, tmpCoordinate1)
|
|
|
+ tmpCoordinate2 := utils.Coordinate{
|
|
|
+ X: float64(betweenDay) + 1,
|
|
|
+ Y: v.Value,
|
|
|
+ }
|
|
|
+ coordinateData = append(coordinateData, tmpCoordinate2)
|
|
|
+
|
|
|
+ a, b = utils.GetLinearResult(coordinateData)
|
|
|
+ if math.IsNaN(a) || math.IsNaN(b) {
|
|
|
+ err = fmt.Errorf("线性方程公式生成失败")
|
|
|
+ return
|
|
|
+ }
|
|
|
+ }
|
|
|
+
|
|
|
+ // 生成对应的值
|
|
|
+ {
|
|
|
+ for i := 1; i < betweenDay; i++ {
|
|
|
+ tmpDataTime := startDataTime.AddDate(0, 0, i)
|
|
|
+ aDecimal := decimal.NewFromFloat(a)
|
|
|
+ xDecimal := decimal.NewFromInt(int64(i) + 1)
|
|
|
+ bDecimal := decimal.NewFromFloat(b)
|
|
|
+
|
|
|
+ val, _ := aDecimal.Mul(xDecimal).Add(bDecimal).Round(4).Float64()
|
|
|
+ handleDataMap[tmpDataTime.Format(utils.FormatDate)] = val
|
|
|
+ newList = append(newList, &edb_data.EdbDataList{
|
|
|
+ DataTime: tmpDataTime.Format(utils.FormatDate),
|
|
|
+ Value: val,
|
|
|
+ })
|
|
|
+ }
|
|
|
+ }
|
|
|
+
|
|
|
+ startEdbInfoData = v
|
|
|
+ }
|
|
|
+ return
|
|
|
+}
|
|
|
+
|
|
|
+// MoveDataDaysToNewDataList 平移指标数据生成新的数据序列
|
|
|
+func MoveDataDaysToNewDataList(dataList []*edb_data.EdbDataList, moveDay int) (newDataList []edb_data.EdbDataList, dateDataMap map[string]float64) {
|
|
|
+ dateMap := make(map[time.Time]float64)
|
|
|
+ var minDate, maxDate time.Time
|
|
|
+ dateDataMap = make(map[string]float64)
|
|
|
+
|
|
|
+ for _, v := range dataList {
|
|
|
+ currDate, _ := time.ParseInLocation(utils.FormatDate, v.DataTime, time.Local)
|
|
|
+ if minDate.IsZero() || currDate.Before(minDate) {
|
|
|
+ minDate = currDate
|
|
|
+ }
|
|
|
+ if maxDate.IsZero() || currDate.After(maxDate) {
|
|
|
+ maxDate = currDate
|
|
|
+ }
|
|
|
+ dateMap[currDate] = v.Value
|
|
|
+ }
|
|
|
+
|
|
|
+ // 处理领先、滞后数据
|
|
|
+ newDateMap := make(map[time.Time]float64)
|
|
|
+ for currDate, value := range dateMap {
|
|
|
+ newDate := currDate.AddDate(0, 0, moveDay)
|
|
|
+ newDateMap[newDate] = value
|
|
|
+ }
|
|
|
+ minDate = minDate.AddDate(0, 0, moveDay)
|
|
|
+ maxDate = maxDate.AddDate(0, 0, moveDay)
|
|
|
+
|
|
|
+ // 获取日期相差日
|
|
|
+ dayNum := utils.GetTimeSubDay(minDate, maxDate)
|
|
|
+
|
|
|
+ for i := 0; i <= dayNum; i++ {
|
|
|
+ currDate := minDate.AddDate(0, 0, i)
|
|
|
+ tmpValue, ok := newDateMap[currDate]
|
|
|
+ if !ok {
|
|
|
+ //找不到数据，那么就用前面的数据吧
|
|
|
+ if len(newDataList)-1 < 0 {
|
|
|
+ tmpValue = 0
|
|
|
+ } else {
|
|
|
+ tmpValue = newDataList[len(newDataList)-1].Value
|
|
|
+ }
|
|
|
+ }
|
|
|
+ tmpData := edb_data.EdbDataList{
|
|
|
+ DataTime: currDate.Format(utils.FormatDate),
|
|
|
+ Value: tmpValue,
|
|
|
+ }
|
|
|
+ dateDataMap[tmpData.DataTime] = tmpData.Value
|
|
|
+ newDataList = append(newDataList, tmpData)
|
|
|
+ }
|
|
|
+ return
|
|
|
+}
|
|
|
+
|
|
|
+// GetChartEdbInfoFormat 相关性图表-获取指标信息
|
|
|
+func GetChartEdbInfoFormat(chartInfoId int, edbInfoMappingA, edbInfoMappingB *chart_edb_mapping.ChartEdbInfoMapping) (edbList []*chart_edb_mapping.ChartEdbInfoMappingList, err error) {
|
|
|
+ edbList = make([]*chart_edb_mapping.ChartEdbInfoMappingList, 0)
|
|
|
+ if edbInfoMappingA == nil || edbInfoMappingB == nil {
|
|
|
+ err = fmt.Errorf("指标信息有误")
|
|
|
+ return
|
|
|
+ }
|
|
|
+
|
|
|
+ edbInfoMappingA.FrequencyEn = chart.GetFrequencyEn(edbInfoMappingA.Frequency)
|
|
|
+ if edbInfoMappingA.Unit == `无` {
|
|
|
+ edbInfoMappingA.Unit = ``
|
|
|
+ }
|
|
|
+ if edbInfoMappingB.Unit == `无` {
|
|
|
+ edbInfoMappingB.Unit = ``
|
|
|
+ }
|
|
|
+ if chartInfoId <= 0 {
|
|
|
+ edbInfoMappingA.IsAxis = 1
|
|
|
+ edbInfoMappingA.LeadValue = 0
|
|
|
+ edbInfoMappingA.LeadUnit = ""
|
|
|
+ edbInfoMappingA.ChartEdbMappingId = 0
|
|
|
+ edbInfoMappingA.ChartInfoId = 0
|
|
|
+ edbInfoMappingA.IsOrder = false
|
|
|
+ edbInfoMappingA.EdbInfoType = 1
|
|
|
+ edbInfoMappingA.ChartStyle = ""
|
|
|
+ edbInfoMappingA.ChartColor = ""
|
|
|
+ edbInfoMappingA.ChartWidth = 0
|
|
|
+
|
|
|
+ edbInfoMappingB.IsAxis = 1
|
|
|
+ edbInfoMappingB.LeadValue = 0
|
|
|
+ edbInfoMappingB.LeadUnit = ""
|
|
|
+ edbInfoMappingB.ChartEdbMappingId = 0
|
|
|
+ edbInfoMappingB.ChartInfoId = 0
|
|
|
+ edbInfoMappingB.IsOrder = false
|
|
|
+ edbInfoMappingB.EdbInfoType = 1
|
|
|
+ edbInfoMappingB.ChartStyle = ""
|
|
|
+ edbInfoMappingB.ChartColor = ""
|
|
|
+ edbInfoMappingB.ChartWidth = 0
|
|
|
+ } else {
|
|
|
+ edbInfoMappingA.LeadUnitEn = chart.GetLeadUnitEn(edbInfoMappingA.LeadUnit)
|
|
|
+ edbInfoMappingB.LeadUnitEn = chart.GetLeadUnitEn(edbInfoMappingB.LeadUnit)
|
|
|
+ }
|
|
|
+ aList := new(chart_edb_mapping.ChartEdbInfoMappingList)
|
|
|
+ aList.ChartEdbInfoMapping = *edbInfoMappingA
|
|
|
+ bList := new(chart_edb_mapping.ChartEdbInfoMappingList)
|
|
|
+ bList.ChartEdbInfoMapping = *edbInfoMappingB
|
|
|
+ edbList = append(edbList, aList, bList)
|
|
|
+ return
|
|
|
+}
|
|
|
+
|
|
|
+// GetChartDataByEdbInfo 相关性图表-根据指标信息获取x轴和y轴
|
|
|
+func GetChartDataByEdbInfo(edbInfoMappingA, edbInfoMappingB *chart_edb_mapping.ChartEdbInfoMapping, leadValue int, leadUnit, startDate, endDate string) (xEdbIdValue []int, yDataList []chart_info.YData, err error) {
|
|
|
+ xData := make([]int, 0)
|
|
|
+ yData := make([]float64, 0)
|
|
|
+ if leadValue == 0 {
|
|
|
+ xData = append(xData, 0)
|
|
|
+ }
|
|
|
+ if leadValue > 0 {
|
|
|
+ leadMin := 0 - leadValue
|
|
|
+ xLen := 2*leadValue + 1
|
|
|
+ for i := 0; i < xLen; i++ {
|
|
|
+ n := leadMin + i
|
|
|
+ xData = append(xData, n)
|
|
|
+ }
|
|
|
+ }
|
|
|
+
|
|
|
+ // 若不同频, 则需要将低频指标变为高频
|
|
|
+ frequencyIntMap := map[string]int{
|
|
|
+ "日度": 1, "周度": 2, "旬度": 3, "月度": 4, "季度": 5, "年度": 6,
|
|
|
+ }
|
|
|
+ baseEdbInfo := edbInfoMappingA
|
|
|
+ changeEdbInfo := edbInfoMappingB
|
|
|
+ if edbInfoMappingA.Frequency != edbInfoMappingB.Frequency {
|
|
|
+ if frequencyIntMap[edbInfoMappingA.Frequency] >= frequencyIntMap[edbInfoMappingB.Frequency] {
|
|
|
+ baseEdbInfo = edbInfoMappingB
|
|
|
+ changeEdbInfo = edbInfoMappingA
|
|
|
+ }
|
|
|
+ }
|
|
|
+
|
|
|
+ // 获取时间基准指标在时间区间内的值
|
|
|
+ baseDataList := make([]*edb_data.EdbDataList, 0)
|
|
|
+ switch baseEdbInfo.EdbInfoCategoryType {
|
|
|
+ case 0:
|
|
|
+ baseDataList, err = edb_data.GetEdbDataList(baseEdbInfo.Source, baseEdbInfo.EdbInfoId, startDate, endDate)
|
|
|
+ case 1:
|
|
|
+ _, baseDataList, _, _, err, _ = chart.GetPredictDataListByPredictEdbInfoId(baseEdbInfo.EdbInfoId, startDate, endDate, false)
|
|
|
+ default:
|
|
|
+ err = errors.New("指标base类型异常")
|
|
|
+ return
|
|
|
+ }
|
|
|
+
|
|
|
+ // 获取变频指标所有日期的值, 插值法完善数据
|
|
|
+ changeDataList := make([]*edb_data.EdbDataList, 0)
|
|
|
+ switch changeEdbInfo.EdbInfoCategoryType {
|
|
|
+ case 0:
|
|
|
+ changeDataList, err = edb_data.GetEdbDataList(changeEdbInfo.Source, changeEdbInfo.EdbInfoId, "", "")
|
|
|
+ case 1:
|
|
|
+ _, changeDataList, _, _, err, _ = chart.GetPredictDataListByPredictEdbInfoId(changeEdbInfo.EdbInfoId, "", "", false)
|
|
|
+ default:
|
|
|
+ err = errors.New("指标change类型异常")
|
|
|
+ return
|
|
|
+ }
|
|
|
+ changeDataMap := make(map[string]float64)
|
|
|
+ newChangeDataList, e := HandleDataByLinearRegression(changeDataList, changeDataMap)
|
|
|
+ if e != nil {
|
|
|
+ err = fmt.Errorf("获取变频指标插值法Map失败, Err: %s", e.Error())
|
|
|
+ return
|
|
|
+ }
|
|
|
+
|
|
|
+ // 计算不领先也不滞后时的相关系数
|
|
|
+ baseCalculateData := make([]float64, 0)
|
|
|
+ baseDataTimeArr := make([]string, 0)
|
|
|
+ for i := range baseDataList {
|
|
|
+ baseDataTimeArr = append(baseDataTimeArr, baseDataList[i].DataTime)
|
|
|
+ baseCalculateData = append(baseCalculateData, baseDataList[i].Value)
|
|
|
+ }
|
|
|
+ zeroCalculateData := make([]float64, 0)
|
|
|
+ for i := range baseDataTimeArr {
|
|
|
+ zeroCalculateData = append(zeroCalculateData, changeDataMap[baseDataTimeArr[i]])
|
|
|
+ }
|
|
|
+ if len(baseCalculateData) != len(zeroCalculateData) {
|
|
|
+ err = fmt.Errorf("相关系数两组序列元素数不一致, %d-%d", len(baseCalculateData), len(zeroCalculateData))
|
|
|
+ return
|
|
|
+ }
|
|
|
+ zeroRatio := utils.CalculateCorrelationByIntArr(baseCalculateData, zeroCalculateData)
|
|
|
+ if leadValue == 0 {
|
|
|
+ yData = append(yData, zeroRatio)
|
|
|
+ }
|
|
|
+
|
|
|
+ // 计算领先/滞后N期
|
|
|
+ if leadValue > 0 {
|
|
|
+ // 平移变频指标领先/滞后的日期(单位天)
|
|
|
+ frequencyDaysMap := map[string]float64{
|
|
|
+ "天": 1, "周": 7, "月": 30, "季": 90, "年": 365,
|
|
|
+ }
|
|
|
+ moveUnitDays := frequencyDaysMap[leadUnit]
|
|
|
+
|
|
|
+ // 此处需注意, 图表代表的意思是A领先/滞后B指标的相关系数, 只是时间序列以高频的为基准
|
|
|
+ // 所以需要判断高频指标是A还是B
|
|
|
+ // 举例: 当高频指标为A时, 即B为变频, newChangeDataList为B的值, A领先B三天则mDays应为+3, 取B滞后三天的值与基准日期的时间相比较
|
|
|
+ for i := range xData {
|
|
|
+ if xData[i] == 0 {
|
|
|
+ yData = append(yData, zeroRatio)
|
|
|
+ continue
|
|
|
+ }
|
|
|
+ yCalculateData := make([]float64, 0)
|
|
|
+
|
|
|
+ // 判断是向前平移还是向后平移
|
|
|
+ mDays := 0
|
|
|
+ if baseEdbInfo.EdbInfoId == edbInfoMappingA.EdbInfoId {
|
|
|
+ mDays = int(moveUnitDays) * -xData[i]
|
|
|
+ } else {
|
|
|
+ mDays = int(moveUnitDays) * xData[i]
|
|
|
+ }
|
|
|
+ _, dMap := MoveDataDaysToNewDataList(newChangeDataList, mDays)
|
|
|
+
|
|
|
+ // 取出对应的基准日期的值
|
|
|
+ for i2 := range baseDataTimeArr {
|
|
|
+ yCalculateData = append(yCalculateData, dMap[baseDataTimeArr[i2]])
|
|
|
+ }
|
|
|
+ if len(baseCalculateData) != len(yCalculateData) {
|
|
|
+ err = fmt.Errorf("领先滞后相关系数两组序列元素数不一致, %d-%d", len(baseCalculateData), len(yCalculateData))
|
|
|
+ return
|
|
|
+ }
|
|
|
+
|
|
|
+ // 公式计算出领先/滞后频度对应点的相关性系数
|
|
|
+ ratio := utils.CalculateCorrelationByIntArr(baseCalculateData, yCalculateData)
|
|
|
+ yData = append(yData, ratio)
|
|
|
+ }
|
|
|
+ }
|
|
|
+
|
|
|
+ xEdbIdValue = xData
|
|
|
+ yDataList = make([]chart_info.YData, 0)
|
|
|
+ yDate := "0000-00-00"
|
|
|
+ yDataList = append(yDataList, chart_info.YData{
|
|
|
+ Date: yDate,
|
|
|
+ Value: yData,
|
|
|
+ })
|
|
|
+ return
|
|
|
+}
|
|
|
+
|
|
|
+// ChartInfoRefresh 图表刷新
|
|
|
+func ChartInfoRefresh(chartInfoId int) (err error) {
|
|
|
+ var errMsg string
|
|
|
+ defer func() {
|
|
|
+ if err != nil {
|
|
|
+ //fmt.Println(err.Error())
|
|
|
+ go alarm_msg.SendAlarmMsg("CorrelationChartInfoRefresh: "+errMsg, 3)
|
|
|
+ }
|
|
|
+ }()
|
|
|
+ correlationChart := new(chart_info_correlation.ChartInfoCorrelation)
|
|
|
+ if err = correlationChart.GetItemById(chartInfoId); err != nil {
|
|
|
+ errMsg = "获取相关性图表失败, Err: " + err.Error()
|
|
|
+ return
|
|
|
+ }
|
|
|
+
|
|
|
+ // 批量刷新ETA指标
|
|
|
+ err, errMsg = chart.EdbInfoRefreshAllFromBase([]int{correlationChart.EdbInfoIdFirst, correlationChart.EdbInfoIdSecond}, false)
|
|
|
+ if err != nil {
|
|
|
+ return
|
|
|
+ }
|
|
|
+
|
|
|
+ // 重新生成数据并更新
|
|
|
+ edbInfoMappingA, err := chart_edb_mapping.GetChartEdbMappingByEdbInfoId(correlationChart.EdbInfoIdFirst)
|
|
|
+ if err != nil {
|
|
|
+ errMsg = "获取相关性图表, A指标mapping信息失败, Err:" + err.Error()
|
|
|
+ return
|
|
|
+ }
|
|
|
+ edbInfoMappingB, err := chart_edb_mapping.GetChartEdbMappingByEdbInfoId(correlationChart.EdbInfoIdSecond)
|
|
|
+ if err != nil {
|
|
|
+ errMsg = "获取相关性图表, B指标mapping信息失败, Err:" + err.Error()
|
|
|
+ return
|
|
|
+ }
|
|
|
+ periodData, correlationData, err := GetChartDataByEdbInfo(edbInfoMappingA, edbInfoMappingB, correlationChart.LeadValue, correlationChart.LeadUnit, correlationChart.StartDate.Format(utils.FormatDate), correlationChart.EndDate.Format(utils.FormatDate))
|
|
|
+ if err != nil {
|
|
|
+ errMsg = "获取相关性图表, 图表计算值失败, Err:" + err.Error()
|
|
|
+ return
|
|
|
+ }
|
|
|
+ periodDataByte, err := json.Marshal(periodData)
|
|
|
+ if err != nil {
|
|
|
+ errMsg = "相关性图表, X轴信息有误, Err:" + err.Error()
|
|
|
+ return
|
|
|
+ }
|
|
|
+ correlationDataByte, err := json.Marshal(correlationData[0].Value)
|
|
|
+ if err != nil {
|
|
|
+ errMsg = "相关性图表, Y轴信息有误, Err:" + err.Error()
|
|
|
+ return
|
|
|
+ }
|
|
|
+ correlationChart.PeriodData = string(periodDataByte)
|
|
|
+ correlationChart.CorrelationData = string(correlationDataByte)
|
|
|
+ correlationChart.ModifyTime = time.Now().Local()
|
|
|
+ correlationUpdateCols := []string{"PeriodData", "CorrelationData", "ModifyTime"}
|
|
|
+ if err = correlationChart.Update(correlationUpdateCols); err != nil {
|
|
|
+ errMsg = "更新相关性图表失败, Err:" + err.Error()
|
|
|
+ return
|
|
|
+ }
|
|
|
+ return
|
|
|
+}
|
