hongze_yb/services/chart/correlation/chart_info.go

package correlation

import (
	"encoding/json"
	"errors"
	"fmt"
	"github.com/shopspring/decimal"
	"hongze/hongze_yb/models/response/chart_info"
	"hongze/hongze_yb/models/tables/chart_edb_mapping"
	"hongze/hongze_yb/models/tables/chart_info_correlation"
	"hongze/hongze_yb/models/tables/edb_data"
	"hongze/hongze_yb/services/alarm_msg"
	"hongze/hongze_yb/services/chart"
	"hongze/hongze_yb/utils"
	"math"
	"time"
)

// HandleDataByLinearRegression 线性方程插值法补全数据
func HandleDataByLinearRegression(originList []*edb_data.EdbDataList, handleDataMap map[string]float64) (newList []*edb_data.EdbDataList, err error) {
	if len(originList) < 2 {
		return
	}

	var startEdbInfoData *edb_data.EdbDataList
	for _, v := range originList {
		handleDataMap[v.DataTime] = v.Value

		// 第一个数据就给过滤了,给后面的试用
		if startEdbInfoData == nil {
			startEdbInfoData = v
			newList = append(newList, &edb_data.EdbDataList{
				DataTime: v.DataTime,
				Value:    v.Value,
			})
			continue
		}

		// 获取两条数据之间相差的天数
		startDataTime, _ := time.ParseInLocation(utils.FormatDate, startEdbInfoData.DataTime, time.Local)
		currDataTime, _ := time.ParseInLocation(utils.FormatDate, v.DataTime, time.Local)
		betweenHour := int(currDataTime.Sub(startDataTime).Hours())
		betweenDay := betweenHour / 24

		// 如果相差一天,那么过滤
		if betweenDay <= 1 {
			startEdbInfoData = v
			newList = append(newList, &edb_data.EdbDataList{
				DataTime: v.DataTime,
				Value:    v.Value,
			})
			continue
		}

		// 生成线性方程式
		var a, b float64
		{
			coordinateData := make([]utils.Coordinate, 0)
			tmpCoordinate1 := utils.Coordinate{
				X: 1,
				Y: startEdbInfoData.Value,
			}
			coordinateData = append(coordinateData, tmpCoordinate1)
			tmpCoordinate2 := utils.Coordinate{
				X: float64(betweenDay) + 1,
				Y: v.Value,
			}
			coordinateData = append(coordinateData, tmpCoordinate2)

			a, b = utils.GetLinearResult(coordinateData)
			if math.IsNaN(a) || math.IsNaN(b) {
				err = fmt.Errorf("线性方程公式生成失败")
				return
			}
		}

		// 生成对应的值
		{
			for i := 1; i < betweenDay; i++ {
				tmpDataTime := startDataTime.AddDate(0, 0, i)
				aDecimal := decimal.NewFromFloat(a)
				xDecimal := decimal.NewFromInt(int64(i) + 1)
				bDecimal := decimal.NewFromFloat(b)

				val, _ := aDecimal.Mul(xDecimal).Add(bDecimal).Round(4).Float64()
				handleDataMap[tmpDataTime.Format(utils.FormatDate)] = val
				newList = append(newList, &edb_data.EdbDataList{
					DataTime: tmpDataTime.Format(utils.FormatDate),
					Value:    val,
				})
			}
		}

		startEdbInfoData = v
	}
	return
}

// MoveDataDaysToNewDataList 平移指标数据生成新的数据序列
func MoveDataDaysToNewDataList(dataList []*edb_data.EdbDataList, moveDay int) (newDataList []edb_data.EdbDataList, dateDataMap map[string]float64) {
	dateMap := make(map[time.Time]float64)
	var minDate, maxDate time.Time
	dateDataMap = make(map[string]float64)

	for _, v := range dataList {
		currDate, _ := time.ParseInLocation(utils.FormatDate, v.DataTime, time.Local)
		if minDate.IsZero() || currDate.Before(minDate) {
			minDate = currDate
		}
		if maxDate.IsZero() || currDate.After(maxDate) {
			maxDate = currDate
		}
		dateMap[currDate] = v.Value
	}

	// 处理领先、滞后数据
	newDateMap := make(map[time.Time]float64)
	for currDate, value := range dateMap {
		newDate := currDate.AddDate(0, 0, moveDay)
		newDateMap[newDate] = value
	}
	minDate = minDate.AddDate(0, 0, moveDay)
	maxDate = maxDate.AddDate(0, 0, moveDay)

	// 获取日期相差日
	dayNum := utils.GetTimeSubDay(minDate, maxDate)

	for i := 0; i <= dayNum; i++ {
		currDate := minDate.AddDate(0, 0, i)
		tmpValue, ok := newDateMap[currDate]
		if !ok {
			//找不到数据，那么就用前面的数据吧
			if len(newDataList)-1 < 0 {
				tmpValue = 0
			} else {
				tmpValue = newDataList[len(newDataList)-1].Value
			}
		}
		tmpData := edb_data.EdbDataList{
			DataTime: currDate.Format(utils.FormatDate),
			Value:    tmpValue,
		}
		dateDataMap[tmpData.DataTime] = tmpData.Value
		newDataList = append(newDataList, tmpData)
	}
	return
}

// GetChartEdbInfoFormat 相关性图表-获取指标信息
func GetChartEdbInfoFormat(chartInfoId int, edbInfoMappingA, edbInfoMappingB *chart_edb_mapping.ChartEdbInfoMapping) (edbList []*chart_edb_mapping.ChartEdbInfoMappingList, err error) {
	edbList = make([]*chart_edb_mapping.ChartEdbInfoMappingList, 0)
	if edbInfoMappingA == nil || edbInfoMappingB == nil {
		err = fmt.Errorf("指标信息有误")
		return
	}

	edbInfoMappingA.FrequencyEn = chart.GetFrequencyEn(edbInfoMappingA.Frequency)
	if edbInfoMappingA.Unit == `无` {
		edbInfoMappingA.Unit = ``
	}
	if edbInfoMappingB.Unit == `无` {
		edbInfoMappingB.Unit = ``
	}
	if chartInfoId <= 0 {
		edbInfoMappingA.IsAxis = 1
		edbInfoMappingA.LeadValue = 0
		edbInfoMappingA.LeadUnit = ""
		edbInfoMappingA.ChartEdbMappingId = 0
		edbInfoMappingA.ChartInfoId = 0
		edbInfoMappingA.IsOrder = false
		edbInfoMappingA.EdbInfoType = 1
		edbInfoMappingA.ChartStyle = ""
		edbInfoMappingA.ChartColor = ""
		edbInfoMappingA.ChartWidth = 0

		edbInfoMappingB.IsAxis = 1
		edbInfoMappingB.LeadValue = 0
		edbInfoMappingB.LeadUnit = ""
		edbInfoMappingB.ChartEdbMappingId = 0
		edbInfoMappingB.ChartInfoId = 0
		edbInfoMappingB.IsOrder = false
		edbInfoMappingB.EdbInfoType = 1
		edbInfoMappingB.ChartStyle = ""
		edbInfoMappingB.ChartColor = ""
		edbInfoMappingB.ChartWidth = 0
	} else {
		edbInfoMappingA.LeadUnitEn = chart.GetLeadUnitEn(edbInfoMappingA.LeadUnit)
		edbInfoMappingB.LeadUnitEn = chart.GetLeadUnitEn(edbInfoMappingB.LeadUnit)
	}
	aList := new(chart_edb_mapping.ChartEdbInfoMappingList)
	aList.ChartEdbInfoMapping = *edbInfoMappingA
	bList := new(chart_edb_mapping.ChartEdbInfoMappingList)
	bList.ChartEdbInfoMapping = *edbInfoMappingB
	edbList = append(edbList, aList, bList)
	return
}

// GetChartDataByEdbInfo 相关性图表-根据指标信息获取x轴和y轴
func GetChartDataByEdbInfo(edbInfoMappingA, edbInfoMappingB *chart_edb_mapping.ChartEdbInfoMapping, leadValue int, leadUnit, startDate, endDate string) (xEdbIdValue []int, yDataList []chart_info.YData, err error) {
	xData := make([]int, 0)
	yData := make([]float64, 0)
	if leadValue == 0 {
		xData = append(xData, 0)
	}
	if leadValue > 0 {
		leadMin := 0 - leadValue
		xLen := 2*leadValue + 1
		for i := 0; i < xLen; i++ {
			n := leadMin + i
			xData = append(xData, n)
		}
	}

	// 计算窗口，不包含第一天
	startDateTime, _ := time.ParseInLocation(utils.FormatDate, startDate, time.Local)
	startDate = startDateTime.AddDate(0, 0, 1).Format(utils.FormatDate)

	//// 2023-03-02 时间序列始终以指标B为基准, 始终是A进行平移
	//baseEdbInfo := edbInfoMappingB
	//changeEdbInfo := edbInfoMappingA
	// 2023-03-17 时间序列始终以指标A为基准, 始终是B进行平移
	baseEdbInfo := edbInfoMappingA
	changeEdbInfo := edbInfoMappingB

	// 获取时间基准指标在时间区间内的值
	aDataList := make([]*edb_data.EdbDataList, 0)
	switch baseEdbInfo.EdbInfoCategoryType {
	case 0:
		aDataList, err = edb_data.GetEdbDataList(baseEdbInfo.Source, baseEdbInfo.EdbInfoId, startDate, endDate)
	case 1:
		_, aDataList, _, _, err, _ = chart.GetPredictDataListByPredictEdbInfoId(baseEdbInfo.EdbInfoId, startDate, endDate, false)
	default:
		err = errors.New("指标base类型异常")
		return
	}

	// 获取变频指标所有日期的值, 插值法完善数据
	bDataList := make([]*edb_data.EdbDataList, 0)
	switch changeEdbInfo.EdbInfoCategoryType {
	case 0:
		bDataList, err = edb_data.GetEdbDataList(changeEdbInfo.Source, changeEdbInfo.EdbInfoId, "", "")
	case 1:
		_, bDataList, _, _, err, _ = chart.GetPredictDataListByPredictEdbInfoId(changeEdbInfo.EdbInfoId, "", "", false)
	default:
		err = errors.New("指标change类型异常")
		return
	}
	//changeDataMap := make(map[string]float64)
	//newChangeDataList, e := HandleDataByLinearRegression(bDataList, changeDataMap)
	//if e != nil {
	//	err = fmt.Errorf("获取变频指标插值法Map失败, Err: %s", e.Error())
	//	return
	//}

	// 2023-03-17 时间序列始终以指标A为基准, 始终是B进行平移
	baseDataList := make([]*edb_data.EdbDataList, 0)
	baseDataMap := make(map[string]float64)
	changeDataList := make([]*edb_data.EdbDataList, 0)
	changeDataMap := make(map[string]float64)

	// 先把低频指标升频为高频
	{
		frequencyIntMap := map[string]int{
			"日度": 1,
			"周度": 2,
			"旬度": 3,
			"月度": 4,
			"季度": 5,
			"年度": 6,
		}

		// 如果A指标是高频，那么就需要对B指标进行升频
		if frequencyIntMap[edbInfoMappingA.Frequency] < frequencyIntMap[edbInfoMappingB.Frequency] {
			tmpNewChangeDataList, e := HandleDataByLinearRegression(aDataList, baseDataMap)
			if e != nil {
				err = fmt.Errorf("获取变频指标插值法Map失败, Err: %s", e.Error())
				return
			}
			baseDataList = tmpNewChangeDataList
		} else {
			baseDataList = aDataList
			for _, v := range baseDataList {
				baseDataMap[v.DataTime] = v.Value
			}
		}
		// 如果B指标是高频，那么就需要对A指标进行升频
		if frequencyIntMap[edbInfoMappingA.Frequency] > frequencyIntMap[edbInfoMappingB.Frequency] {
			tmpNewChangeDataList, e := HandleDataByLinearRegression(bDataList, changeDataMap)
			if e != nil {
				err = fmt.Errorf("获取变频指标插值法Map失败, Err: %s", e.Error())
				return
			}
			changeDataList = tmpNewChangeDataList
		} else {
			changeDataList = bDataList
			for _, v := range changeDataList {
				changeDataMap[v.DataTime] = v.Value
			}
		}

	}

	// 计算不领先也不滞后时的相关系数
	baseCalculateData := make([]float64, 0)
	baseDataTimeArr := make([]string, 0)
	for i := range baseDataList {
		baseDataTimeArr = append(baseDataTimeArr, baseDataList[i].DataTime)
		baseCalculateData = append(baseCalculateData, baseDataList[i].Value)
	}

	zeroBaseData := make([]float64, 0)
	zeroCalculateData := make([]float64, 0)
	for i := range baseDataTimeArr {
		tmpBaseVal, ok1 := baseDataMap[baseDataTimeArr[i]]
		tmpCalculateVal, ok2 := changeDataMap[baseDataTimeArr[i]]
		if ok1 && ok2 {
			zeroBaseData = append(zeroBaseData, tmpBaseVal)
			zeroCalculateData = append(zeroCalculateData, tmpCalculateVal)
		}
	}
	if len(zeroBaseData) != len(zeroCalculateData) {
		err = fmt.Errorf("相关系数两组序列元素数不一致, %d-%d", len(baseCalculateData), len(zeroCalculateData))
		return
	}
	zeroRatio := utils.CalculateCorrelationByIntArr(zeroBaseData, zeroCalculateData)
	if leadValue == 0 {
		yData = append(yData, zeroRatio)
	}

	// 计算领先/滞后N期
	if leadValue > 0 {
		// 平移变频指标领先/滞后的日期(单位天)
		moveUnitDays := utils.FrequencyDaysMap[leadUnit]

		for i := range xData {
			if xData[i] == 0 {
				yData = append(yData, zeroRatio)
				continue
			}
			xCalculateData := make([]float64, 0)
			yCalculateData := make([]float64, 0)

			// 平移指定天数
			mDays := int(moveUnitDays) * xData[i]
			_, dMap := MoveDataDaysToNewDataList(changeDataList, mDays)

			// 取出对应的基准日期的值
			for i2 := range baseDataTimeArr {
				if yVal, ok := dMap[baseDataTimeArr[i2]]; ok {
					xCalculateData = append(xCalculateData, baseCalculateData[i2])
					yCalculateData = append(yCalculateData, yVal)
				}
			}
			if len(yCalculateData) <= 0 {
				//err = fmt.Errorf("领先滞后相关系数两组序列元素数不一致, %d-%d", len(baseCalculateData), len(yCalculateData))
				//return
				// 领先滞后后，没有可以计算的数据了
				continue
			}

			// 公式计算出领先/滞后频度对应点的相关性系数
			ratio := utils.CalculateCorrelationByIntArr(xCalculateData, yCalculateData)
			yData = append(yData, ratio)
		}
	}

	xEdbIdValue = xData
	yDataList = make([]chart_info.YData, 0)
	yDate := "0000-00-00"
	yDataList = append(yDataList, chart_info.YData{
		Date:  yDate,
		Value: yData,
	})
	return
}

// GetRollingCorrelationChartDataByEdbInfo 滚动相关性计算
func GetRollingCorrelationChartDataByEdbInfo(edbInfoMappingA, edbInfoMappingB *chart_edb_mapping.ChartEdbInfoMapping, leadValue int, leadUnit string, calculateValue int, calculateUnit string, startDate, endDate string) (xDateTimeValue []string, yDataList []chart_info.YData, err error) {
	xDateTimeValue = make([]string, 0)
	yData := make([]float64, 0)

	// 计算窗口，不包含第一天
	startDateTime, _ := time.ParseInLocation(utils.FormatDate, startDate, time.Local)
	startDate = startDateTime.AddDate(0, 0, 1).Format(utils.FormatDate)

	baseEdbInfo := edbInfoMappingA
	changeEdbInfo := edbInfoMappingB

	// 获取时间基准指标在时间区间内的值
	aDataList := make([]*edb_data.EdbDataList, 0)
	switch baseEdbInfo.EdbInfoCategoryType {
	case 0:
		aDataList, err = edb_data.GetEdbDataList(baseEdbInfo.Source, baseEdbInfo.EdbInfoId, startDate, endDate)
	case 1:
		_, aDataList, _, _, err, _ = chart.GetPredictDataListByPredictEdbInfoId(baseEdbInfo.EdbInfoId, startDate, endDate, false)
	default:
		err = errors.New("指标base类型异常")
		return
	}

	// 获取变频指标所有日期的值, 插值法完善数据
	bDataList := make([]*edb_data.EdbDataList, 0)
	switch changeEdbInfo.EdbInfoCategoryType {
	case 0:
		bDataList, err = edb_data.GetEdbDataList(changeEdbInfo.Source, changeEdbInfo.EdbInfoId, "", "")
	case 1:
		_, bDataList, _, _, err, _ = chart.GetPredictDataListByPredictEdbInfoId(changeEdbInfo.EdbInfoId, "", "", false)
	default:
		err = errors.New("指标change类型异常")
		return
	}

	// 数据平移变频指标领先/滞后的日期(单位天)
	// 2023-03-17 时间序列始终以指标A为基准, 始终是B进行平移
	//baseDataList := make([]*edb_data.EdbDataList, 0)
	baseDataMap := make(map[string]float64)
	changeDataList := make([]*edb_data.EdbDataList, 0)
	changeDataMap := make(map[string]float64)

	// A指标不管三七二十一，先变个频再说
	{
		_, e := HandleDataByLinearRegression(aDataList, baseDataMap)
		if e != nil {
			err = fmt.Errorf("获取变频指标插值法Map失败, Err: %s", e.Error())
			return
		}
		//baseDataList = tmpNewChangeDataList
	}
	// B指标不管三七二十一，先变个频再说
	{
		tmpNewChangeDataList, e := HandleDataByLinearRegression(bDataList, changeDataMap)
		if e != nil {
			err = fmt.Errorf("获取变频指标插值法Map失败, Err: %s", e.Error())
			return
		}
		changeDataList = tmpNewChangeDataList

		// 平移下日期
		moveUnitDays := utils.FrequencyDaysMap[leadUnit]
		_, changeDataMap = MoveDataDaysToNewDataList(changeDataList, leadValue*moveUnitDays)
	}

	// 计算计算时，需要多少个日期内数据
	calculateDay := utils.FrequencyDaysMap[calculateUnit] * calculateValue

	// 计算 每个日期的相关性值
	{
		startDateTime, _ := time.ParseInLocation(utils.FormatDate, startDate, time.Local)
		endDateTime, _ := time.ParseInLocation(utils.FormatDate, baseEdbInfo.EndDate, time.Local)
		endDateTime = endDateTime.AddDate(0, 0, -(calculateDay - 1))

		for currDay := startDateTime; !currDay.After(endDateTime); currDay = currDay.AddDate(0, 0, 1) {
			yCalculateData := make([]float64, 0)
			baseCalculateData := make([]float64, 0)

			// 取出对应的基准日期的值
			for i := 0; i < calculateDay; i++ {
				iDay := currDay.AddDate(0, 0, i).Format(utils.FormatDate)

				tmpBaseValue, ok1 := baseDataMap[iDay]
				tmpChangeValue, ok2 := changeDataMap[iDay]
				if ok1 && ok2 {
					baseCalculateData = append(baseCalculateData, tmpBaseValue)
					yCalculateData = append(yCalculateData, tmpChangeValue)
				} else {
					continue
				}
			}

			// 公式计算出领先/滞后频度对应点的相关性系数
			var ratio float64
			if len(baseCalculateData) > 0 {
				ratio = utils.CalculateCorrelationByIntArr(baseCalculateData, yCalculateData)
			}
			yData = append(yData, ratio)
			xDateTimeValue = append(xDateTimeValue, currDay.AddDate(0, 0, calculateDay-1).Format(utils.FormatDate))
		}
	}

	yDataList = make([]chart_info.YData, 0)
	yDate := "0000-00-00"
	yDataList = append(yDataList, chart_info.YData{
		Date:  yDate,
		Value: yData,
	})
	return
}

// ChartInfoRefresh 图表刷新
func ChartInfoRefresh(chartInfoId int) (err error) {
	var errMsg string
	defer func() {
		if err != nil {
			//fmt.Println(err.Error())
			go alarm_msg.SendAlarmMsg("CorrelationChartInfoRefresh: "+errMsg, 3)
		}
	}()
	correlationChart := new(chart_info_correlation.ChartInfoCorrelation)
	if err = correlationChart.GetItemById(chartInfoId); err != nil {
		errMsg = "获取相关性图表失败, Err: " + err.Error()
		return
	}

	// 批量刷新ETA指标
	err, errMsg = chart.EdbInfoRefreshAllFromBase([]int{correlationChart.EdbInfoIdFirst, correlationChart.EdbInfoIdSecond}, false)
	if err != nil {
		return
	}

	// 重新生成数据并更新
	edbInfoMappingA, err := chart_edb_mapping.GetChartEdbMappingByEdbInfoId(correlationChart.EdbInfoIdFirst)
	if err != nil {
		errMsg = "获取相关性图表, A指标mapping信息失败, Err:" + err.Error()
		return
	}
	edbInfoMappingB, err := chart_edb_mapping.GetChartEdbMappingByEdbInfoId(correlationChart.EdbInfoIdSecond)
	if err != nil {
		errMsg = "获取相关性图表, B指标mapping信息失败, Err:" + err.Error()
		return
	}
	periodData, correlationData, err := GetChartDataByEdbInfo(edbInfoMappingA, edbInfoMappingB, correlationChart.LeadValue, correlationChart.LeadUnit, correlationChart.StartDate.Format(utils.FormatDate), correlationChart.EndDate.Format(utils.FormatDate))
	if err != nil {
		errMsg = "获取相关性图表, 图表计算值失败, Err:" + err.Error()
		return
	}
	periodDataByte, err := json.Marshal(periodData)
	if err != nil {
		errMsg = "相关性图表, X轴信息有误, Err:" + err.Error()
		return
	}
	correlationDataByte, err := json.Marshal(correlationData[0].Value)
	if err != nil {
		errMsg = "相关性图表, Y轴信息有误, Err:" + err.Error()
		return
	}
	correlationChart.PeriodData = string(periodDataByte)
	correlationChart.CorrelationData = string(correlationDataByte)
	correlationChart.ModifyTime = time.Now().Local()
	correlationUpdateCols := []string{"PeriodData", "CorrelationData", "ModifyTime"}
	if err = correlationChart.Update(correlationUpdateCols); err != nil {
		errMsg = "更新相关性图表失败, Err:" + err.Error()
		return
	}
	return
}