package correlation import ( "encoding/json" "errors" "eta/eta_api/models/data_manage" "eta/eta_api/models/system" "eta/eta_api/services/alarm_msg" "eta/eta_api/services/data" "eta/eta_api/utils" "fmt" "github.com/shopspring/decimal" "math" "sort" "strconv" "strings" "sync" "time" ) // HandleDataByLinearRegression 线性方程插值法补全数据 func HandleDataByLinearRegression(originList []*data_manage.EdbDataList, handleDataMap map[string]float64) (newList []*data_manage.EdbDataList, err error) { if len(originList) < 2 { return } var startEdbInfoData *data_manage.EdbDataList for _, v := range originList { handleDataMap[v.DataTime] = v.Value // 第一个数据就给过滤了,给后面的试用 if startEdbInfoData == nil { startEdbInfoData = v newList = append(newList, &data_manage.EdbDataList{ DataTime: v.DataTime, Value: v.Value, }) continue } // 获取两条数据之间相差的天数 startDataTime, _ := time.ParseInLocation(utils.FormatDate, startEdbInfoData.DataTime, time.Local) currDataTime, _ := time.ParseInLocation(utils.FormatDate, v.DataTime, time.Local) betweenHour := int(currDataTime.Sub(startDataTime).Hours()) betweenDay := betweenHour / 24 // 如果相差一天,那么过滤 if betweenDay <= 1 { startEdbInfoData = v newList = append(newList, &data_manage.EdbDataList{ DataTime: v.DataTime, Value: v.Value, }) continue } // 生成线性方程式 var a, b float64 { coordinateData := make([]utils.Coordinate, 0) tmpCoordinate1 := utils.Coordinate{ X: 1, Y: startEdbInfoData.Value, } coordinateData = append(coordinateData, tmpCoordinate1) tmpCoordinate2 := utils.Coordinate{ X: float64(betweenDay) + 1, Y: v.Value, } coordinateData = append(coordinateData, tmpCoordinate2) a, b = utils.GetLinearResult(coordinateData) if math.IsNaN(a) || math.IsNaN(b) { err = fmt.Errorf("线性方程公式生成失败") return } } // 生成对应的值 { for i := 1; i < betweenDay; i++ { tmpDataTime := startDataTime.AddDate(0, 0, i) aDecimal := decimal.NewFromFloat(a) xDecimal := decimal.NewFromInt(int64(i) + 1) bDecimal := decimal.NewFromFloat(b) val, _ := aDecimal.Mul(xDecimal).Add(bDecimal).Round(4).Float64() handleDataMap[tmpDataTime.Format(utils.FormatDate)] = val newList = append(newList, &data_manage.EdbDataList{ DataTime: tmpDataTime.Format(utils.FormatDate), Value: val, }) } } // 最后将自己赋值 newList = append(newList, &data_manage.EdbDataList{ EdbDataId: v.EdbDataId, DataTime: v.DataTime, Value: v.Value, }) startEdbInfoData = v } return } // MoveDataDaysToNewDataList 平移指标数据生成新的数据序列 func MoveDataDaysToNewDataList(dataList []*data_manage.EdbDataList, moveDay int) (newDataList []data_manage.EdbDataList, dateDataMap map[string]float64) { dateMap := make(map[time.Time]float64) var minDate, maxDate time.Time dateDataMap = make(map[string]float64) for _, v := range dataList { currDate, _ := time.ParseInLocation(utils.FormatDate, v.DataTime, time.Local) if minDate.IsZero() || currDate.Before(minDate) { minDate = currDate } if maxDate.IsZero() || currDate.After(maxDate) { maxDate = currDate } dateMap[currDate] = v.Value } // 处理领先、滞后数据 newDateMap := make(map[time.Time]float64) for currDate, value := range dateMap { newDate := currDate.AddDate(0, 0, moveDay) newDateMap[newDate] = value } minDate = minDate.AddDate(0, 0, moveDay) maxDate = maxDate.AddDate(0, 0, moveDay) // 获取日期相差日 dayNum := utils.GetTimeSubDay(minDate, maxDate) for i := 0; i <= dayNum; i++ { currDate := minDate.AddDate(0, 0, i) tmpValue, ok := newDateMap[currDate] if !ok { //找不到数据,那么就用前面的数据吧 if len(newDataList)-1 < 0 { tmpValue = 0 } else { tmpValue = newDataList[len(newDataList)-1].Value } } tmpData := data_manage.EdbDataList{ DataTime: currDate.Format(utils.FormatDate), Value: tmpValue, } dateDataMap[tmpData.DataTime] = tmpData.Value newDataList = append(newDataList, tmpData) } return } // GetChartEdbInfoFormat 相关性图表-获取指标信息 func GetChartEdbInfoFormat(chartInfoId int, edbInfoMappingA, edbInfoMappingB *data_manage.ChartEdbInfoMapping) (edbList []*data_manage.ChartEdbInfoMapping, err error) { edbList = make([]*data_manage.ChartEdbInfoMapping, 0) if edbInfoMappingA == nil || edbInfoMappingB == nil { err = fmt.Errorf("指标信息有误") return } edbInfoMappingA.FrequencyEn = data.GetFrequencyEn(edbInfoMappingA.Frequency) if edbInfoMappingA.Unit == `无` { edbInfoMappingA.Unit = `` } if edbInfoMappingB.Unit == `无` { edbInfoMappingB.Unit = `` } if chartInfoId <= 0 { edbInfoMappingA.IsAxis = 1 edbInfoMappingA.LeadValue = 0 edbInfoMappingA.LeadUnit = "" edbInfoMappingA.ChartEdbMappingId = 0 edbInfoMappingA.ChartInfoId = 0 edbInfoMappingA.IsOrder = false edbInfoMappingA.EdbInfoType = 1 edbInfoMappingA.ChartStyle = "" edbInfoMappingA.ChartColor = "" edbInfoMappingA.ChartWidth = 0 edbInfoMappingB.IsAxis = 1 edbInfoMappingB.LeadValue = 0 edbInfoMappingB.LeadUnit = "" edbInfoMappingB.ChartEdbMappingId = 0 edbInfoMappingB.ChartInfoId = 0 edbInfoMappingB.IsOrder = false edbInfoMappingB.EdbInfoType = 1 edbInfoMappingB.ChartStyle = "" edbInfoMappingB.ChartColor = "" edbInfoMappingB.ChartWidth = 0 } else { edbInfoMappingA.LeadUnitEn = data.GetLeadUnitEn(edbInfoMappingA.LeadUnit) edbInfoMappingB.LeadUnitEn = data.GetLeadUnitEn(edbInfoMappingB.LeadUnit) } edbList = append(edbList, edbInfoMappingA, edbInfoMappingB) return } // GetChartDataByEdbInfo 相关性图表-根据指标信息获取x轴和y轴 func GetChartDataByEdbInfo(edbInfoMappingA, edbInfoMappingB *data_manage.ChartEdbInfoMapping, leadValue int, leadUnit, startDate, endDate, extraConfig string) (xEdbIdValue []int, yDataList []data_manage.YData, err error) { xData := make([]int, 0) yData := make([]float64, 0) if leadValue == 0 { xData = append(xData, 0) } if leadValue > 0 { leadMin := 0 - leadValue xLen := 2*leadValue + 1 for i := 0; i < xLen; i++ { n := leadMin + i xData = append(xData, n) } } // 计算窗口,不包含第一天 startDateTime, _ := time.ParseInLocation(utils.FormatDate, startDate, time.Local) startDate = startDateTime.AddDate(0, 0, 1).Format(utils.FormatDate) //// 2023-03-02 时间序列始终以指标B为基准, 始终是A进行平移 //baseEdbInfo := edbInfoMappingB //changeEdbInfo := edbInfoMappingA // 2023-03-17 时间序列始终以指标A为基准, 始终是B进行平移 baseEdbInfo := edbInfoMappingA changeEdbInfo := edbInfoMappingB // 获取时间基准指标在时间区间内的值 aDataList := make([]*data_manage.EdbDataList, 0) switch baseEdbInfo.EdbInfoCategoryType { case 0: aDataList, err = data_manage.GetEdbDataList(baseEdbInfo.Source, baseEdbInfo.SubSource, baseEdbInfo.EdbInfoId, startDate, endDate) case 1: _, aDataList, _, _, err, _ = data.GetPredictDataListByPredictEdbInfoId(baseEdbInfo.EdbInfoId, startDate, endDate, false) default: err = errors.New("指标base类型异常") return } // 获取变频指标所有日期的值, 插值法完善数据 bDataList := make([]*data_manage.EdbDataList, 0) switch changeEdbInfo.EdbInfoCategoryType { case 0: bDataList, err = data_manage.GetEdbDataList(changeEdbInfo.Source, changeEdbInfo.SubSource, changeEdbInfo.EdbInfoId, "", "") case 1: _, bDataList, _, _, err, _ = data.GetPredictDataListByPredictEdbInfoId(changeEdbInfo.EdbInfoId, "", "", false) default: err = errors.New("指标change类型异常") return } //changeDataMap := make(map[string]float64) //newChangeDataList, e := HandleDataByLinearRegression(bDataList, changeDataMap) //if e != nil { // err = fmt.Errorf("获取变频指标插值法Map失败, Err: %s", e.Error()) // return //} // 2023-03-17 时间序列始终以指标A为基准, 始终是B进行平移 baseDataList := make([]*data_manage.EdbDataList, 0) baseDataMap := make(map[string]float64) changeDataList := make([]*data_manage.EdbDataList, 0) changeDataMap := make(map[string]float64) // 先把低频指标升频为高频 { frequencyIntMap := map[string]int{ "日度": 1, "周度": 2, "旬度": 3, "月度": 4, "季度": 5, "年度": 6, } // 如果A指标是高频,那么就需要对B指标进行升频 if frequencyIntMap[edbInfoMappingA.Frequency] < frequencyIntMap[edbInfoMappingB.Frequency] { tmpNewChangeDataList, e := HandleDataByLinearRegression(bDataList, changeDataMap) if e != nil { err = fmt.Errorf("获取变频指标插值法Map失败, Err: %s", e.Error()) return } changeDataList = tmpNewChangeDataList baseDataList = aDataList for _, v := range baseDataList { baseDataMap[v.DataTime] = v.Value } } else if frequencyIntMap[edbInfoMappingA.Frequency] > frequencyIntMap[edbInfoMappingB.Frequency] { // 如果B指标是高频,那么就需要对A指标进行升频 tmpNewChangeDataList, e := HandleDataByLinearRegression(aDataList, baseDataMap) if e != nil { err = fmt.Errorf("获取变频指标插值法Map失败, Err: %s", e.Error()) return } baseDataList = tmpNewChangeDataList changeDataList = bDataList for _, v := range changeDataList { changeDataMap[v.DataTime] = v.Value } } else { baseDataList = aDataList for _, v := range baseDataList { baseDataMap[v.DataTime] = v.Value } changeDataList = bDataList for _, v := range changeDataList { changeDataMap[v.DataTime] = v.Value } } } // 计算不领先也不滞后时的相关系数 baseCalculateData := make([]float64, 0) baseDataTimeArr := make([]string, 0) for i := range baseDataList { baseDataTimeArr = append(baseDataTimeArr, baseDataList[i].DataTime) baseCalculateData = append(baseCalculateData, baseDataList[i].Value) } //zeroBaseData := make([]float64, 0) //zeroCalculateData := make([]float64, 0) //for i := range baseDataTimeArr { // tmpBaseVal, ok1 := baseDataMap[baseDataTimeArr[i]] // tmpCalculateVal, ok2 := changeDataMap[baseDataTimeArr[i]] // if ok1 && ok2 { // zeroBaseData = append(zeroBaseData, tmpBaseVal) // zeroCalculateData = append(zeroCalculateData, tmpCalculateVal) // } //} //if len(zeroBaseData) != len(zeroCalculateData) { // err = fmt.Errorf("相关系数两组序列元素数不一致, %d-%d", len(baseCalculateData), len(zeroCalculateData)) // return //} //zeroRatio := utils.CalculateCorrelationByIntArr(zeroBaseData, zeroCalculateData) //if leadValue == 0 { // yData = append(yData, zeroRatio) //} // 计算领先/滞后N期 if leadValue > 0 { // 平移变频指标领先/滞后的日期(单位天) moveUnitDays := utils.FrequencyDaysMap[leadUnit] for i := range xData { //if xData[i] == 0 { // yData = append(yData, zeroRatio) // continue //} xCalculateData := make([]float64, 0) yCalculateData := make([]float64, 0) // 平移指定天数 mDays := int(moveUnitDays) * xData[i] _, dMap := MoveDataDaysToNewDataList(changeDataList, mDays) // 取出对应的基准日期的值 for i2 := range baseDataTimeArr { tmpDate := baseDataTimeArr[i2] if yVal, ok := dMap[tmpDate]; ok { xCalculateData = append(xCalculateData, baseCalculateData[i2]) yCalculateData = append(yCalculateData, yVal) } } if len(yCalculateData) <= 0 { //err = fmt.Errorf("领先滞后相关系数两组序列元素数不一致, %d-%d", len(baseCalculateData), len(yCalculateData)) //return // 领先滞后后,没有可以计算的数据了 continue } // 公式计算出领先/滞后频度对应点的相关性系数 ratio := utils.CalculateCorrelationByIntArr(xCalculateData, yCalculateData) yData = append(yData, ratio) } } // 图例 var extra data_manage.CorrelationChartInfoExtraConfig legend := new(data_manage.CorrelationChartLegend) if extraConfig != "" { if e := json.Unmarshal([]byte(extraConfig), &extra); e != nil { err = fmt.Errorf("图例解析异常, err: %v", e) return } if len(extra.LegendConfig) > 0 { legend = extra.LegendConfig[0] } } xEdbIdValue = xData yDataList = make([]data_manage.YData, 0) yDate := "0000-00-00" var y data_manage.YData y.Date = yDate y.Value = yData if legend != nil { y.Name = legend.LegendName y.Color = legend.Color } yDataList = append(yDataList, y) return } // RollingCorrelationChartDataResp 滚动相关性图表数据 type RollingCorrelationChartDataResp struct { MaxData float64 MinData float64 LatestDate string `description:"真实数据的最后日期"` EdbInfoCategoryType int ChartColor string ChartStyle string PredictChartColor string ChartType int ChartWidth int EdbName string EdbNameEn string Unit string UnitEn string IsAxis int DataList []data_manage.EdbDataList } // GetRollingCorrelationChartDataByEdbInfo 滚动相关性计算 func GetRollingCorrelationChartDataByEdbInfo(edbInfoMappingA, edbInfoMappingB *data_manage.ChartEdbInfoMapping, leadValue int, leadUnit string, calculateValue int, calculateUnit string, startDate, endDate, chartName, chartNameEn string) (dataResp RollingCorrelationChartDataResp, err error) { dataResp = RollingCorrelationChartDataResp{ DataList: make([]data_manage.EdbDataList, 0), MaxData: 0, MinData: 0, ChartColor: "#00f", ChartStyle: `spline`, PredictChartColor: `#00f`, ChartType: 0, ChartWidth: 3, EdbName: chartName, EdbNameEn: chartNameEn, IsAxis: 1, } dataList := make([]data_manage.EdbDataList, 0) // 计算窗口,不包含第一天 startDateTime, _ := time.ParseInLocation(utils.FormatDate, startDate, time.Local) startDate = startDateTime.AddDate(0, 0, 1).Format(utils.FormatDate) baseEdbInfo := edbInfoMappingA changeEdbInfo := edbInfoMappingB // 获取时间基准指标在时间区间内的值 aDataList := make([]*data_manage.EdbDataList, 0) switch baseEdbInfo.EdbInfoCategoryType { case 0: aDataList, err = data_manage.GetEdbDataList(baseEdbInfo.Source, baseEdbInfo.SubSource, baseEdbInfo.EdbInfoId, startDate, endDate) case 1: _, aDataList, _, _, err, _ = data.GetPredictDataListByPredictEdbInfoId(baseEdbInfo.EdbInfoId, startDate, endDate, true) default: err = errors.New("指标base类型异常") return } // 获取变频指标所有日期的值, 插值法完善数据 bDataList := make([]*data_manage.EdbDataList, 0) switch changeEdbInfo.EdbInfoCategoryType { case 0: bDataList, err = data_manage.GetEdbDataList(changeEdbInfo.Source, changeEdbInfo.SubSource, changeEdbInfo.EdbInfoId, "", "") case 1: _, bDataList, _, _, err, _ = data.GetPredictDataListByPredictEdbInfoId(changeEdbInfo.EdbInfoId, "", "", false) default: err = errors.New("指标change类型异常") return } // 数据平移变频指标领先/滞后的日期(单位天) // 2023-03-17 时间序列始终以指标A为基准, 始终是B进行平移 //baseDataList := make([]*data_manage.EdbDataList, 0) baseDataMap := make(map[string]float64) changeDataList := make([]*data_manage.EdbDataList, 0) changeDataMap := make(map[string]float64) // A指标不管三七二十一,先变个频再说 { _, e := HandleDataByLinearRegression(aDataList, baseDataMap) if e != nil { err = fmt.Errorf("获取变频指标插值法Map失败, Err: %s", e.Error()) return } //baseDataList = tmpNewChangeDataList } // B指标不管三七二十一,先变个频再说 { tmpNewChangeDataList, e := HandleDataByLinearRegression(bDataList, changeDataMap) if e != nil { err = fmt.Errorf("获取变频指标插值法Map失败, Err: %s", e.Error()) return } changeDataList = tmpNewChangeDataList // 平移下日期 moveUnitDays := utils.FrequencyDaysMap[leadUnit] _, changeDataMap = MoveDataDaysToNewDataList(changeDataList, leadValue*moveUnitDays) } // 计算计算时,需要多少个日期内数据 calculateDay := utils.FrequencyDaysMap[calculateUnit] * calculateValue // 计算 每个日期的相关性值 { startDateTime, _ := time.ParseInLocation(utils.FormatDate, startDate, time.Local) if endDate == `` { endDate = baseEdbInfo.EndDate } endDateTime, _ := time.ParseInLocation(utils.FormatDate, endDate, time.Local) endDateTime = endDateTime.AddDate(0, 0, -(calculateDay - 1)) // 是否开始第一条数据 var isStart, isNotFirst bool for currDay := startDateTime; !currDay.After(endDateTime); currDay = currDay.AddDate(0, 0, 1) { yCalculateData := make([]float64, 0) baseCalculateData := make([]float64, 0) // 取出对应的基准日期的值 for i := 0; i < calculateDay; i++ { iDay := currDay.AddDate(0, 0, i).Format(utils.FormatDate) tmpBaseValue, ok1 := baseDataMap[iDay] tmpChangeValue, ok2 := changeDataMap[iDay] if ok1 && ok2 { baseCalculateData = append(baseCalculateData, tmpBaseValue) yCalculateData = append(yCalculateData, tmpChangeValue) } else { continue } } // 公式计算出领先/滞后频度对应点的相关性系数 var ratio float64 if len(baseCalculateData) > 0 { ratio = utils.CalculateCorrelationByIntArr(baseCalculateData, yCalculateData) } else { // 没有数据的话,那就不返回 continue } // 过滤前面都是0的数据 { if ratio != 0 { isStart = true } if !isStart { continue } } dataTime := currDay.AddDate(0, 0, calculateDay-1) dataList = append(dataList, data_manage.EdbDataList{ //EdbDataId: 0, EdbInfoId: 0, DataTime: dataTime.Format(utils.FormatDate), DataTimestamp: dataTime.UnixNano() / 1e6, Value: ratio, }) if !isNotFirst { dataResp.MinData = ratio dataResp.MaxData = ratio isNotFirst = true } if dataResp.MinData > ratio { dataResp.MinData = ratio } if dataResp.MaxData < ratio { dataResp.MaxData = ratio } } dataResp.DataList = dataList } return } // ChartInfoRefresh 图表刷新 func ChartInfoRefresh(chartInfoId int, uniqueCode string) (isAsync bool, err error) { var errMsg string defer func() { if err != nil { tips := fmt.Sprintf("CorrelationChartInfoRefresh: %s", errMsg) utils.FileLog.Info(tips) go alarm_msg.SendAlarmMsg(tips, 3) } }() correlationChart := new(data_manage.ChartInfoCorrelation) if err = correlationChart.GetItemById(chartInfoId); err != nil { errMsg = "获取相关性图表失败, Err: " + err.Error() return } // 多因子刷新-异步 if correlationChart.AnalysisMode == 1 { isAsync = true go func() { // 1.刷新图表关联的指标 mappings, e := data_manage.GetChartEdbMappingList(chartInfoId) if e != nil { utils.FileLog.Info(fmt.Sprintf("获取图表关联指标失败, err: %v", e)) return } if len(mappings) == 0 { utils.FileLog.Info("图表无关联指标") return } var edbIds []int for _, v := range mappings { edbIds = append(edbIds, v.EdbInfoId) } if e, _ = data.EdbInfoRefreshAllFromBaseV3(edbIds, false, true, false); e != nil { utils.FileLog.Info(fmt.Sprintf("批量刷新指标失败, err: %v", e)) return } // 2.刷新指标系列计算数据 for _, v := range mappings { _, e = data.PostRefreshFactorEdbRecalculate(v.EdbInfoId, v.EdbCode) if e != nil { utils.FileLog.Info(fmt.Sprintf("PostRefreshFactorEdbRecalculate err: %v", e)) continue } } // 3.刷新图表矩阵 _, e = data.PostRefreshFactorEdbChartRecalculate(chartInfoId) if e != nil { utils.FileLog.Info(fmt.Sprintf("PostRefreshFactorEdbRecalculate err: %v", e)) return } // 4.清除图表缓存 key := utils.HZ_CHART_LIB_DETAIL + uniqueCode _ = utils.Rc.Delete(key) }() return } // 批量刷新ETA指标 err, _ = data.EdbInfoRefreshAllFromBaseV3([]int{correlationChart.EdbInfoIdFirst, correlationChart.EdbInfoIdSecond}, false, true, false) if err != nil { return } // 重新生成数据并更新 edbInfoMappingA, err := data_manage.GetChartEdbMappingByEdbInfoId(correlationChart.EdbInfoIdFirst) if err != nil { errMsg = "获取相关性图表, A指标mapping信息失败, Err:" + err.Error() return } edbInfoMappingB, err := data_manage.GetChartEdbMappingByEdbInfoId(correlationChart.EdbInfoIdSecond) if err != nil { errMsg = "获取相关性图表, B指标mapping信息失败, Err:" + err.Error() return } periodData, correlationData, err := GetChartDataByEdbInfo(edbInfoMappingA, edbInfoMappingB, correlationChart.LeadValue, correlationChart.LeadUnit, correlationChart.StartDate.Format(utils.FormatDate), correlationChart.EndDate.Format(utils.FormatDate), "") if err != nil { errMsg = "获取相关性图表, 图表计算值失败, Err:" + err.Error() return } periodDataByte, err := json.Marshal(periodData) if err != nil { errMsg = "相关性图表, X轴信息有误, Err:" + err.Error() return } correlationDataByte, err := json.Marshal(correlationData[0].Value) if err != nil { errMsg = "相关性图表, Y轴信息有误, Err:" + err.Error() return } correlationChart.PeriodData = string(periodDataByte) correlationChart.CorrelationData = string(correlationDataByte) correlationChart.ModifyTime = time.Now().Local() correlationUpdateCols := []string{"PeriodData", "CorrelationData", "ModifyTime"} if err = correlationChart.Update(correlationUpdateCols); err != nil { errMsg = "更新相关性图表失败, Err:" + err.Error() return } return } // GetChartAndCorrelationInfo 获取图表信息和相关信息信息 func GetChartAndCorrelationInfo(chartInfoId int) (chartInfo *data_manage.ChartInfo, correlationInfo *data_manage.ChartInfoCorrelation, tips string, err error) { item, e := data_manage.GetChartInfoById(chartInfoId) if e != nil { if e.Error() == utils.ErrNoRow() { tips = "图表已被删除, 请刷新页面" err = fmt.Errorf("图表已被删除, 请刷新页面") return } err = fmt.Errorf("获取图表信息失败, Err: %s", e.Error()) return } if item.Source != utils.CHART_SOURCE_CORRELATION { tips = "该图不是相关性图表" err = fmt.Errorf("该图不是相关性图表") return } chartInfo = item correlationInfo = new(data_manage.ChartInfoCorrelation) if e = correlationInfo.GetItemById(chartInfo.ChartInfoId); e != nil { err = fmt.Errorf("获取图表相关性信息失败, Err: %s", e.Error()) return } return } // AddChartInfo 添加图表 func AddChartInfo(req data_manage.AddChartInfoReq, source int, sysUser *system.Admin, lang string) (chartInfo *data_manage.ChartInfo, err error, errMsg string, isSendEmail bool) { isSendEmail = true req.ChartName = strings.Trim(req.ChartName, " ") if req.ChartName == "" { errMsg = "请填写图表名称!" err = errors.New(errMsg) isSendEmail = false return } if req.ChartClassifyId <= 0 { errMsg = "分类参数错误!" err = errors.New(errMsg) isSendEmail = false return } // 相关性图表配置 if req.CorrelationChartInfo.LeadValue == 0 && source == utils.CHART_SOURCE_CORRELATION { errMsg = "请输入领先期数" err = errors.New(errMsg) isSendEmail = false return } if req.CorrelationChartInfo.LeadUnit == "" { errMsg = "请填写领先单位" err = errors.New(errMsg) isSendEmail = false return } //if req.CorrelationChartInfo.StartDate == "" || req.CorrelationChartInfo.EndDate == "" { // errMsg = "请填写开始结束日期" // err = errors.New(errMsg) // isSendEmail = false // return //} //startDate, e := time.Parse(utils.FormatDate, req.CorrelationChartInfo.StartDate) //if e != nil { // errMsg = "开始日期格式有误" // err = errors.New(errMsg) // isSendEmail = false // return //} //endDate, e := time.Parse(utils.FormatDate, req.CorrelationChartInfo.EndDate) //if e != nil { // errMsg = "结束日期格式有误" // err = errors.New(errMsg) // isSendEmail = false // return //} if len(req.CorrelationChartInfo.EdbInfoIdList) != 2 { errMsg = "请选择AB指标" err = errors.New(errMsg) isSendEmail = false return } chartClassify, err := data_manage.GetChartClassifyById(req.ChartClassifyId) if err != nil { if err.Error() == utils.ErrNoRow() { errMsg = "分类不存在" err = errors.New(errMsg) isSendEmail = false return } errMsg = "获取分类信息失败" err = errors.New("获取分类信息失败,Err:" + err.Error()) return } if chartClassify == nil { errMsg = "分类不存在" err = errors.New(errMsg) isSendEmail = false return } var edbInfoIdArr []int for _, v := range req.CorrelationChartInfo.EdbInfoIdList { edbInfoId := v.EdbInfoId edbInfo, tmpErr := data_manage.GetEdbInfoById(edbInfoId) if tmpErr != nil { if tmpErr.Error() == utils.ErrNoRow() { errMsg = "指标不存在!" err = errors.New("指标不存在,edbInfoId:" + strconv.Itoa(edbInfoId)) return } else { errMsg = "获取指标信息失败!" err = errors.New("获取图表的指标信息失败,Err:" + tmpErr.Error()) return } } if edbInfo == nil { errMsg = "指标已被删除,请重新选择!" err = errors.New("指标不存在,ChartInfoId:" + strconv.Itoa(edbInfoId)) return } else { if edbInfo.EdbInfoId <= 0 { errMsg = "指标已被删除,请重新选择!" err = errors.New("指标不存在,ChartInfoId:" + strconv.Itoa(edbInfoId)) return } } edbInfoIdArr = append(edbInfoIdArr, edbInfoId) edbInfo.EdbNameSource = edbInfo.EdbName } sort.Ints(edbInfoIdArr) var edbInfoIdArrStr []string for _, v := range edbInfoIdArr { edbInfoIdArrStr = append(edbInfoIdArrStr, strconv.Itoa(v)) } edbInfoIdStr := strings.Join(edbInfoIdArrStr, ",") var chartInfoId int // 判断图表是否存在 { var condition string var pars []interface{} switch lang { case utils.EnLangVersion: condition += " AND chart_name_en = ? AND source = ? " default: condition += " AND chart_name=? AND source = ? " } pars = append(pars, req.ChartName, source) count, tmpErr := data_manage.GetChartInfoCountByCondition(condition, pars) if tmpErr != nil { errMsg = "判断图表名称是否存在失败" err = errors.New("判断图表名称是否存在失败,Err:" + tmpErr.Error()) return } if count > 0 { errMsg = "图表已存在,请重新填写" err = errors.New(errMsg) isSendEmail = false return } } disableVal := data.CheckIsDisableChart(edbInfoIdArr) chartInfo = new(data_manage.ChartInfo) chartInfo.ChartName = req.ChartName chartInfo.ChartNameEn = req.ChartName chartInfo.EdbInfoIds = edbInfoIdStr chartInfo.ChartClassifyId = req.ChartClassifyId chartInfo.SysUserId = sysUser.AdminId chartInfo.SysUserRealName = sysUser.RealName chartInfo.CreateTime = time.Now() chartInfo.ModifyTime = time.Now() chartInfo.IsSetName = 0 timestamp := strconv.FormatInt(time.Now().UnixNano(), 10) chartInfo.UniqueCode = utils.MD5(utils.CHART_PREFIX + "_" + timestamp) chartInfo.ChartType = 9 // 相关性图 chartInfo.Calendar = "公历" chartInfo.DateType = 6 chartInfo.StartDate = req.StartDate chartInfo.EndDate = req.EndDate chartInfo.SeasonStartDate = req.StartDate chartInfo.SeasonEndDate = req.EndDate chartInfo.LeftMin = req.LeftMin chartInfo.LeftMax = req.LeftMax chartInfo.RightMin = req.RightMin chartInfo.RightMax = req.RightMax chartInfo.Disabled = disableVal chartInfo.Source = source chartInfo.ChartThemeId = req.ChartThemeId chartInfo.SourcesFrom = req.SourcesFrom chartInfo.Instructions = req.Instructions chartInfo.MarkersLines = req.MarkersLines chartInfo.MarkersAreas = req.MarkersAreas if req.ExtraConfig != "" { chartInfo.ExtraConfig = req.ExtraConfig } // 指标信息 mapList := make([]*data_manage.ChartEdbMapping, 0) for _, v := range req.CorrelationChartInfo.EdbInfoIdList { mapItem := new(data_manage.ChartEdbMapping) mapItem.EdbInfoId = v.EdbInfoId mapItem.CreateTime = time.Now() mapItem.ModifyTime = time.Now() edbTimestamp := strconv.FormatInt(time.Now().UnixNano(), 10) mapItem.UniqueCode = utils.MD5(utils.CHART_PREFIX + "_" + edbTimestamp + "_" + strconv.Itoa(v.EdbInfoId)) mapItem.IsOrder = true mapItem.IsAxis = 1 mapItem.EdbInfoType = 1 mapItem.Source = utils.CHART_SOURCE_CORRELATION mapList = append(mapList, mapItem) } // 相关性图表扩展信息 correlationChart := new(data_manage.ChartInfoCorrelation) correlationChart.LeadValue = req.CorrelationChartInfo.LeadValue correlationChart.LeadUnit = req.CorrelationChartInfo.LeadUnit correlationChart.CalculateValue = req.CorrelationChartInfo.CalculateValue correlationChart.CalculateUnit = req.CorrelationChartInfo.CalculateUnit correlationChart.BaseCalculateValue = req.CorrelationChartInfo.BaseCalculateValue correlationChart.BaseCalculateUnit = req.CorrelationChartInfo.BaseCalculateUnit // 滚动相关性会有日期等信息 if source == utils.CHART_SOURCE_ROLLING_CORRELATION { correlationChart.DateType = req.CorrelationChartInfo.DateType if req.CorrelationChartInfo.StartDate != `` { startDateTime, tmpErr := time.ParseInLocation(utils.FormatDate, req.CorrelationChartInfo.StartDate, time.Local) if tmpErr != nil { err = tmpErr return } correlationChart.StartDate = startDateTime } if req.CorrelationChartInfo.EndDate != `` { endDateTime, tmpErr := time.ParseInLocation(utils.FormatDate, req.CorrelationChartInfo.EndDate, time.Local) if tmpErr != nil { err = tmpErr return } correlationChart.EndDate = endDateTime } } correlationChart.EdbInfoIdFirst = req.CorrelationChartInfo.EdbInfoIdList[0].EdbInfoId correlationChart.EdbInfoIdSecond = req.CorrelationChartInfo.EdbInfoIdList[1].EdbInfoId correlationChart.CreateTime = time.Now().Local() correlationChart.ModifyTime = time.Now().Local() //// 生成图表x轴y轴数据 //edbInfoMappingA, e := data_manage.GetChartEdbMappingByEdbInfoId(req.CorrelationChartInfo.EdbInfoIdList[0].EdbInfoId) //if e != nil { // errMsg = "获取失败" // err = errors.New("获取相关性图表, A指标mapping信息失败, Err:" + e.Error()) // return //} //edbInfoMappingB, e := data_manage.GetChartEdbMappingByEdbInfoId(req.CorrelationChartInfo.EdbInfoIdList[1].EdbInfoId) //if e != nil { // errMsg = "获取失败" // err = errors.New("获取相关性图表, B指标mapping信息失败, Err:" + e.Error()) // return //} //periodData, correlationData, e := GetChartDataByEdbInfo(edbInfoMappingA, edbInfoMappingB, req.CorrelationChartInfo.LeadValue, req.CorrelationChartInfo.LeadUnit, req.CorrelationChartInfo.StartDate, req.CorrelationChartInfo.EndDate) //if e != nil { // errMsg = "获取失败" // err = errors.New("获取相关性图表, 图表计算值失败, Err:" + e.Error()) // return //} //periodDataByte, e := json.Marshal(periodData) //if e != nil { // errMsg = "获取失败" // err = errors.New("相关性图表, X轴信息有误, Err:" + e.Error()) // return //} //correlationDataByte, e := json.Marshal(correlationData[0].Value) //if e != nil { // errMsg = "获取失败" // err = errors.New("相关性图表, Y轴信息有误, Err:" + e.Error()) // return //} //correlationChart.PeriodData = string(periodDataByte) //correlationChart.CorrelationData = string(correlationDataByte) // 新增图表和指标mapping chartInfoId, e := data_manage.CreateCorrelationChartAndEdb(chartInfo, mapList, correlationChart) if e != nil { errMsg = "操作失败" err = errors.New("新增相关性图表失败, Err: " + e.Error()) return } // 添加指标引用记录 _ = data.SaveChartEdbInfoRelation(edbInfoIdArr, chartInfo) //添加es数据 go data.EsAddOrEditChartInfo(chartInfoId) return } // EditChartInfo 编辑图表 func EditChartInfo(req data_manage.EditChartInfoReq, sysUser *system.Admin, lang string) (chartItem *data_manage.ChartInfo, err error, errMsg string, isSendEmail bool) { isSendEmail = true chartItem, err = data_manage.GetChartInfoById(req.ChartInfoId) if err != nil { if err.Error() == utils.ErrNoRow() { errMsg = "图表已被删除,请刷新页面" err = errors.New(errMsg) isSendEmail = false return } errMsg = "获取图表信息失败" err = errors.New("获取图表信息失败,Err:" + err.Error()) return } if chartItem.Source != utils.CHART_SOURCE_CORRELATION && chartItem.Source != utils.CHART_SOURCE_ROLLING_CORRELATION { errMsg = "该图不是相关性图表!" err = errors.New(errMsg) isSendEmail = false return } req.ChartName = strings.Trim(req.ChartName, " ") if req.ChartClassifyId <= 0 { errMsg = "分类参数错误!" err = errors.New(errMsg) isSendEmail = false return } // 相关性图表配置 if req.CorrelationChartInfo.LeadValue == 0 && chartItem.Source == utils.CHART_SOURCE_CORRELATION { errMsg = "请输入领先期数" err = errors.New(errMsg) isSendEmail = false return } if req.CorrelationChartInfo.LeadUnit == "" { errMsg = "请填写领先单位" err = errors.New(errMsg) isSendEmail = false return } //if req.CorrelationChartInfo.StartDate == "" || req.CorrelationChartInfo.EndDate == "" { // errMsg = "请填写开始结束日期" // err = errors.New(errMsg) // isSendEmail = false // return //} startDate, e := time.Parse(utils.FormatDate, req.CorrelationChartInfo.StartDate) if e != nil { errMsg = "开始日期格式有误" err = errors.New(errMsg) isSendEmail = false return } var endDate time.Time if req.CorrelationChartInfo.EndDate != `` { endDate, e = time.Parse(utils.FormatDate, req.CorrelationChartInfo.EndDate) if e != nil { errMsg = "结束日期格式有误" err = errors.New(errMsg) isSendEmail = false return } } if len(req.CorrelationChartInfo.EdbInfoIdList) != 2 { errMsg = "请选择AB指标" err = errors.New(errMsg) isSendEmail = false return } chartClassify, err := data_manage.GetChartClassifyById(req.ChartClassifyId) if err != nil { if err.Error() == utils.ErrNoRow() { errMsg = "分类不存在" err = errors.New(errMsg) isSendEmail = false return } errMsg = "获取分类信息失败" err = errors.New("获取分类信息失败,Err:" + err.Error()) return } if chartClassify == nil { errMsg = "分类不存在" err = errors.New(errMsg) isSendEmail = false return } // 图表操作权限 ok := data.CheckOpChartPermission(sysUser, chartItem.SysUserId, true) if !ok { errMsg = "没有该图表的操作权限" err = errors.New(errMsg) isSendEmail = false return } var edbInfoIdArr []int for _, v := range req.CorrelationChartInfo.EdbInfoIdList { edbInfoId := v.EdbInfoId edbInfo, tmpErr := data_manage.GetEdbInfoById(edbInfoId) if tmpErr != nil { if tmpErr.Error() == utils.ErrNoRow() { errMsg = "图表不存在!" err = errors.New("图表指标不存在,ChartInfoId:" + strconv.Itoa(edbInfoId)) return } else { errMsg = "获取图表信息失败!" err = errors.New("获取图表的指标信息失败,Err:" + tmpErr.Error()) return } } if edbInfo == nil { errMsg = "指标不存在!" err = errors.New("指标不存在,ChartInfoId:" + strconv.Itoa(edbInfoId)) return } edbInfoIdArr = append(edbInfoIdArr, edbInfoId) } sort.Ints(edbInfoIdArr) var edbInfoIdArrStr []string for _, v := range edbInfoIdArr { edbInfoIdArrStr = append(edbInfoIdArrStr, strconv.Itoa(v)) } edbInfoIdStr := strings.Join(edbInfoIdArrStr, ",") //判断图表是否存在 { var condition string var pars []interface{} condition += " AND chart_info_id <> ? " pars = append(pars, req.ChartInfoId) switch lang { case utils.EnLangVersion: condition += " AND chart_name_en = ? AND source = ? " default: condition += " AND chart_name=? AND source = ? " } pars = append(pars, req.ChartName, chartItem.Source) count, tmpErr := data_manage.GetChartInfoCountByCondition(condition, pars) if tmpErr != nil { errMsg = "判断图表名称是否存在失败" err = errors.New("判断图表名称是否存在失败,Err:" + tmpErr.Error()) return } if count > 0 { errMsg = "图表已存在,请重新填写" err = errors.New(errMsg) isSendEmail = false return } } correlationChart := new(data_manage.ChartInfoCorrelation) if e := correlationChart.GetItemById(chartItem.ChartInfoId); e != nil { errMsg = "操作失败" err = errors.New("图表相关性信息不存在, Err: " + e.Error()) return } // 图表启用与否 disableVal := data.CheckIsDisableChart(edbInfoIdArr) // 重新生成图表值, 并修改相关性图表扩展信息 //edbInfoMappingA, e := data_manage.GetChartEdbMappingByEdbInfoId(req.CorrelationChartInfo.EdbInfoIdList[0].EdbInfoId) //if e != nil { // errMsg = "获取失败" // err = errors.New("获取相关性图表, A指标mapping信息失败, Err:" + e.Error()) // return //} //edbInfoMappingB, e := data_manage.GetChartEdbMappingByEdbInfoId(req.CorrelationChartInfo.EdbInfoIdList[1].EdbInfoId) //if e != nil { // errMsg = "获取失败" // err = errors.New("获取相关性图表, B指标mapping信息失败, Err:" + e.Error()) // return //} //periodData, correlationData, e := GetChartDataByEdbInfo(edbInfoMappingA, edbInfoMappingB, req.CorrelationChartInfo.LeadValue, req.CorrelationChartInfo.LeadUnit, req.CorrelationChartInfo.StartDate, req.CorrelationChartInfo.EndDate) //if e != nil { // errMsg = "获取失败" // err = errors.New("获取相关性图表, 图表计算值失败, Err:" + e.Error()) // return //} //periodDataByte, e := json.Marshal(periodData) //if e != nil { // errMsg = "获取失败" // err = errors.New("相关性图表, X轴信息有误, Err:" + e.Error()) // return //} //correlationDataByte, e := json.Marshal(correlationData[0].Value) //if e != nil { // errMsg = "获取失败" // err = errors.New("相关性图表, Y轴信息有误, Err:" + e.Error()) // return //} correlationChart.LeadValue = req.CorrelationChartInfo.LeadValue correlationChart.LeadUnit = req.CorrelationChartInfo.LeadUnit correlationChart.CalculateValue = req.CorrelationChartInfo.CalculateValue correlationChart.CalculateUnit = req.CorrelationChartInfo.CalculateUnit correlationChart.StartDate = startDate correlationChart.EndDate = endDate correlationChart.EdbInfoIdFirst = req.CorrelationChartInfo.EdbInfoIdList[0].EdbInfoId correlationChart.EdbInfoIdSecond = req.CorrelationChartInfo.EdbInfoIdList[1].EdbInfoId // 滚动相关性会有日期等信息 if chartItem.Source == utils.CHART_SOURCE_ROLLING_CORRELATION { correlationChart.DateType = req.CorrelationChartInfo.DateType } //correlationChart.PeriodData = string(periodDataByte) //correlationChart.CorrelationData = string(correlationDataByte) correlationChart.ModifyTime = time.Now().Local() //correlationUpdateCols := []string{"LeadValue", "LeadUnit", "StartDate", "EndDate", "EdbInfoIdFirst", "EdbInfoIdSecond", "PeriodData","CorrelationData", "ModifyTime"} correlationUpdateCols := []string{"LeadValue", "LeadUnit", "CalculateValue", "CalculateUnit", "DateType", "StartDate", "EndDate", "EdbInfoIdFirst", "EdbInfoIdSecond", "ModifyTime"} // 修改图表与指标mapping req.ChartType = 9 err = data_manage.EditCorrelationChartInfoAndMapping(&req, edbInfoIdStr, "公历", 6, disableVal, ``, correlationChart, correlationUpdateCols) if err != nil { errMsg = "保存失败" err = errors.New("保存失败,Err:" + err.Error()) return } resp := new(data_manage.AddChartInfoResp) resp.ChartInfoId = chartItem.ChartInfoId resp.UniqueCode = chartItem.UniqueCode resp.ChartType = req.ChartType // 添加指标引用记录 _ = data.SaveChartEdbInfoRelation(edbInfoIdArr, chartItem) //添加es数据 go data.EsAddOrEditChartInfo(chartItem.ChartInfoId) //修改my eta es数据 go data.EsAddOrEditMyChartInfoByChartInfoId(chartItem.ChartInfoId) return } // CopyChartInfo 复制图表 func CopyChartInfo(configId, classifyId int, chartName string, correlationChartInfoReq data_manage.CorrelationChartInfoReq, oldChartInfo *data_manage.ChartInfo, sysUser *system.Admin, lang string) (chartInfo *data_manage.ChartInfo, err error, errMsg string, isSendEmail bool) { configSource := 2 isSendEmail = true // 获取相关性图的配置 multipleGraphConfigChartMapping, err := data_manage.GetMultipleGraphConfigChartMappingByIdAndSource(configId, configSource) if err != nil { return } multipleGraphConfig, err := data_manage.GetMultipleGraphConfigById(configId) if err != nil { return } multipleGraphConfig.MultipleGraphConfigId = 0 err = data_manage.AddMultipleGraphConfig(multipleGraphConfig) if err != nil { return } // 添加图 addChartReq := data_manage.AddChartInfoReq{ ChartClassifyId: classifyId, ChartName: chartName, ChartType: utils.CHART_TYPE_CURVE, Calendar: "公历", CorrelationChartInfo: correlationChartInfoReq, ChartThemeId: oldChartInfo.ChartThemeId, SourcesFrom: oldChartInfo.SourcesFrom, Instructions: oldChartInfo.Instructions, MarkersLines: oldChartInfo.MarkersLines, MarkersAreas: oldChartInfo.MarkersAreas, } chartSource := utils.CHART_SOURCE_CORRELATION // 默认是相关性图 chartInfo, err, errMsg, isSendEmail = AddChartInfo(addChartReq, chartSource, sysUser, lang) if err != nil { return } // 添加关系 multipleGraphConfigChartMapping = &data_manage.MultipleGraphConfigChartMapping{ //Id: 0, MultipleGraphConfigId: multipleGraphConfig.MultipleGraphConfigId, ChartInfoId: chartInfo.ChartInfoId, Source: configSource, ModifyTime: time.Now(), CreateTime: time.Now(), } err = data_manage.AddMultipleGraphConfigChartMapping(multipleGraphConfigChartMapping) if err != nil { return } //添加es数据 go data.EsAddOrEditChartInfo(chartInfo.ChartInfoId) return } // CalculateCorrelation 计算相关性-获取x轴和y轴 func CalculateCorrelation(leadValue int, leadUnit, frequencyA, frequencyB string, dataListA, dataListB []*data_manage.EdbDataList) (xEdbIdValue []int, yDataList []data_manage.YData, err error) { xData := make([]int, 0) yData := make([]float64, 0) if leadValue == 0 { xData = append(xData, 0) } if leadValue > 0 { leadMin := 0 - leadValue xLen := 2*leadValue + 1 for i := 0; i < xLen; i++ { n := leadMin + i xData = append(xData, n) } } // 计算窗口,不包含第一天 //startDateTime, _ := time.ParseInLocation(utils.FormatDate, startDate, time.Local) //startDate = startDateTime.AddDate(0, 0, 1).Format(utils.FormatDate) //// 2023-03-02 时间序列始终以指标B为基准, 始终是A进行平移 //baseEdbInfo := edbInfoMappingB //changeEdbInfo := edbInfoMappingA // 2023-03-17 时间序列始终以指标A为基准, 始终是B进行平移 //baseEdbInfo := edbInfoMappingA //changeEdbInfo := edbInfoMappingB // 获取时间基准指标在时间区间内的值 //aDataList := make([]*data_manage.EdbDataList, 0) //switch baseEdbInfo.EdbInfoCategoryType { //case 0: // aDataList, err = data_manage.GetEdbDataList(baseEdbInfo.Source, baseEdbInfo.SubSource, baseEdbInfo.EdbInfoId, startDate, endDate) //case 1: // _, aDataList, _, _, err, _ = data.GetPredictDataListByPredictEdbInfoId(baseEdbInfo.EdbInfoId, startDate, endDate, false) //default: // err = errors.New("指标base类型异常") // return //} // //// 获取变频指标所有日期的值, 插值法完善数据 //bDataList := make([]*data_manage.EdbDataList, 0) //switch changeEdbInfo.EdbInfoCategoryType { //case 0: // bDataList, err = data_manage.GetEdbDataList(changeEdbInfo.Source, changeEdbInfo.SubSource, changeEdbInfo.EdbInfoId, "", "") //case 1: // _, bDataList, _, _, err, _ = data.GetPredictDataListByPredictEdbInfoId(changeEdbInfo.EdbInfoId, "", "", false) //default: // err = errors.New("指标change类型异常") // return //} //changeDataMap := make(map[string]float64) //newChangeDataList, e := HandleDataByLinearRegression(bDataList, changeDataMap) //if e != nil { // err = fmt.Errorf("获取变频指标插值法Map失败, Err: %s", e.Error()) // return //} // 2023-03-17 时间序列始终以指标A为基准, 始终是B进行平移 baseDataList := make([]*data_manage.EdbDataList, 0) baseDataMap := make(map[string]float64) changeDataList := make([]*data_manage.EdbDataList, 0) changeDataMap := make(map[string]float64) // 先把低频指标升频为高频 { frequencyIntMap := map[string]int{ "日度": 1, "周度": 2, "旬度": 3, "月度": 4, "季度": 5, "年度": 6, } // 如果A指标是高频,那么就需要对B指标进行升频 if frequencyIntMap[frequencyA] < frequencyIntMap[frequencyB] { tmpNewChangeDataList, e := HandleDataByLinearRegression(dataListB, changeDataMap) if e != nil { err = fmt.Errorf("获取变频指标插值法Map失败, Err: %s", e.Error()) return } changeDataList = tmpNewChangeDataList baseDataList = dataListA for _, v := range baseDataList { baseDataMap[v.DataTime] = v.Value } } else if frequencyIntMap[frequencyA] > frequencyIntMap[frequencyB] { // 如果B指标是高频,那么就需要对A指标进行升频 tmpNewChangeDataList, e := HandleDataByLinearRegression(dataListA, baseDataMap) if e != nil { err = fmt.Errorf("获取变频指标插值法Map失败, Err: %s", e.Error()) return } baseDataList = tmpNewChangeDataList changeDataList = dataListB for _, v := range changeDataList { changeDataMap[v.DataTime] = v.Value } } else { baseDataList = dataListA for _, v := range baseDataList { baseDataMap[v.DataTime] = v.Value } changeDataList = dataListB for _, v := range changeDataList { changeDataMap[v.DataTime] = v.Value } } } // 计算不领先也不滞后时的相关系数 baseCalculateData := make([]float64, 0) baseDataTimeArr := make([]string, 0) for i := range baseDataList { baseDataTimeArr = append(baseDataTimeArr, baseDataList[i].DataTime) baseCalculateData = append(baseCalculateData, baseDataList[i].Value) } //zeroBaseData := make([]float64, 0) //zeroCalculateData := make([]float64, 0) //for i := range baseDataTimeArr { // tmpBaseVal, ok1 := baseDataMap[baseDataTimeArr[i]] // tmpCalculateVal, ok2 := changeDataMap[baseDataTimeArr[i]] // if ok1 && ok2 { // zeroBaseData = append(zeroBaseData, tmpBaseVal) // zeroCalculateData = append(zeroCalculateData, tmpCalculateVal) // } //} //if len(zeroBaseData) != len(zeroCalculateData) { // err = fmt.Errorf("相关系数两组序列元素数不一致, %d-%d", len(baseCalculateData), len(zeroCalculateData)) // return //} //zeroRatio := utils.CalculateCorrelationByIntArr(zeroBaseData, zeroCalculateData) //if leadValue == 0 { // yData = append(yData, zeroRatio) //} // 计算领先/滞后N期 if leadValue > 0 { // 平移变频指标领先/滞后的日期(单位天) moveUnitDays := utils.FrequencyDaysMap[leadUnit] for i := range xData { //if xData[i] == 0 { // yData = append(yData, zeroRatio) // continue //} xCalculateData := make([]float64, 0) yCalculateData := make([]float64, 0) // 平移指定天数 mDays := int(moveUnitDays) * xData[i] _, dMap := MoveDataDaysToNewDataList(changeDataList, mDays) // 取出对应的基准日期的值 for i2 := range baseDataTimeArr { tmpDate := baseDataTimeArr[i2] if yVal, ok := dMap[tmpDate]; ok { xCalculateData = append(xCalculateData, baseCalculateData[i2]) yCalculateData = append(yCalculateData, yVal) } } if len(yCalculateData) <= 0 { //err = fmt.Errorf("领先滞后相关系数两组序列元素数不一致, %d-%d", len(baseCalculateData), len(yCalculateData)) //return // 领先滞后后,没有可以计算的数据了 continue } // 公式计算出领先/滞后频度对应点的相关性系数 ratio := utils.CalculateCorrelationByIntArr(xCalculateData, yCalculateData) yData = append(yData, ratio) } } xEdbIdValue = xData yDataList = make([]data_manage.YData, 0) yDate := "0000-00-00" yDataList = append(yDataList, data_manage.YData{ Date: yDate, Value: yData, }) return } // GetFactorChartDataByChartId 获取多因子相关性图表数据 func GetFactorChartDataByChartId(chartInfoId int, extraConfig string) (xEdbIdValue []int, yDataList []data_manage.YData, err error) { if chartInfoId <= 0 { return } // 指标对应的图例 extra := new(data_manage.CorrelationChartInfoExtraConfig) if extraConfig != "" { if e := json.Unmarshal([]byte(extraConfig), extra); e != nil { err = fmt.Errorf("解析图表额外配置失败, err: %v", e) return } } legends := make(map[string]*data_manage.CorrelationChartLegend) if extra != nil { for _, v := range extra.LegendConfig { s := fmt.Sprintf("%d-%d", v.SeriesId, v.EdbInfoId) legends[s] = v } } // 获取图表引用到的系列指标 chartMappingOb := new(data_manage.FactorEdbSeriesChartMapping) cond := fmt.Sprintf(" AND %s = ? AND %s = 1", chartMappingOb.Cols().ChartInfoId, chartMappingOb.Cols().EdbUsed) pars := make([]interface{}, 0) pars = append(pars, chartInfoId) chartMappings, e := chartMappingOb.GetItemsByCondition(cond, pars, []string{}, "") if e != nil { err = fmt.Errorf("获取图表引用系列指标失败") return } // 取出计算结果 yDataList = make([]data_manage.YData, 0) yDate := "0000-00-00" for k, m := range chartMappings { var values []data_manage.FactorEdbSeriesCorrelationMatrixValues if m.CalculateData != "" { e = json.Unmarshal([]byte(m.CalculateData), &values) if e != nil { err = fmt.Errorf("系列指标计算数据有误, err: %v", e) return } } var y []float64 for _, v := range values { if k == 0 { xEdbIdValue = append(xEdbIdValue, v.XData) } y = append(y, v.YData) } var yData data_manage.YData yData.Date = yDate yData.Value = y yData.SeriesEdb.SeriesId = m.FactorEdbSeriesId yData.SeriesEdb.EdbInfoId = m.EdbInfoId // 图例 s := fmt.Sprintf("%d-%d", m.FactorEdbSeriesId, m.EdbInfoId) legend := legends[s] if legend != nil { yData.Name = legend.LegendName yData.Color = legend.Color } yDataList = append(yDataList, yData) } return } // RemoveCorrelationRelate 删除相关性图表关联信息 func RemoveCorrelationRelate(chartInfoId int) (err error) { if chartInfoId <= 0 { return } // 相关性图表 chartCorrelate := new(data_manage.ChartInfoCorrelation) if e := chartCorrelate.GetItemById(chartInfoId); e != nil && e.Error() != utils.ErrNoRow() { err = fmt.Errorf("获取相关性图表信息失败, %v", e) return } if chartCorrelate == nil { return } // 删除相关性图 if e := chartCorrelate.Delete(); e != nil { err = fmt.Errorf("删除相关性图表失败, %v", e) return } // 多因子 if chartCorrelate.AnalysisMode != 1 { return } seriesIds := make([]int, 0) // 删除图表关联 chartMappingOb := new(data_manage.FactorEdbSeriesChartMapping) { cond := fmt.Sprintf(" AND %s = ?", chartMappingOb.Cols().ChartInfoId) pars := make([]interface{}, 0) pars = append(pars, chartCorrelate.CorrelationChartInfoId) items, e := chartMappingOb.GetItemsByCondition(cond, pars, []string{}, "") if e != nil { err = fmt.Errorf("获取图表关联指标系列失败, %v", e) return } for _, v := range items { if !utils.InArrayByInt(seriesIds, v.FactorEdbSeriesId) { seriesIds = append(seriesIds, v.FactorEdbSeriesId) } } removeCond := fmt.Sprintf(" %s = ?", chartMappingOb.Cols().ChartInfoId) if e = chartMappingOb.RemoveByCondition(removeCond, pars); e != nil { err = fmt.Errorf("删除图表关联指标系列失败, %v", e) return } } // 删除系列 if len(seriesIds) == 0 { return } seriesOb := new(data_manage.FactorEdbSeries) if e := seriesOb.MultiRemove(seriesIds); e != nil { err = fmt.Errorf("删除系列失败, %v", e) return } edbMappingOb := new(data_manage.FactorEdbSeriesMapping) { cond := fmt.Sprintf(" %s IN (%s)", edbMappingOb.Cols().FactorEdbSeriesId, utils.GetOrmInReplace(len(seriesIds))) pars := make([]interface{}, 0) pars = append(pars, seriesIds) if e := edbMappingOb.RemoveByCondition(cond, pars); e != nil { err = fmt.Errorf("删除系列指标失败, %v", e) return } } calculateOb := new(data_manage.FactorEdbSeriesCalculateData) { cond := fmt.Sprintf(" %s IN (%s)", calculateOb.Cols().FactorEdbSeriesId, utils.GetOrmInReplace(len(seriesIds))) pars := make([]interface{}, 0) pars = append(pars, seriesIds) if e := calculateOb.RemoveByCondition(cond, pars); e != nil { err = fmt.Errorf("删除系列指标计算失败, %v", e) return } } return } // CalculateCorrelationMatrix 计算相关性矩阵 func CalculateCorrelationMatrix(req data_manage.CalculateCorrelationMatrixPars) (resp data_manage.FactorEdbSeriesCorrelationMatrixResp, chartMappings []*data_manage.FactorEdbSeriesChartMapping, err error) { if req.BaseEdbInfoId <= 0 { err = fmt.Errorf("请选择标的指标") return } if len(req.SeriesIds) == 0 { err = fmt.Errorf("请选择因子指标系列") return } if req.Correlation.LeadValue <= 0 { err = fmt.Errorf("分析周期不允许设置为负数或0") return } if req.Correlation.LeadUnit == "" { err = fmt.Errorf("请选择分析周期频度") return } leadUnitDays, ok := utils.FrequencyDaysMap[req.Correlation.LeadUnit] if !ok { err = fmt.Errorf("错误的分析周期频度: %s", req.Correlation.LeadUnit) return } if req.Correlation.CalculateUnit == "" { err = fmt.Errorf("请选择计算窗口频度") return } calculateUnitDays, ok := utils.FrequencyDaysMap[req.Correlation.CalculateUnit] if !ok { err = fmt.Errorf("计算窗口频度有误: %s", req.Correlation.CalculateUnit) return } leadDays := 2 * req.Correlation.LeadValue * leadUnitDays calculateDays := req.Correlation.CalculateValue * calculateUnitDays if calculateDays < leadDays { err = fmt.Errorf("计算窗口必须≥2*分析周期") return } // 获取标的指标信息及数据 baseEdb, e := data_manage.GetEdbInfoById(req.BaseEdbInfoId) if e != nil { err = fmt.Errorf("获取标的指标失败, %v", e) return } dataListA := make([]*data_manage.EdbDataList, 0) { // 标的指标数据日期区间 startDate := time.Now().AddDate(0, 0, -calculateDays).Format(utils.FormatDate) endDate := time.Now().Format(utils.FormatDate) startDateTime, _ := time.ParseInLocation(utils.FormatDate, startDate, time.Local) startDate = startDateTime.AddDate(0, 0, 1).Format(utils.FormatDate) // 不包含第一天 switch baseEdb.EdbInfoType { case 0: dataListA, e = data_manage.GetEdbDataList(baseEdb.Source, baseEdb.SubSource, baseEdb.EdbInfoId, startDate, endDate) case 1: _, dataListA, _, _, e, _ = data.GetPredictDataListByPredictEdbInfoId(baseEdb.EdbInfoId, startDate, endDate, false) default: err = fmt.Errorf("标的指标类型异常: %d", baseEdb.EdbInfoType) return } if e != nil { err = fmt.Errorf("获取标的指标数据失败, %v", e) return } } // 获取因子系列 seriesIdItem := make(map[int]*data_manage.FactorEdbSeries) { ob := new(data_manage.FactorEdbSeries) cond := fmt.Sprintf(" AND %s IN (%s)", ob.Cols().PrimaryId, utils.GetOrmInReplace(len(req.SeriesIds))) pars := make([]interface{}, 0) pars = append(pars, req.SeriesIds) items, e := ob.GetItemsByCondition(cond, pars, []string{}, fmt.Sprintf("%s ASC", ob.Cols().PrimaryId)) if e != nil { err = fmt.Errorf("获取因子指标系列失败, %v", e) return } for _, v := range items { seriesIdItem[v.FactorEdbSeriesId] = v } } // 获取因子指标 edbMappings := make([]*data_manage.FactorEdbSeriesMapping, 0) edbInfoIds := make([]int, 0) { ob := new(data_manage.FactorEdbSeriesMapping) cond := fmt.Sprintf(" AND %s IN (%s)", ob.Cols().FactorEdbSeriesId, utils.GetOrmInReplace(len(req.SeriesIds))) pars := make([]interface{}, 0) pars = append(pars, req.SeriesIds) order := fmt.Sprintf("%s ASC, %s ASC", ob.Cols().FactorEdbSeriesId, ob.Cols().EdbInfoId) items, e := ob.GetItemsByCondition(cond, pars, []string{}, order) if e != nil { err = fmt.Errorf("获取系列指标失败, %v", e) return } for _, v := range items { edbInfoIds = append(edbInfoIds, v.EdbInfoId) } edbMappings = items } edbIdItem := make(map[int]*data_manage.EdbInfo) edbItems, e := data_manage.GetEdbInfoByIdList(edbInfoIds) if e != nil { err = fmt.Errorf("获取因子指标信息失败, %v", e) return } for _, v := range edbItems { edbIdItem[v.EdbInfoId] = v } calculateDataOb := new(data_manage.FactorEdbSeriesCalculateData) calculateWorkers := make(chan struct{}, 10) wg := sync.WaitGroup{} edbExists := make(map[string]bool) chartKeyMap := make(map[string]*data_manage.FactorEdbSeriesChartMapping) for _, v := range edbMappings { existsKey := fmt.Sprintf("%d-%d", v.FactorEdbSeriesId, v.EdbInfoId) if edbExists[existsKey] { continue } edbExists[existsKey] = true edbItem := edbIdItem[v.EdbInfoId] if edbItem == nil { continue } seriesItem := seriesIdItem[v.FactorEdbSeriesId] if seriesItem == nil { continue } wg.Add(1) go func(mapping *data_manage.FactorEdbSeriesMapping, edb *data_manage.EdbInfo, series *data_manage.FactorEdbSeries) { defer func() { wg.Done() <-calculateWorkers }() calculateWorkers <- struct{}{} var item data_manage.FactorEdbSeriesCorrelationMatrixItem item.SeriesId = series.FactorEdbSeriesId item.EdbInfoId = edb.EdbInfoId item.EdbCode = edb.EdbCode item.EdbName = edb.EdbName // 指标来源 edbList := make([]*data_manage.ChartEdbInfoMapping, 0) edbList = append(edbList, &data_manage.ChartEdbInfoMapping{ EdbInfoId: edb.EdbInfoId, EdbInfoCategoryType: edb.EdbInfoType, EdbType: edb.EdbType, Source: edb.Source, SourceName: edb.SourceName, }) sourceNameList, sourceNameEnList := data.GetEdbSourceByEdbInfoIdList(edbList) item.SourceName = strings.Join(sourceNameList, ",") item.SourceNameEn = strings.Join(sourceNameEnList, ",") // 获取指标数据 dataListB := make([]*data_manage.EdbDataList, 0) if series.CalculateState == data_manage.FactorEdbSeriesCalculated { cond := fmt.Sprintf(" AND %s = ? AND %s = ?", calculateDataOb.Cols().FactorEdbSeriesId, calculateDataOb.Cols().EdbInfoId) pars := make([]interface{}, 0) pars = append(pars, mapping.FactorEdbSeriesId, mapping.EdbInfoId) dataItems, e := calculateDataOb.GetItemsByCondition(cond, pars, []string{calculateDataOb.Cols().DataTime, calculateDataOb.Cols().Value}, fmt.Sprintf("%s ASC", calculateDataOb.Cols().DataTime)) if e != nil { item.Msg = fmt.Sprintf("计算失败") item.ErrMsg = fmt.Sprintf("获取计算数据失败, err: %v", e) resp.Fail = append(resp.Fail, item) return } dataListB = data_manage.TransEdbSeriesCalculateData2EdbDataList(dataItems) } else { switch edb.EdbInfoType { case 0: dataListB, e = data_manage.GetEdbDataList(edb.Source, edb.SubSource, edb.EdbInfoId, "", "") case 1: _, dataListB, _, _, e, _ = data.GetPredictDataListByPredictEdbInfoId(edb.EdbInfoId, "", "", false) default: item.Msg = fmt.Sprintf("计算失败") item.ErrMsg = fmt.Sprintf("指标类型异常, edbType: %d", edb.EdbInfoType) resp.Fail = append(resp.Fail, item) return } } // 计算相关性 xEdbIdValue, yDataList, e := CalculateCorrelation(req.Correlation.LeadValue, req.Correlation.LeadUnit, baseEdb.Frequency, edb.Frequency, dataListA, dataListB) if e != nil { item.Msg = fmt.Sprintf("计算失败") item.ErrMsg = fmt.Sprintf("相关性计算失败, err: %v", e) resp.Fail = append(resp.Fail, item) return } // X及Y轴数据 yData := yDataList[0].Value yLen := len(yData) values := make([]data_manage.FactorEdbSeriesCorrelationMatrixValues, len(xEdbIdValue)) for k, x := range xEdbIdValue { var y float64 if k >= 0 && k < yLen { y = yData[k] } y = utils.SubFloatToFloat(y, 2) values[k] = data_manage.FactorEdbSeriesCorrelationMatrixValues{ XData: x, YData: y, } } // 图表关联 newMapping := new(data_manage.FactorEdbSeriesChartMapping) newMapping.CalculateType = data_manage.FactorEdbSeriesChartCalculateTypeCorrelation // 计算参数 var calculatePars data_manage.FactorEdbSeriesChartCalculateCorrelationReq calculatePars.BaseEdbInfoId = req.BaseEdbInfoId calculatePars.LeadValue = req.Correlation.LeadValue calculatePars.LeadUnit = req.Correlation.LeadUnit calculatePars.CalculateValue = req.Correlation.CalculateValue calculatePars.CalculateUnit = req.Correlation.CalculateUnit bc, e := json.Marshal(calculatePars) if e != nil { item.Msg = fmt.Sprintf("计算失败") item.ErrMsg = fmt.Sprintf("计算参数JSON格式化失败, err: %v", e) resp.Fail = append(resp.Fail, item) return } newMapping.CalculatePars = string(bc) // 计算结果, 注此处保存的是排序前的顺序 bv, e := json.Marshal(values) if e != nil { item.Msg = fmt.Sprintf("计算失败") item.ErrMsg = fmt.Sprintf("计算结果JSON格式化失败, err: %v", e) resp.Fail = append(resp.Fail, item) return } newMapping.CalculateData = string(bv) newMapping.FactorEdbSeriesId = mapping.FactorEdbSeriesId newMapping.EdbInfoId = mapping.EdbInfoId newMapping.CreateTime = time.Now().Local() newMapping.ModifyTime = time.Now().Local() chartKeyMap[existsKey] = newMapping // 按照固定规则排期数[0 1 2 3 -1 -2 -3], 仅矩阵展示为此顺序 sort.Sort(data_manage.FactorEdbSeriesCorrelationMatrixOrder(values)) item.Msg = "计算成功" item.Values = values resp.Success = append(resp.Success, item) }(v, edbItem, seriesItem) } wg.Wait() // 新增图表关联, 此处按照顺序添加 chartMappings = make([]*data_manage.FactorEdbSeriesChartMapping, 0) for _, v := range edbMappings { k := fmt.Sprintf("%d-%d", v.FactorEdbSeriesId, v.EdbInfoId) item := chartKeyMap[k] if item == nil { continue } chartMappings = append(chartMappings, item) } return }