残差分析 指标刷新

models/calculate_residual_analysis_config_mapping.go

@@ -0,0 +1,52 @@
+package models
+
+import (
+	"fmt"
+	"github.com/beego/beego/v2/client/orm"
+	"time"
+)
+
+type CalculateResidualAnalysisConfigMapping struct {
+	CalculateResidualAnalysisConfigMappingId int       `orm:"column(calculate_residual_analysis_config_mapping_id);pk;auto" description:"自增id"`
+	CalculateResidualAnalysisConfigId        int       `orm:"column(calculate_residual_analysis_config_id)" description:"残差分析配置id"`
+	EdbInfoId                                int64     `orm:"column(edb_info_id)" description:"指标id"`
+	ResidualType                             int       `orm:"column(residual_type)" description:"残差类型: 1-映射残差 2-拟合残差"`
+	IndexType                                int       `orm:"column(index_type)" description:"指标类型：1-映射指标 2-残差指标 3-因变量指标 4-自变量指标"`
+	CreateTime                               time.Time `orm:"column(create_time)" description:"创建时间"`
+	ModifyTime                               time.Time `orm:"column(modify_time)" description:"修改时间"`
+}
+
+func init() {
+	orm.RegisterModel(new(CalculateResidualAnalysisConfigMapping))
+}
+
+// GetConfigMappingListByConfigId 根据配置配置id查询配置信息
+func GetConfigMappingListByConfigId(configId int) (items []CalculateResidualAnalysisConfigMapping, err error) {
+	o := orm.NewOrmUsingDB("data")
+	sql := `select * from calculate_residual_analysis_config_mapping where calculate_residual_analysis_config_id = ?`
+
+	_, err = o.Raw(sql, configId).QueryRows(&items)
+	return items, nil
+}
+
+// GetConfigMappingListByCondition 通过条件查询指标配置映射
+func GetConfigMappingListByCondition(condition string, pars []interface{}) (items []CalculateResidualAnalysisConfigMapping, err error) {
+	o := orm.NewOrmUsingDB("data")
+	sql := `select * from calculate_residual_analysis_config_mapping where 1=1 `
+
+	sql += condition
+
+	_, err = o.Raw(sql, pars).QueryRows(&items)
+	return items, nil
+}
+
+// GetConfigMappingListByConditionNotBase 查询非基础指标的配置映射
+func GetConfigMappingListByConditionNotBase(edbInfoId int) (configMapping []CalculateResidualAnalysisConfigMapping, err error) {
+	o := orm.NewOrmUsingDB("data")
+	sql := `select * from calculate_residual_analysis_config_mapping where 1=1 and edb_info_id in (select edb_info_id from edb_info where index_type != 3 and index_type != 4 and edb_info_id = ?) `
+	_, err = o.Raw(sql, edbInfoId).QueryRows(&configMapping)
+	if err != nil {
+		return nil, fmt.Errorf("查询数据时出错: %v", err)
+	}
+	return configMapping, nil
+}

models/edb_data_residual_analysis.go

@@ -8,8 +8,35 @@ import (
 	"math"
 	"sort"
 	"strconv"
+	"time"
 )
 
+type edbDataResidualAnalysis struct {
+	EdbDataId     int       `orm:"column(edb_data_id);pk;auto" description:"自增id"`
+	EdbInfoId     int       `orm:"column(edb_info_id)" description:"指标id"`
+	EdbCode       string    `orm:"column(edb_code)" description:"指标编码"`
+	DataTime      string    `orm:"column(data_time)" description:"数据日期"`
+	Value         float64   `orm:"column(value)" description:"数据值"`
+	CreateTime    time.Time `orm:"column(create_time)" description:"创建时间"`
+	ModifyTime    time.Time `orm:"column(modify_time)" description:"修改时间"`
+	DataTimeStamp int64     `orm:"column(data_timestamp)"`
+}
+
+func init() {
+	orm.RegisterModel(new(edbDataResidualAnalysis))
+}
+
+// AddResidualAnalysisData 新增指标数据
+func AddResidualAnalysisData(dataList []edbDataResidualAnalysis) (num int64, err error) {
+	o := orm.NewOrmUsingDB("data")
+	num, err = o.InsertMulti(len(dataList), dataList)
+	if err != nil {
+		return 0, err
+	}
+
+	return num, nil
+}
+
 // RefreshAllCalculateResidualAnalysis 刷新残差分析
 func RefreshAllCalculateResidualAnalysis(edbInfoId, source, subSource, formulaInt, moveType int, fromEdbInfo *EdbInfo, edbCode, startDate, endDate, moveFrequency string) (err error) {
 	o := orm.NewOrm()
@@ -25,33 +52,33 @@ func RefreshAllCalculateResidualAnalysis(edbInfoId, source, subSource, formulaIn
 			_ = to.Commit()
 		}
 	}()
+	configMapping, err := GetConfigMappingListByConditionNotBase(edbInfoId)
+	if err != nil {
+		return err
+	}
+
+	var edbInfoIdList []int64
+	for _, v := range configMapping {
+		edbInfoIdList = append(edbInfoIdList, v.EdbInfoId)
+	}
 
 	//清空原有数据
-	sql := ` DELETE FROM edb_data_residual_analysis WHERE edb_info_id = ? `
-	_, err = to.Raw(sql, edbInfoId).Exec()
+	sql := ` DELETE FROM edb_data_residual_analysis WHERE edb_info_id in (` + utils.GetOrmInReplace(len(edbInfoIdList)) + `) `
+	_, err = to.Raw(sql, edbInfoIdList).Exec()
 	if err != nil {
 		return
 	}
 
 	//计算数据
-	err = refreshAllCalculateResidualAnalysis(to, edbInfoId, source, subSource, formulaInt, moveType, fromEdbInfo, edbCode, startDate, endDate, moveFrequency)
+	err = refreshAllCalculateResidualAnalysis(to, edbInfoId, source, subSource, formulaInt, moveType, fromEdbInfo, edbCode, startDate, endDate, moveFrequency, configMapping)
 
 	return
 }
 
 // refreshAllCalculateResidualAnalysis 刷新所有残差分析
-func refreshAllCalculateResidualAnalysis(to orm.TxOrmer, edbInfoId, source, subSource, formulaInt, moveType int, fromEdbInfo *EdbInfo, edbCode, startDate, endDate, moveFrequency string) (err error) {
+func refreshAllCalculateResidualAnalysis(to orm.TxOrmer, edbInfoId, source, subSource, formulaInt, moveType int, fromEdbInfo *EdbInfo, edbCode, startDate, endDate, moveFrequency string, configMapping []CalculateResidualAnalysisConfigMapping) (err error) {
 	fmt.Println("refreshAllCalculateResidualAnalysis startDate:", startDate)
 
-	//计算数据
-	/*dataList, err := GetEdbDataListAllByTo(to, fromEdbInfo.Source, fromEdbInfo.SubSource, FindEdbDataListAllCond{
-		EdbInfoId:         fromEdbInfo.EdbInfoId,
-		StartDataTime:     startDate,
-		StartDataTimeCond: ">=",
-	}, 0)
-	if err != nil {
-		return err
-	}*/
 	calculateMappingList, err := GetCalculateMappingListByEdbInfoId(edbInfoId)
 	if err != nil {
 		return err
@@ -125,27 +152,159 @@ func refreshAllCalculateResidualAnalysis(to orm.TxOrmer, edbInfoId, source, subS
 		return err
 	}
 
-	mappingChartInfo := createChartInfoResp(req, startDate, endDate, edbInfoMappingA.EdbName+"与"+edbInfoMappingB.EdbName+"映射"+edbInfoMappingA.EdbName)
+	// 残差图表信息
+	residualEdbList, _, err := fillResidualChartInfo(config, fromEdbInfo, edbInfoMappingA, edbInfoMappingB, mappingEdbList)
+
+	// 映射指标 与 残差指标 同步刷新
+	for _, mapping := range configMapping {
+		var edbDataResidualAnalysisList []edbDataResidualAnalysis
+		if mapping.IndexType == 1 {
+			for _, edbData := range mappingEdbList[1].DataList {
+				value, _ := strconv.ParseFloat(edbData.Value, 64)
+				edbDataResidualAnalysisList = append(edbDataResidualAnalysisList, edbDataResidualAnalysis{
+					EdbInfoId:     int(mapping.EdbInfoId),
+					EdbCode:       edbData.EdbCode,
+					DataTime:      edbData.DataTime,
+					Value:         value,
+					CreateTime:    time.Now(),
+					ModifyTime:    time.Now(),
+					DataTimeStamp: edbData.DataTimestamp,
+				})
+			}
+
+			_, err = AddResidualAnalysisData(edbDataResidualAnalysisList)
+			if err != nil {
+				return err
+			}
+		} else if mapping.IndexType == 2 {
+			for _, edbData := range residualEdbList[1].DataList {
+				value, _ := strconv.ParseFloat(edbData.Value, 64)
+				edbDataResidualAnalysisList = append(edbDataResidualAnalysisList, edbDataResidualAnalysis{
+					EdbInfoId:     int(mapping.EdbInfoId),
+					EdbCode:       edbData.EdbCode,
+					DataTime:      edbData.DataTime,
+					Value:         value,
+					CreateTime:    time.Now(),
+					ModifyTime:    time.Now(),
+					DataTimeStamp: edbData.DataTimestamp,
+				})
+			}
 
-	resp.MappingChartData = residual_analysis_model.ChartResp{
-		ChartInfo:   mappingChartInfo,
-		EdbInfoList: mappingEdbList,
+			_, err = AddResidualAnalysisData(edbDataResidualAnalysisList)
+			if err != nil {
+				return err
+			}
+		}
 	}
 
-	// 残差图表信息
-	residualEdbList, R2, err := fillResidualChartInfo(req, edbInfoMappingA, edbInfoMappingB, mappingEdbList)
-	if err != nil {
-		return residual_analysis_model.ResidualAnalysisResp{}, err
+	return
+}
+
+func fillResidualChartInfo(config ResidualAnalysisConfig, req *EdbInfo, edbInfoMappingA *EdbInfoList, edbInfoMappingB *EdbInfoList, mappingEdbList []EdbInfoList) ([]EdbInfoList, float64, error) {
+	// 计算公式 映射残差 = 因变量指标 - 映射指标
+	var edbInfoA, edbInfoB EdbInfoList
+	if mappingEdbList[0].EdbInfoId == edbInfoMappingA.EdbInfoId {
+		edbInfoA = mappingEdbList[0]
+		edbInfoB = mappingEdbList[1]
+	} else {
+		edbInfoA = mappingEdbList[1]
+		edbInfoB = mappingEdbList[0]
+	}
+	dataAList := edbInfoA.DataList
+	edbData := make([]*EdbDataList, len(dataAList))
+	for i, data := range dataAList {
+		f, _ := strconv.ParseFloat(data.Value, 64)
+		edbData[i] = &EdbDataList{
+			Value:         f,
+			DataTimestamp: data.DataTimestamp,
+			DataTime:      data.DataTime,
+			EdbInfoId:     data.EdbInfoId,
+			EdbDataId:     data.EdbDataId,
+		}
 	}
 
-	residualChartInfo := createChartInfoResp(req, startDate, endDate, edbInfoMappingA.EdbName+"与"+edbInfoMappingA.EdbName+"映射残差/"+edbInfoMappingB.EdbName)
+	dataBList := edbInfoB.DataList
 
-	resp.ResidualChartData = residual_analysis_model.ChartResp{
-		ChartInfo:   residualChartInfo,
-		EdbInfoList: residualEdbList,
+	// 映射指标开始时间
+	var startTime string
+	if len(dataBList) > 0 {
+		startTime = dataBList[0].DataTime
 	}
 
-	return
+	var indexDataBMap = make(map[string]*EdbData)
+	for _, data := range dataBList {
+		indexDataBMap[data.DataTime] = data
+	}
+	// 求R2
+	var valueB, sumValueA, averageValueA, residualQuadraticSum, totalQuadraticSum, R2 float64
+
+	for _, indexData := range edbData {
+		// 因变量的值总和
+		sumValueA += indexData.Value
+	}
+	// 因变量平均值
+	averageValueA = sumValueA / float64(len(edbData))
+
+	var indexMax, indexMin float64
+
+	var edbDataResp []*EdbDataList
+	if len(edbData) > 0 {
+		indexMax = edbData[0].Value
+		indexMin = edbData[0].Value
+		for _, indexData := range edbData {
+			if dataB, ok := indexDataBMap[indexData.DataTime]; ok {
+				f, _ := strconv.ParseFloat(dataB.Value, 64)
+				valueB = f
+			} else {
+				continue
+			}
+
+			// 总因变量平方和
+			totalQuadraticSum += math.Pow(indexData.Value-averageValueA, 2)
+
+			// 补全残差值
+
+			indexData.Value = math.Round((indexData.Value-valueB)*10000) / 10000
+
+			// 残差平方和
+			residualQuadraticSum += math.Pow(indexData.Value, 2)
+
+			if indexData.Value > indexMax {
+				indexMax = indexData.Value
+			}
+			if indexData.Value < indexMin {
+				indexMin = indexData.Value
+			}
+
+			// 获取映射指标之后的数据
+			if startTime != "" && utils.CompareDate(startTime, indexData.DataTime) {
+				edbDataResp = append(edbDataResp, indexData)
+			}
+		}
+	}
+
+	// 计算R2 公式：R2=1-SSE/SST R2越大，越符合线性  R2 = 1 - 残差平方和/总平方和
+	R2 = 1 - residualQuadraticSum/totalQuadraticSum
+
+	mappingEdb := make([]EdbInfoList, len(mappingEdbList))
+	copy(mappingEdb, mappingEdbList)
+
+	for i, mapping := range mappingEdb {
+		if mapping.EdbInfoId != edbInfoMappingA.EdbInfoId {
+
+			toEdbData := convertEdbDataListToEdbData(edbDataResp)
+
+			mappingEdb[i].DataList = toEdbData
+			mappingEdb[i].EdbName = edbInfoMappingA.EdbName + "映射残差/" + edbInfoMappingB.EdbName
+			if config.IndexType == 2 {
+				if config.LeadValue > 0 {
+					mappingEdb[i].EdbName = edbInfoMappingA.EdbName + "映射残差/" + edbInfoMappingB.EdbName + "（领先" + strconv.Itoa(config.LeadValue) + config.LeadFrequency + "）"
+				}
+			}
+		}
+	}
+
+	return mappingEdb, R2, nil
 }
 
 func fillMappingChartInfo(config ResidualAnalysisConfig, req *EdbInfo, edbInfoMappingA *EdbInfoList, edbInfoMappingB *EdbInfoList, originalEdbList []EdbInfoList, indexADataMap map[string]*EdbData, startDate string, endDate string, fullADataList []*EdbDataList, fullBDataList []*EdbDataList) ([]EdbInfoList, float64, float64, float64, error) {
@@ -355,7 +514,9 @@ func fillMappingChartInfo(config ResidualAnalysisConfig, req *EdbInfo, edbInfoMa
 				}
 			}
 
-			mappingEdbList[i].DataList = dataBList
+			edbData := convertEdbDataListToEdbData(dataBList)
+
+			mappingEdbList[i].DataList = edbData
 		}
 	}
 	return mappingEdbList, a, b, r, nil