edb_data_residual_analysis.go 21 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662
  1. package models
  2. import (
  3. "encoding/json"
  4. "eta/eta_index_lib/utils"
  5. "fmt"
  6. "github.com/beego/beego/v2/client/orm"
  7. "math"
  8. "sort"
  9. "strconv"
  10. "time"
  11. )
  12. type edbDataResidualAnalysis struct {
  13. EdbDataId int `orm:"column(edb_data_id);pk;auto" description:"自增id"`
  14. EdbInfoId int `orm:"column(edb_info_id)" description:"指标id"`
  15. EdbCode string `orm:"column(edb_code)" description:"指标编码"`
  16. DataTime string `orm:"column(data_time)" description:"数据日期"`
  17. Value float64 `orm:"column(value)" description:"数据值"`
  18. CreateTime time.Time `orm:"column(create_time)" description:"创建时间"`
  19. ModifyTime time.Time `orm:"column(modify_time)" description:"修改时间"`
  20. DataTimeStamp int64 `orm:"column(data_timestamp)"`
  21. }
  22. func init() {
  23. orm.RegisterModel(new(edbDataResidualAnalysis))
  24. }
  25. // AddResidualAnalysisData 新增指标数据
  26. func AddResidualAnalysisData(dataList []edbDataResidualAnalysis) (num int64, err error) {
  27. o := orm.NewOrm()
  28. num, err = o.InsertMulti(len(dataList), dataList)
  29. if err != nil {
  30. return 0, err
  31. }
  32. return num, nil
  33. }
  34. // RefreshAllCalculateResidualAnalysis 刷新残差分析
  35. func RefreshAllCalculateResidualAnalysis(edbInfoId, source, subSource, formulaInt, moveType int, fromEdbInfo *EdbInfo, edbCode, startDate, endDate, moveFrequency string) (err error) {
  36. to := orm.NewOrm()
  37. /*to, err := o.Begin()
  38. if err != nil {
  39. return
  40. }
  41. defer func() {
  42. if err != nil {
  43. fmt.Println("RefreshAllCalculateResidualAnalysis,Err:" + err.Error())
  44. _ = to.Rollback()
  45. } else {
  46. _ = to.Commit()
  47. }
  48. }()*/
  49. //只清空残差分析数据自身指标的数据
  50. configMapping, err := GetConfigMappingListByConditionNotBase(edbInfoId)
  51. //if err != nil {
  52. // return err
  53. //}
  54. //
  55. //var edbInfoIdList []int64
  56. //for _, v := range configMapping {
  57. // edbInfoIdList = append(edbInfoIdList, v.EdbInfoId)
  58. //}
  59. //清空原有数据
  60. //sql := ` DELETE FROM edb_data_residual_analysis WHERE edb_info_id in (` + utils.GetOrmInReplace(len(edbInfoIdList)) + `) `
  61. //var params []interface{}
  62. //for _, i := range edbInfoIdList {
  63. // params = append(params, i)
  64. //}
  65. var params []interface{}
  66. sql := ` DELETE FROM edb_data_residual_analysis WHERE edb_info_id = ?`
  67. params = append(params, edbInfoId)
  68. //var params []interface{}
  69. //for _, i := range edbInfoIdList {
  70. // params = append(params, i)
  71. //}
  72. _, err = to.Raw(sql, params...).Exec()
  73. if err != nil {
  74. return
  75. }
  76. //计算数据
  77. err = refreshAllCalculateResidualAnalysis(edbInfoId, source, subSource, formulaInt, moveType, fromEdbInfo, edbCode, startDate, endDate, moveFrequency, configMapping)
  78. return
  79. }
  80. // refreshAllCalculateResidualAnalysis 刷新所有残差分析
  81. func refreshAllCalculateResidualAnalysis(edbInfoId, source, subSource, formulaInt, moveType int, fromEdbInfo *EdbInfo, edbCode, startDate, endDate, moveFrequency string, configMapping []CalculateResidualAnalysisConfigMapping) (err error) {
  82. fmt.Println("refreshAllCalculateResidualAnalysis startDate:", startDate)
  83. calculateMappingList, err := GetCalculateMappingListByEdbInfoId(edbInfoId)
  84. if err != nil {
  85. return err
  86. }
  87. edbInfoIdA := calculateMappingList[0].FromEdbInfoId
  88. edbInfoIdB := calculateMappingList[1].FromEdbInfoId
  89. mappingList, err := GetEdbInfoListByIds([]int{edbInfoIdA, edbInfoIdB})
  90. if err != nil {
  91. return err
  92. }
  93. var edbInfoMappingA, edbInfoMappingB *EdbInfoList
  94. for _, v := range mappingList {
  95. if v.Unit == "无" {
  96. v.Unit = ""
  97. }
  98. if v.EdbInfoId == edbInfoIdA {
  99. edbInfoMappingA = v
  100. }
  101. if v.EdbInfoId == edbInfoIdB {
  102. edbInfoMappingB = v
  103. }
  104. }
  105. if edbInfoMappingA == nil {
  106. return fmt.Errorf("指标A不存在")
  107. }
  108. if edbInfoMappingB == nil {
  109. return fmt.Errorf("指标B不存在")
  110. }
  111. // 从配置中取出时间范围
  112. analysisConfig, err := GetResidualAnalysisConfigById(edbInfoId)
  113. if err != nil {
  114. return err
  115. }
  116. configString := analysisConfig.Config
  117. var config ResidualAnalysisConfig
  118. err = json.Unmarshal([]byte(configString), &config)
  119. // 时间处理
  120. switch config.DateType {
  121. case -1:
  122. startDate = config.StartDate
  123. endDate = config.EndDate
  124. case 0:
  125. startDate = config.StartDate
  126. endDate = ""
  127. default:
  128. startDate = utils.GetPreYearTime(config.DateType)
  129. endDate = ""
  130. }
  131. // 原始图表信息
  132. originalEdbList := make([]EdbInfoList, 0)
  133. originalEdbList, fullADataList, fullBDataList, err := fillOriginalChart(config, fromEdbInfo, mappingList, startDate, endDate, edbInfoMappingA, edbInfoMappingB, originalEdbList)
  134. if err != nil {
  135. return err
  136. }
  137. indexADataMap := map[string]*EdbData{}
  138. for _, indexData := range edbInfoMappingA.DataList {
  139. indexADataMap[indexData.DataTime] = indexData
  140. }
  141. // 映射图表信息
  142. mappingEdbList, _, _, _, err := fillMappingChartInfo(config, fromEdbInfo, edbInfoMappingA, edbInfoMappingB, originalEdbList, indexADataMap, startDate, endDate, fullADataList, fullBDataList)
  143. if err != nil {
  144. return err
  145. }
  146. // 残差图表信息
  147. residualEdbList, _, err := fillResidualChartInfo(config, fromEdbInfo, edbInfoMappingA, edbInfoMappingB, mappingEdbList)
  148. // 映射指标 与 残差指标 同步刷新
  149. for _, mapping := range configMapping {
  150. var edbDataResidualAnalysisList []edbDataResidualAnalysis
  151. if mapping.IndexType == 1 {
  152. edbInfo, err := GetEdbInfoById(int(mapping.EdbInfoId))
  153. if err != nil {
  154. return err
  155. }
  156. for _, edbData := range mappingEdbList[1].DataList {
  157. value, _ := strconv.ParseFloat(edbData.Value, 64)
  158. edbDataResidualAnalysisList = append(edbDataResidualAnalysisList, edbDataResidualAnalysis{
  159. EdbInfoId: int(mapping.EdbInfoId),
  160. EdbCode: edbInfo.EdbCode,
  161. DataTime: edbData.DataTime,
  162. Value: value,
  163. CreateTime: time.Now(),
  164. ModifyTime: time.Now(),
  165. DataTimeStamp: edbData.DataTimestamp,
  166. })
  167. }
  168. _, err = AddResidualAnalysisData(edbDataResidualAnalysisList)
  169. if err != nil {
  170. return err
  171. }
  172. } else if mapping.IndexType == 2 {
  173. edbInfo, err := GetEdbInfoById(int(mapping.EdbInfoId))
  174. if err != nil {
  175. return err
  176. }
  177. for _, edbData := range residualEdbList[1].DataList {
  178. value, _ := strconv.ParseFloat(edbData.Value, 64)
  179. edbDataResidualAnalysisList = append(edbDataResidualAnalysisList, edbDataResidualAnalysis{
  180. EdbInfoId: int(mapping.EdbInfoId),
  181. EdbCode: edbInfo.EdbCode,
  182. DataTime: edbData.DataTime,
  183. Value: value,
  184. CreateTime: time.Now(),
  185. ModifyTime: time.Now(),
  186. DataTimeStamp: edbData.DataTimestamp,
  187. })
  188. }
  189. _, err = AddResidualAnalysisData(edbDataResidualAnalysisList)
  190. if err != nil {
  191. return err
  192. }
  193. }
  194. }
  195. return
  196. }
  197. func fillResidualChartInfo(config ResidualAnalysisConfig, req *EdbInfo, edbInfoMappingA *EdbInfoList, edbInfoMappingB *EdbInfoList, mappingEdbList []EdbInfoList) ([]EdbInfoList, float64, error) {
  198. // 计算公式 映射残差 = 因变量指标 - 映射指标
  199. var edbInfoA, edbInfoB EdbInfoList
  200. if mappingEdbList[0].EdbInfoId == edbInfoMappingA.EdbInfoId {
  201. edbInfoA = mappingEdbList[0]
  202. edbInfoB = mappingEdbList[1]
  203. } else {
  204. edbInfoA = mappingEdbList[1]
  205. edbInfoB = mappingEdbList[0]
  206. }
  207. dataAList := edbInfoA.DataList
  208. edbData := make([]*EdbDataList, len(dataAList))
  209. for i, data := range dataAList {
  210. f, _ := strconv.ParseFloat(data.Value, 64)
  211. edbData[i] = &EdbDataList{
  212. Value: f,
  213. DataTimestamp: data.DataTimestamp,
  214. DataTime: data.DataTime,
  215. EdbInfoId: data.EdbInfoId,
  216. EdbDataId: data.EdbDataId,
  217. }
  218. }
  219. dataBList := edbInfoB.DataList
  220. // 映射指标开始时间
  221. var startTime string
  222. if len(dataBList) > 0 {
  223. startTime = dataBList[0].DataTime
  224. }
  225. var indexDataBMap = make(map[string]*EdbData)
  226. for _, data := range dataBList {
  227. indexDataBMap[data.DataTime] = data
  228. }
  229. // 求R2
  230. var valueB, sumValueA, averageValueA, residualQuadraticSum, totalQuadraticSum, R2 float64
  231. for _, indexData := range edbData {
  232. // 因变量的值总和
  233. sumValueA += indexData.Value
  234. }
  235. // 因变量平均值
  236. averageValueA = sumValueA / float64(len(edbData))
  237. var indexMax, indexMin float64
  238. var edbDataResp []*EdbDataList
  239. if len(edbData) > 0 {
  240. indexMax = edbData[0].Value
  241. indexMin = edbData[0].Value
  242. for _, indexData := range edbData {
  243. if dataB, ok := indexDataBMap[indexData.DataTime]; ok {
  244. f, _ := strconv.ParseFloat(dataB.Value, 64)
  245. valueB = f
  246. } else {
  247. continue
  248. }
  249. // 总因变量平方和
  250. totalQuadraticSum += math.Pow(indexData.Value-averageValueA, 2)
  251. // 补全残差值
  252. indexData.Value = math.Round((indexData.Value-valueB)*10000) / 10000
  253. // 残差平方和
  254. residualQuadraticSum += math.Pow(indexData.Value, 2)
  255. if indexData.Value > indexMax {
  256. indexMax = indexData.Value
  257. }
  258. if indexData.Value < indexMin {
  259. indexMin = indexData.Value
  260. }
  261. // 获取映射指标之后的数据
  262. if startTime != "" && utils.CompareDate(startTime, indexData.DataTime) {
  263. edbDataResp = append(edbDataResp, indexData)
  264. }
  265. }
  266. }
  267. // 计算R2 公式:R2=1-SSE/SST R2越大,越符合线性 R2 = 1 - 残差平方和/总平方和
  268. R2 = 1 - residualQuadraticSum/totalQuadraticSum
  269. mappingEdb := make([]EdbInfoList, len(mappingEdbList))
  270. copy(mappingEdb, mappingEdbList)
  271. for i, mapping := range mappingEdb {
  272. if mapping.EdbInfoId != edbInfoMappingA.EdbInfoId {
  273. toEdbData := convertEdbDataListToEdbData(edbDataResp)
  274. mappingEdb[i].DataList = toEdbData
  275. mappingEdb[i].EdbName = edbInfoMappingA.EdbName + "映射残差/" + edbInfoMappingB.EdbName
  276. if config.IndexType == 2 {
  277. if config.LeadValue > 0 {
  278. mappingEdb[i].EdbName = edbInfoMappingA.EdbName + "映射残差/" + edbInfoMappingB.EdbName + "(领先" + strconv.Itoa(config.LeadValue) + config.LeadFrequency + ")"
  279. }
  280. }
  281. }
  282. }
  283. return mappingEdb, R2, nil
  284. }
  285. func fillMappingChartInfo(config ResidualAnalysisConfig, req *EdbInfo, edbInfoMappingA *EdbInfoList, edbInfoMappingB *EdbInfoList, originalEdbList []EdbInfoList, indexADataMap map[string]*EdbData, startDate string, endDate string, fullADataList []*EdbDataList, fullBDataList []*EdbDataList) ([]EdbInfoList, float64, float64, float64, error) {
  286. // 计算公式:Y=aX+b,Y为映射后的指标,X为自变量指标
  287. // 正序:a=(L2-L1)/(R2-R1) b=L2-R2*a
  288. // 逆序:a=(L2-L1)/(R1-R2) b=L2-R1*a
  289. // L2:左轴下限 R2:右轴上限 L1:左轴上限 R1:右轴下限
  290. var a, b, r float64
  291. // 映射残差 计算a,b
  292. if config.ResidualType == 1 {
  293. if config.IsOrder {
  294. a = (config.LeftIndexMax - config.LeftIndexMin) / (config.RightIndexMin - config.RightIndexMax)
  295. b = config.LeftIndexMax - config.RightIndexMin*a
  296. } else {
  297. a = (config.LeftIndexMax - config.LeftIndexMin) / (config.RightIndexMax - config.RightIndexMin)
  298. b = config.LeftIndexMax - config.RightIndexMax*a
  299. }
  300. }
  301. dataList := edbInfoMappingB.DataList
  302. // 指标B数据补充
  303. // 新建一个切片来保存补充的数据
  304. var replenishDataList []*EdbData
  305. for index := 0; index < len(dataList)-1; index++ {
  306. // 获取当前数据和下一个数据
  307. beforeIndexData := dataList[index]
  308. afterIndexData := dataList[index+1]
  309. // 从最早时间开始,补充时间为自然日
  310. for utils.IsMoreThanOneDay(beforeIndexData.DataTime, afterIndexData.DataTime) {
  311. // 创建补充数据
  312. nextDay := utils.GetNextDay(beforeIndexData.DataTime)
  313. toTime := utils.StringToTime(nextDay)
  314. replenishIndexData := EdbData{
  315. DataTime: nextDay, // 计算下一个自然日
  316. DataTimestamp: toTime.UnixMilli(),
  317. Value: beforeIndexData.Value, // 可以选择使用前一天的值,或者其他逻辑来计算值
  318. }
  319. // 将补充数据加入补充数据列表
  320. replenishDataList = append(replenishDataList, &replenishIndexData)
  321. // 更新 beforeIndexData 为新创建的补充数据
  322. beforeIndexData = &replenishIndexData
  323. }
  324. }
  325. // 将补充数据插入原始数据列表
  326. dataList = append(dataList, replenishDataList...)
  327. // 排序
  328. sort.Sort(ByDataTime(dataList))
  329. // 拟合残差 计算a,b
  330. var coordinateList []utils.Coordinate
  331. var replenishADataList []*EdbDataList
  332. var replenishBDataList []*EdbDataList
  333. if config.ResidualType == 2 {
  334. //
  335. // 因变量指标也转换为日度
  336. for index := 0; index < len(fullADataList)-1; index++ {
  337. // 获取当前数据和下一个数据
  338. beforeIndexData := fullADataList[index]
  339. afterIndexData := fullADataList[index+1]
  340. replenishADataList = append(replenishADataList, beforeIndexData)
  341. // 从最早时间开始,补充时间为自然日
  342. if utils.IsMoreThanOneDay(beforeIndexData.DataTime, afterIndexData.DataTime) {
  343. for {
  344. // 创建补充数据
  345. nextDay := utils.GetNextDay(beforeIndexData.DataTime)
  346. toTime := utils.StringToTime(nextDay)
  347. replenishIndexData := EdbDataList{
  348. DataTime: nextDay, // 计算下一个自然日
  349. DataTimestamp: toTime.UnixMilli(),
  350. Value: beforeIndexData.Value, // 可以选择使用前一天的值,或者其他逻辑来计算值
  351. }
  352. // 将补充数据加入补充数据列表
  353. replenishADataList = append(replenishADataList, &replenishIndexData)
  354. // 更新 beforeIndexData 为新创建的补充数据
  355. beforeIndexData = &replenishIndexData
  356. // 检查是否还需要继续补充数据
  357. if !utils.IsMoreThanOneDay(beforeIndexData.DataTime, afterIndexData.DataTime) {
  358. break
  359. }
  360. }
  361. }
  362. }
  363. replenishADataList = append(replenishADataList, fullADataList[len(fullADataList)-1])
  364. // 自变量指标也转换为日度
  365. for index := 0; index < len(fullBDataList)-1; index++ {
  366. // 获取当前数据和下一个数据
  367. beforeIndexData := fullBDataList[index]
  368. afterIndexData := fullBDataList[index+1]
  369. replenishBDataList = append(replenishBDataList, beforeIndexData)
  370. // 从最早时间开始,补充时间为自然日
  371. if utils.IsMoreThanOneDay(beforeIndexData.DataTime, afterIndexData.DataTime) {
  372. for {
  373. // 创建补充数据
  374. nextDay := utils.GetNextDay(beforeIndexData.DataTime)
  375. toTime := utils.StringToTime(nextDay)
  376. replenishIndexData := EdbDataList{
  377. DataTime: nextDay, // 计算下一个自然日
  378. DataTimestamp: toTime.UnixMilli(),
  379. Value: beforeIndexData.Value, // 可以选择使用前一天的值,或者其他逻辑来计算值
  380. }
  381. // 将补充数据加入补充数据列表
  382. replenishBDataList = append(replenishBDataList, &replenishIndexData)
  383. // 更新 beforeIndexData 为新创建的补充数据
  384. beforeIndexData = &replenishIndexData
  385. // 检查是否还需要继续补充数据
  386. if !utils.IsMoreThanOneDay(beforeIndexData.DataTime, afterIndexData.DataTime) {
  387. break
  388. }
  389. }
  390. }
  391. }
  392. replenishBDataList = append(replenishBDataList, fullBDataList[len(fullBDataList)-1])
  393. // replenishADataList --> map
  394. replenishADataMap := make(map[string]*EdbDataList)
  395. for _, indexData := range replenishADataList {
  396. if (utils.StringToTime(indexData.DataTime).After(utils.StringToTime(startDate)) || utils.StringToTime(indexData.DataTime).Equal(utils.StringToTime(startDate))) && (endDate == "" || utils.StringToTime(indexData.DataTime).Before(utils.StringToTime(endDate))) {
  397. replenishADataMap[indexData.DataTime] = indexData
  398. }
  399. }
  400. for _, indexData := range replenishBDataList {
  401. if _, ok := replenishADataMap[indexData.DataTime]; ok {
  402. coordinate := utils.Coordinate{
  403. X: indexData.Value,
  404. Y: replenishADataMap[indexData.DataTime].Value,
  405. }
  406. coordinateList = append(coordinateList, coordinate)
  407. }
  408. }
  409. a, b = utils.GetLinearResult(coordinateList)
  410. r = utils.ComputeCorrelation(coordinateList)
  411. }
  412. // 填充映射指标值 使得时间长度一致
  413. dataList = FillDataBList(dataList, edbInfoMappingA)
  414. // 根据指标A的时间key,在B的映射指标中筛选出对应的值
  415. var dataBList []*EdbDataList
  416. var indexMax, indexMin string
  417. if len(dataList) > 0 {
  418. indexMax = dataList[0].Value
  419. indexMin = dataList[0].Value
  420. for _, indexData := range dataList {
  421. if _, ok := indexADataMap[indexData.DataTime]; ok {
  422. indexDataCopy := *indexData
  423. // 计算指标B映射值
  424. f, _ := strconv.ParseFloat(indexData.Value, 64)
  425. indexDataCopy.Value = fmt.Sprintf("%f", math.Round((a*f+b)*10000)/10000)
  426. // 比较最大值
  427. if indexData.Value > indexMax {
  428. indexMax = indexData.Value
  429. }
  430. // 比较最小值
  431. if indexData.Value < indexMin {
  432. indexMin = indexData.Value
  433. }
  434. // 将副本添加到 dataBList
  435. copyValue, _ := strconv.ParseFloat(indexDataCopy.Value, 64)
  436. dataBList = append(dataBList, &EdbDataList{
  437. DataTime: indexDataCopy.DataTime,
  438. DataTimestamp: indexDataCopy.DataTimestamp,
  439. EdbDataId: indexDataCopy.EdbDataId,
  440. EdbInfoId: indexDataCopy.EdbInfoId,
  441. Value: copyValue,
  442. })
  443. }
  444. }
  445. }
  446. mappingEdbList := make([]EdbInfoList, len(originalEdbList))
  447. copy(mappingEdbList, originalEdbList)
  448. for i, mapping := range mappingEdbList {
  449. if mapping.EdbInfoId != edbInfoMappingA.EdbInfoId {
  450. mappingEdbList[i].EdbInfoId = 0
  451. mappingEdbList[i].EdbCode = ""
  452. mappingEdbList[i].EdbName = edbInfoMappingB.EdbName + "映射" + edbInfoMappingA.EdbName
  453. if config.IndexType == 2 {
  454. if config.LeadValue > 0 {
  455. mappingEdbList[i].EdbName = edbInfoMappingB.EdbName + "映射" + edbInfoMappingA.EdbName + "(领先" + strconv.Itoa(config.LeadValue) + config.LeadFrequency + ")"
  456. }
  457. }
  458. edbData := convertEdbDataListToEdbData(dataBList)
  459. mappingEdbList[i].DataList = edbData
  460. }
  461. }
  462. return mappingEdbList, a, b, r, nil
  463. }
  464. // FillDataBList 填充B的数据 使得与A的时间保持一致
  465. func FillDataBList(dataList []*EdbData, edbInfoMappingA *EdbInfoList) []*EdbData {
  466. dataAList := edbInfoMappingA.DataList
  467. for utils.StringToTime(dataList[len(dataList)-1].DataTime).Before(utils.StringToTime(dataAList[len(dataAList)-1].DataTime)) {
  468. // 使用A的时间填充时间差
  469. timeDiff := utils.GetNextDayN(dataList[len(dataList)-1].DataTime, 1)
  470. // 创建新的数据点并填充 前值填充
  471. newDataPoint := &EdbData{
  472. DataTime: timeDiff,
  473. Value: dataList[len(dataList)-1].Value,
  474. DataTimestamp: utils.StringToTime(timeDiff).UnixMilli(),
  475. }
  476. // 将新数据点添加到dataList末尾
  477. dataList = append(dataList, newDataPoint)
  478. }
  479. return dataList
  480. }
  481. func fillOriginalChart(config ResidualAnalysisConfig, req *EdbInfo, mappingList []*EdbInfoList, startDate string, endDate string, edbInfoMappingA *EdbInfoList, edbInfoMappingB *EdbInfoList, originalEdbList []EdbInfoList) ([]EdbInfoList, []*EdbDataList, []*EdbDataList, error) {
  482. var fullADataList, fullBDataList []*EdbDataList
  483. for _, v := range mappingList {
  484. var edbInfoMapping EdbInfoList
  485. edbInfoMapping.EdbName = v.EdbName
  486. // 获取图表中的指标数据
  487. dataList, err := GetEdbDataList(v.Source, v.SubSource, v.EdbInfoId, startDate, endDate)
  488. if err != nil {
  489. return nil, nil, nil, fmt.Errorf("获取指标数据失败,Err:%s", err.Error())
  490. }
  491. data := convertEdbDataListToEdbData(dataList)
  492. // 重新获取指标数据 产品要求需要和计算指标-拟合残差逻辑保持一致
  493. fullDataList, err := GetEdbDataList(v.Source, v.SubSource, v.EdbInfoId, "", "")
  494. if err != nil {
  495. return nil, nil, nil, fmt.Errorf("获取指标数据失败,Err:%s", err.Error())
  496. }
  497. if v.EdbInfoId == edbInfoMappingB.EdbInfoId {
  498. // 领先指标 dataList进行数据处理
  499. if config.IndexType == 2 {
  500. if config.LeadValue < 0 {
  501. return nil, nil, nil, fmt.Errorf("领先值不能小于0")
  502. } else if config.LeadValue > 0 {
  503. edbInfoMapping.EdbName = v.EdbName + "(领先" + strconv.Itoa(config.LeadValue) + config.LeadFrequency + ")"
  504. for _, indexData := range dataList {
  505. switch config.LeadFrequency {
  506. case "天":
  507. indexData.DataTime = utils.GetNextDayN(indexData.DataTime, config.LeadValue)
  508. case "周":
  509. indexData.DataTime = utils.GetNextDayN(indexData.DataTime, config.LeadValue*7)
  510. case "月":
  511. indexData.DataTime = utils.TimeToString(utils.AddDate(utils.StringToTime(indexData.DataTime), 0, config.LeadValue), utils.YearMonthDay)
  512. case "季":
  513. indexData.DataTime = utils.TimeToString(utils.AddDate(utils.StringToTime(indexData.DataTime), 0, config.LeadValue*3), utils.YearMonthDay)
  514. case "年":
  515. indexData.DataTime = utils.TimeToString(utils.AddDate(utils.StringToTime(indexData.DataTime), config.LeadValue, 0), utils.YearMonthDay)
  516. }
  517. indexData.DataTimestamp = utils.StringToTime(indexData.DataTime).UnixMilli()
  518. }
  519. }
  520. }
  521. edbInfoMappingB.DataList = data
  522. fullBDataList = fullDataList
  523. } else {
  524. edbInfoMappingA.DataList = data
  525. fullADataList = fullDataList
  526. }
  527. edbInfoMapping.EdbInfoId = v.EdbInfoId
  528. edbInfoMapping.EdbCode = v.EdbCode
  529. edbInfoMapping.Unit = v.Unit
  530. edbInfoMapping.Frequency = v.Frequency
  531. edbInfoMapping.Source = v.Source
  532. edbInfoMapping.SourceName = v.SourceName
  533. edbInfoMapping.LatestDate = v.LatestDate
  534. edbInfoMapping.LatestValue = v.LatestValue
  535. edbInfoMapping.DataList = data
  536. originalEdbList = append(originalEdbList, edbInfoMapping)
  537. }
  538. return originalEdbList, fullADataList, fullBDataList, nil
  539. }
  540. func convertEdbDataListToEdbData(edbDataLists []*EdbDataList) []*EdbData {
  541. var edbDataList []*EdbData
  542. for _, edbData := range edbDataLists {
  543. data := &EdbData{
  544. DataTime: edbData.DataTime,
  545. DataTimestamp: edbData.DataTimestamp,
  546. EdbDataId: edbData.EdbDataId,
  547. EdbInfoId: edbData.EdbInfoId,
  548. Value: fmt.Sprintf("%f", edbData.Value),
  549. }
  550. edbDataList = append(edbDataList, data)
  551. }
  552. return edbDataList
  553. }
  554. type ByDataTime []*EdbData
  555. func (a ByDataTime) Len() int {
  556. return len(a)
  557. }
  558. func (a ByDataTime) Swap(i, j int) {
  559. a[i], a[j] = a[j], a[i]
  560. }
  561. func (a ByDataTime) Less(i, j int) bool {
  562. t1 := utils.StringToTime(a[i].DataTime)
  563. t2 := utils.StringToTime(a[j].DataTime)
  564. return t1.Before(t2)
  565. }