predict_edb_data_calculate_nhcc.go 23 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709
  1. package models
  2. import (
  3. "errors"
  4. "eta_gn/eta_index_lib/global"
  5. "eta_gn/eta_index_lib/utils"
  6. "fmt"
  7. "github.com/shopspring/decimal"
  8. "gorm.io/gorm"
  9. "math"
  10. "strings"
  11. "time"
  12. )
  13. // EdbDataPredictCalculateNhcc 拟合残差数据结构体
  14. type EdbDataPredictCalculateNhcc struct {
  15. EdbDataId int `gorm:"primaryKey;autoIncrement;column:edb_data_id"`
  16. EdbInfoId int `gorm:"column:edb_info_id" description:"指标ID"`
  17. EdbCode string `gorm:"column:edb_code" description:"指标编码"`
  18. DataTime string `gorm:"column:data_time" description:"数据日期"`
  19. Value float64 `gorm:"column:value" description:"数据值"`
  20. Status int `gorm:"column:status" description:"状态"`
  21. CreateTime time.Time `gorm:"column:status" description:"创建时间"`
  22. ModifyTime time.Time `gorm:"column:create_time" description:"修改时间"`
  23. DataTimestamp int64 `gorm:"column:modify_time" description:"数据日期时间戳"`
  24. }
  25. // SavePredictCalculateNhcc 新增拟合残差数据
  26. func SavePredictCalculateNhcc(req *EdbInfoCalculateBatchSaveReq, firstEdbInfo, secondEdbInfo *EdbInfo, edbCode, uniqueCode string, nhccDate NhccDate, sysUserId int, sysUserRealName, lang string) (edbInfo *EdbInfo, latestDateStr string, latestValue float64, err error) {
  27. to := global.DEFAULT_DmSQL.Begin()
  28. defer func() {
  29. if err != nil {
  30. to.Rollback()
  31. } else {
  32. to.Commit()
  33. }
  34. }()
  35. var existItemA, existItemB *EdbInfoCalculateMapping
  36. if req.EdbInfoId <= 0 {
  37. edbInfo = &EdbInfo{
  38. EdbInfoType: 1,
  39. SourceName: "预测拟合残差",
  40. Source: utils.DATA_SOURCE_PREDICT_CALCULATE_NHCC,
  41. EdbCode: edbCode,
  42. EdbName: req.EdbName,
  43. EdbNameSource: req.EdbName,
  44. Frequency: req.Frequency,
  45. Unit: req.Unit,
  46. StartDate: firstEdbInfo.StartDate,
  47. EndDate: firstEdbInfo.EndDate,
  48. ClassifyId: req.ClassifyId,
  49. SysUserId: sysUserId,
  50. SysUserRealName: sysUserRealName,
  51. UniqueCode: uniqueCode,
  52. CreateTime: time.Now(),
  53. ModifyTime: time.Now(),
  54. CalculateFormula: req.Formula,
  55. EdbNameEn: req.EdbName,
  56. UnitEn: req.Unit,
  57. EdbType: 2,
  58. Sort: GetAddEdbMaxSortByClassifyId(req.ClassifyId, utils.PREDICT_EDB_INFO_TYPE),
  59. }
  60. tmpErr := to.Create(edbInfo).Error
  61. if tmpErr != nil {
  62. err = tmpErr
  63. return
  64. }
  65. //第一个指标
  66. {
  67. existItemA = &EdbInfoCalculateMapping{
  68. EdbInfoCalculateMappingId: 0,
  69. EdbInfoId: edbInfo.EdbInfoId,
  70. Source: edbInfo.Source,
  71. SourceName: edbInfo.SourceName,
  72. EdbCode: edbInfo.EdbCode,
  73. FromEdbInfoId: firstEdbInfo.EdbInfoId,
  74. FromEdbCode: firstEdbInfo.EdbCode,
  75. FromEdbName: firstEdbInfo.EdbName,
  76. FromSource: firstEdbInfo.Source,
  77. FromSourceName: firstEdbInfo.SourceName,
  78. FromTag: "A",
  79. MoveValue: req.EdbInfoIdArr[0].MoveValue,
  80. Sort: 1,
  81. CreateTime: time.Now(),
  82. ModifyTime: time.Now(),
  83. }
  84. tmpErr := to.Create(existItemA).Error
  85. if tmpErr != nil {
  86. err = tmpErr
  87. return
  88. }
  89. }
  90. //第二个指标
  91. {
  92. existItemB = &EdbInfoCalculateMapping{
  93. EdbInfoCalculateMappingId: 0,
  94. EdbInfoId: edbInfo.EdbInfoId,
  95. Source: edbInfo.Source,
  96. SourceName: edbInfo.SourceName,
  97. EdbCode: edbInfo.EdbCode,
  98. FromEdbInfoId: secondEdbInfo.EdbInfoId,
  99. FromEdbCode: secondEdbInfo.EdbCode,
  100. FromEdbName: secondEdbInfo.EdbName,
  101. FromSource: secondEdbInfo.Source,
  102. FromSourceName: secondEdbInfo.SourceName,
  103. FromTag: "B",
  104. MoveValue: req.EdbInfoIdArr[1].MoveValue,
  105. Sort: 1,
  106. CreateTime: time.Now(),
  107. ModifyTime: time.Now(),
  108. }
  109. tmpErr := to.Create(existItemB).Error
  110. if tmpErr != nil {
  111. err = tmpErr
  112. return
  113. }
  114. }
  115. } else {
  116. edbInfo, err = GetEdbInfoById(req.EdbInfoId)
  117. if err != nil {
  118. return
  119. }
  120. latestDateStr = edbInfo.LatestDate
  121. latestValue = edbInfo.LatestValue
  122. nowEdbInfo := *edbInfo // 现在的指标信息
  123. //修改指标信息
  124. switch lang {
  125. case utils.EnLangVersion:
  126. edbInfo.EdbNameEn = req.EdbName
  127. edbInfo.UnitEn = req.Unit
  128. default:
  129. edbInfo.EdbName = req.EdbName
  130. edbInfo.Unit = req.Unit
  131. edbInfo.EdbNameSource = req.EdbName
  132. }
  133. edbInfo.Frequency = req.Frequency
  134. edbInfo.ClassifyId = req.ClassifyId
  135. edbInfo.CalculateFormula = req.Formula
  136. edbInfo.ModifyTime = time.Now()
  137. err = to.Model(edbInfo).Select([]string{"EdbName", "EdbNameSource", "Frequency", "Unit", "ClassifyId", "CalculateFormula", "ModifyTime", "EdbNameEn", "UnitEn"}).Updates(edbInfo).Error
  138. if err != nil {
  139. return
  140. }
  141. var existCondition string
  142. var existPars []interface{}
  143. existCondition += " AND edb_info_id=? "
  144. existPars = append(existPars, edbInfo.EdbInfoId)
  145. //查询出所有的关联指标
  146. var existList []*EdbInfoCalculateMapping
  147. existList, err = GetEdbInfoCalculateListByCondition(existCondition, existPars)
  148. if err != nil {
  149. err = fmt.Errorf("判断指标是否改变失败,Err:" + err.Error())
  150. return
  151. }
  152. for _, existItem := range existList {
  153. if existItem.FromTag == "A" {
  154. existItemA = existItem
  155. } else if existItem.FromTag == "B" {
  156. existItemB = existItem
  157. }
  158. }
  159. if existItemA == nil {
  160. err = errors.New("原自变量指标不存在")
  161. return
  162. }
  163. if existItemB == nil {
  164. err = errors.New("原因变量指标不存在")
  165. return
  166. }
  167. // 是否需要删除数据重新计算
  168. isNeedCalculateData := false
  169. // 如果截止日期变更,那么需要重新计算
  170. if req.Formula != nowEdbInfo.CalculateFormula {
  171. isNeedCalculateData = true
  172. }
  173. var isDeleteA, isDeleteB bool
  174. // 如果指标变了,那么需要删除关系
  175. {
  176. if existItemA.FromEdbInfoId != firstEdbInfo.EdbInfoId {
  177. //删除之前的A指标关联关系
  178. sql := ` DELETE FROM edb_info_calculate_mapping WHERE edb_info_id = ? and from_edb_info_id = ?`
  179. err = to.Exec(sql, edbInfo.EdbInfoId, existItemA.FromEdbInfoId).Error
  180. if err != nil {
  181. err = fmt.Errorf("删除指标A关联关系失败,Err:" + err.Error())
  182. return
  183. }
  184. isDeleteA = true
  185. }
  186. //并重新计算
  187. if existItemB.FromEdbInfoId != secondEdbInfo.EdbInfoId {
  188. //删除之前的B指标关联关系
  189. sql := ` DELETE FROM edb_info_calculate_mapping WHERE edb_info_id = ? and from_edb_info_id = ?`
  190. err = to.Exec(sql, edbInfo.EdbInfoId, existItemB.FromEdbInfoId).Error
  191. if err != nil {
  192. err = fmt.Errorf("删除指标B关联关系失败,Err:" + err.Error())
  193. return
  194. }
  195. isDeleteB = true
  196. }
  197. }
  198. //第一个指标数据
  199. {
  200. // 如果指标变了,那么需要删除关系,并重新计算
  201. if isDeleteA {
  202. //添加新的指标关系
  203. {
  204. existItemA = &EdbInfoCalculateMapping{
  205. EdbInfoCalculateMappingId: 0,
  206. EdbInfoId: edbInfo.EdbInfoId,
  207. Source: edbInfo.Source,
  208. SourceName: edbInfo.SourceName,
  209. EdbCode: edbInfo.EdbCode,
  210. FromEdbInfoId: firstEdbInfo.EdbInfoId,
  211. FromEdbCode: firstEdbInfo.EdbCode,
  212. FromEdbName: firstEdbInfo.EdbName,
  213. FromSource: firstEdbInfo.Source,
  214. FromSourceName: firstEdbInfo.SourceName,
  215. FromTag: "A",
  216. MoveValue: req.EdbInfoIdArr[0].MoveValue,
  217. Sort: 1,
  218. CreateTime: time.Now(),
  219. ModifyTime: time.Now(),
  220. }
  221. tmpErr := to.Create(existItemA).Error
  222. if tmpErr != nil {
  223. err = tmpErr
  224. return
  225. }
  226. isNeedCalculateData = true
  227. }
  228. } else if existItemA.MoveValue != req.EdbInfoIdArr[0].MoveValue {
  229. // 如果平移天数不一致的话,
  230. existItemA.ModifyTime = time.Now()
  231. existItemA.MoveValue = req.EdbInfoIdArr[0].MoveValue
  232. //_, err = to.Update(existItemA, "ModifyTime", "MoveValue")
  233. err = to.Model(existItemA).Select([]string{"ModifyTime", "MoveValue"}).Updates(existItemA).Error
  234. if err != nil {
  235. return
  236. }
  237. isNeedCalculateData = true
  238. }
  239. }
  240. //第二个指标数据
  241. {
  242. // 如果指标变了,那么需要删除关系,并重新计算
  243. if isDeleteB {
  244. // 添加新的指标关联关系
  245. existItemB = &EdbInfoCalculateMapping{
  246. EdbInfoCalculateMappingId: 0,
  247. EdbInfoId: edbInfo.EdbInfoId,
  248. Source: edbInfo.Source,
  249. SourceName: edbInfo.SourceName,
  250. EdbCode: edbInfo.EdbCode,
  251. FromEdbInfoId: secondEdbInfo.EdbInfoId,
  252. FromEdbCode: secondEdbInfo.EdbCode,
  253. FromEdbName: secondEdbInfo.EdbName,
  254. FromSource: secondEdbInfo.Source,
  255. FromSourceName: secondEdbInfo.SourceName,
  256. FromTag: "B",
  257. MoveValue: req.EdbInfoIdArr[1].MoveValue,
  258. Sort: 2,
  259. CreateTime: time.Now(),
  260. ModifyTime: time.Now(),
  261. }
  262. tmpErr := to.Create(existItemB).Error
  263. if tmpErr != nil {
  264. err = tmpErr
  265. return
  266. }
  267. isNeedCalculateData = true
  268. } else if existItemB.MoveValue != req.EdbInfoIdArr[1].MoveValue {
  269. // 如果平移天数不一致的话,
  270. existItemB.ModifyTime = time.Now()
  271. existItemB.MoveValue = req.EdbInfoIdArr[1].MoveValue
  272. //_, err = to.Update(existItemB, "ModifyTime", "MoveValue")
  273. err = to.Model(existItemB).Select([]string{"ModifyTime", "MoveValue"}).Updates(existItemB).Error
  274. if err != nil {
  275. return
  276. }
  277. isNeedCalculateData = true
  278. }
  279. }
  280. // 如果需要重新计算,那么先删除所有的指标数据,然后再重新计算
  281. if isNeedCalculateData {
  282. // 删除之前所有的指标数据
  283. tableName := GetEdbDataTableName(edbInfo.Source, edbInfo.SubSource)
  284. sql := fmt.Sprintf(` DELETE FROM %s WHERE edb_info_id = ? `, tableName)
  285. err = to.Exec(sql, edbInfo.EdbInfoId).Error
  286. if err != nil {
  287. err = fmt.Errorf("删除历史数据失败,Err:" + err.Error())
  288. return
  289. }
  290. } else {
  291. return
  292. }
  293. }
  294. //拼接数据
  295. latestDateStr, latestValue, err = refreshAllPredictCalculateNhcc(to, edbInfo, firstEdbInfo, secondEdbInfo, existItemA, existItemB, nhccDate)
  296. return
  297. }
  298. // RefreshAllPredictCalculateNhcc 刷新所有 拟合残差 数据
  299. func RefreshAllPredictCalculateNhcc(edbInfo *EdbInfo) (latestDateStr string, latestValue float64, err error) {
  300. to := global.DEFAULT_DmSQL.Begin()
  301. defer func() {
  302. if err != nil {
  303. to.Rollback()
  304. } else {
  305. to.Commit()
  306. }
  307. }()
  308. //查询关联指标信息
  309. var existCondition string
  310. var existPars []interface{}
  311. existCondition += " AND edb_info_id=? "
  312. existPars = append(existPars, edbInfo.EdbInfoId)
  313. existList, err := GetEdbInfoCalculateListByCondition(existCondition, existPars)
  314. if err != nil {
  315. err = fmt.Errorf("判断指标是否改变失败,Err:" + err.Error())
  316. return
  317. }
  318. var existItemA, existItemB *EdbInfoCalculateMapping
  319. for _, existItem := range existList {
  320. if existItem.FromTag == "A" {
  321. existItemA = existItem
  322. } else if existItem.FromTag == "B" {
  323. existItemB = existItem
  324. }
  325. }
  326. if existItemA == nil {
  327. err = errors.New("原自变量指标不存在")
  328. return
  329. }
  330. if existItemB == nil {
  331. err = errors.New("原因变量指标不存在")
  332. return
  333. }
  334. fromEdbInfo, err := GetEdbInfoById(existItemA.FromEdbInfoId)
  335. if err != nil {
  336. err = fmt.Errorf("GetEdbInfoById Err:" + err.Error())
  337. return
  338. }
  339. secondEdbInfo, err := GetEdbInfoById(existItemB.FromEdbInfoId)
  340. if err != nil {
  341. err = fmt.Errorf("GetEdbInfoById Err:" + err.Error())
  342. return
  343. }
  344. timeList := strings.Split(edbInfo.CalculateFormula, ",")
  345. startDate, err := time.ParseInLocation(utils.FormatDate, timeList[0], time.Local)
  346. if err != nil {
  347. return
  348. }
  349. endDate, err := time.ParseInLocation(utils.FormatDate, timeList[1], time.Local)
  350. if err != nil {
  351. return
  352. }
  353. if endDate.Sub(startDate).Hours() < 48 {
  354. return
  355. }
  356. nhccDate := NhccDate{
  357. StartDate: startDate,
  358. EndDate: endDate,
  359. }
  360. // 刷新数据
  361. latestDateStr, latestValue, err = refreshAllPredictCalculateNhcc(to, edbInfo, fromEdbInfo, secondEdbInfo, existItemA, existItemB, nhccDate)
  362. return
  363. }
  364. // refreshAllPredictCalculateNhcc 刷新所有 拟合残差 数据
  365. func refreshAllPredictCalculateNhcc(to *gorm.DB, edbInfo, firstEdbInfo, secondEdbInfo *EdbInfo, existItemA, existItemB *EdbInfoCalculateMapping, nhccDate NhccDate) (latestDateStr string, latestValue float64, err error) {
  366. var dataList []*EdbDataPredictCalculateNhcc
  367. sql := ` SELECT * FROM edb_data_predict_calculate_nhcc WHERE edb_info_id=? ORDER BY data_time DESC `
  368. err = to.Raw(sql, edbInfo.EdbInfoId).Scan(&dataList).Error
  369. if err != nil {
  370. return
  371. }
  372. // 计算最新的实际值的日期
  373. latestDateA, _ := time.ParseInLocation(utils.FormatDate, firstEdbInfo.LatestDate, time.Local)
  374. if existItemA.MoveValue != 0 {
  375. latestDateA = latestDateA.AddDate(0, 0, existItemA.MoveValue)
  376. }
  377. latestDateB, _ := time.ParseInLocation(utils.FormatDate, secondEdbInfo.LatestDate, time.Local)
  378. if existItemB.MoveValue != 0 {
  379. latestDateB = latestDateA.AddDate(0, 0, existItemB.MoveValue)
  380. }
  381. if latestDateA.Before(latestDateB) {
  382. latestDateStr = latestDateA.Format(utils.FormatDate)
  383. } else {
  384. latestDateStr = latestDateB.Format(utils.FormatDate)
  385. }
  386. var dateArr []string
  387. dataMap := make(map[string]*EdbDataPredictCalculateNhcc)
  388. removeDataTimeMap := make(map[string]int) //需要移除的日期数据
  389. for _, v := range dataList {
  390. dateArr = append(dateArr, v.DataTime)
  391. dataMap[v.DataTime] = v
  392. removeDataTimeMap[v.DataTime] = 1
  393. }
  394. addDataList := make([]*EdbDataPredictCalculateNhcc, 0)
  395. aDataList, secondDataList, aDataMap, bDataMap, err := getPredictNhccData(firstEdbInfo, secondEdbInfo, existItemA, existItemB, nhccDate)
  396. // 计算线性方程公式
  397. var a, b float64
  398. {
  399. coordinateData := make([]utils.Coordinate, 0)
  400. for i := nhccDate.StartDate; i.Before(nhccDate.EndDate) || i.Equal(nhccDate.EndDate); i = i.AddDate(0, 0, 1) {
  401. dateStr := i.Format(utils.FormatDate)
  402. xValue, ok := aDataMap[dateStr]
  403. if !ok {
  404. err = errors.New("指标A日期:" + dateStr + "数据异常,导致计算线性方程公式失败")
  405. return
  406. }
  407. yValue, ok := bDataMap[dateStr]
  408. if !ok {
  409. err = errors.New("指标B日期:" + dateStr + "数据异常,导致计算线性方程公式失败")
  410. return
  411. }
  412. tmpCoordinate := utils.Coordinate{
  413. X: xValue,
  414. Y: yValue,
  415. }
  416. coordinateData = append(coordinateData, tmpCoordinate)
  417. }
  418. a, b = utils.GetLinearResult(coordinateData)
  419. }
  420. if math.IsNaN(a) || math.IsNaN(b) {
  421. err = errors.New("线性方程公式生成失败")
  422. return
  423. }
  424. //fmt.Println("a:", a, ";======b:", b)
  425. //计算B’
  426. newBDataMap := make(map[string]float64)
  427. {
  428. //B’=aA+b
  429. aDecimal := decimal.NewFromFloat(a)
  430. bDecimal := decimal.NewFromFloat(b)
  431. for _, aData := range aDataList {
  432. xDecimal := decimal.NewFromFloat(aData.Value)
  433. val, _ := aDecimal.Mul(xDecimal).Add(bDecimal).Round(4).Float64()
  434. newBDataMap[aData.DataTime] = val
  435. }
  436. }
  437. // Delta=B-B‘
  438. for _, bData := range secondDataList {
  439. currDate := bData.DataTime
  440. //校验待删除日期数据里面是否存在该元素,如果存在的话,那么移除该元素
  441. if _, ok := removeDataTimeMap[currDate]; ok {
  442. delete(removeDataTimeMap, currDate)
  443. }
  444. b2Val, ok := newBDataMap[currDate]
  445. if !ok {
  446. // 如果B`不存在数据,那么就退出当前循环
  447. continue
  448. }
  449. bDecimal := decimal.NewFromFloat(bData.Value)
  450. b2Decimal := decimal.NewFromFloat(b2Val)
  451. val, _ := bDecimal.Sub(b2Decimal).Round(4).Float64()
  452. // 判断之前有没有该数据
  453. existData, ok := dataMap[currDate]
  454. if !ok { //不存在那么就添加吧
  455. currentDate, _ := time.ParseInLocation(utils.FormatDate, currDate, time.Local)
  456. timestamp := currentDate.UnixNano() / 1e6
  457. edbDataNhcc := &EdbDataPredictCalculateNhcc{
  458. EdbInfoId: edbInfo.EdbInfoId,
  459. EdbCode: edbInfo.EdbCode,
  460. DataTime: currDate,
  461. Value: val,
  462. Status: 1,
  463. CreateTime: time.Now(),
  464. ModifyTime: time.Now(),
  465. DataTimestamp: timestamp,
  466. }
  467. addDataList = append(addDataList, edbDataNhcc)
  468. } else {
  469. // 如果有的话,还需要判断值是否一致,一致则不处理,不一致则修改
  470. if existData.Value != val {
  471. existData.Value = val
  472. //_, err = to.Update(existData, "Value")
  473. err = to.Model(existData).Select([]string{"Value"}).Updates(existData).Error
  474. if err != nil {
  475. return
  476. }
  477. }
  478. }
  479. }
  480. //删除已经不存在的拟合残差指标数据(由于该指标当日的数据删除了)
  481. {
  482. removeDateList := make([]string, 0)
  483. for dateTime := range removeDataTimeMap {
  484. removeDateList = append(removeDateList, dateTime)
  485. }
  486. removeNum := len(removeDateList)
  487. if removeNum > 0 {
  488. //如果拼接指标变更了,那么需要删除所有的指标数据
  489. tableName := GetEdbDataTableName(edbInfo.Source, edbInfo.SubSource)
  490. sql := fmt.Sprintf(` DELETE FROM %s WHERE edb_info_id = ? and data_time in (`+utils.GetOrmInReplace(removeNum)+`) `, tableName)
  491. err = to.Exec(sql, edbInfo.EdbInfoId, removeDateList).Error
  492. if err != nil {
  493. err = fmt.Errorf("删除不存在的拟合残差指标数据失败,Err:" + err.Error())
  494. return
  495. }
  496. }
  497. }
  498. //数据入库
  499. if len(addDataList) > 0 {
  500. tmpAddDataList := make([]*EdbDataPredictCalculateNhcc, 0)
  501. i := 0
  502. for _, v := range addDataList {
  503. tmpAddDataList = append(tmpAddDataList, v)
  504. i++
  505. if i >= 500 {
  506. err = to.CreateInBatches(tmpAddDataList, 500).Error
  507. if err != nil {
  508. return
  509. }
  510. i = 0
  511. tmpAddDataList = make([]*EdbDataPredictCalculateNhcc, 0)
  512. }
  513. }
  514. if len(tmpAddDataList) > 0 {
  515. err = to.CreateInBatches(tmpAddDataList, 500).Error
  516. if err != nil {
  517. return
  518. }
  519. }
  520. }
  521. //确定最终值
  522. var finalLast EdbInfoSearchData
  523. sql = ` SELECT data_time , value FROM edb_data_predict_calculate_nhcc WHERE edb_info_id=? and data_time<=? ORDER BY data_time DESC `
  524. tmpErr := to.Raw(sql, edbInfo.EdbInfoId, latestDateStr).First(&finalLast).Error
  525. if tmpErr != nil {
  526. if tmpErr.Error() != utils.ErrNoRow() {
  527. err = tmpErr
  528. }
  529. return
  530. } else {
  531. latestDateStr = finalLast.DataTime
  532. latestValue = finalLast.Value
  533. }
  534. return
  535. }
  536. // CalculatePredictComputeCorrelation 计算相关性结果
  537. func CalculatePredictComputeCorrelation(req *EdbInfoCalculateBatchSaveReq, firstEdbInfo, secondEdbInfo *EdbInfo, nhccDate NhccDate) (r float64, err error) {
  538. var existItemA, existItemB *EdbInfoCalculateMapping
  539. //第一个指标
  540. {
  541. existItemA = &EdbInfoCalculateMapping{
  542. EdbInfoCalculateMappingId: 0,
  543. FromEdbInfoId: firstEdbInfo.EdbInfoId,
  544. FromEdbCode: firstEdbInfo.EdbCode,
  545. FromEdbName: firstEdbInfo.EdbName,
  546. FromSource: firstEdbInfo.Source,
  547. FromSourceName: firstEdbInfo.SourceName,
  548. FromTag: "A",
  549. MoveValue: req.EdbInfoIdArr[0].MoveValue,
  550. Sort: 1,
  551. CreateTime: time.Now(),
  552. ModifyTime: time.Now(),
  553. }
  554. }
  555. //第二个指标
  556. {
  557. existItemB = &EdbInfoCalculateMapping{
  558. EdbInfoCalculateMappingId: 0,
  559. FromEdbInfoId: secondEdbInfo.EdbInfoId,
  560. FromEdbCode: secondEdbInfo.EdbCode,
  561. FromEdbName: secondEdbInfo.EdbName,
  562. FromSource: secondEdbInfo.Source,
  563. FromSourceName: secondEdbInfo.SourceName,
  564. FromTag: "B",
  565. MoveValue: req.EdbInfoIdArr[1].MoveValue,
  566. Sort: 1,
  567. CreateTime: time.Now(),
  568. ModifyTime: time.Now(),
  569. }
  570. }
  571. to := global.DEFAULT_DmSQL.Begin()
  572. defer func() {
  573. if err != nil {
  574. to.Rollback()
  575. } else {
  576. to.Commit()
  577. }
  578. }()
  579. // 获取相关数据
  580. _, _, aDataMap, bDataMap, err := getPredictNhccData(firstEdbInfo, secondEdbInfo, existItemA, existItemB, nhccDate)
  581. // 计算相关性
  582. coordinateData := make([]utils.Coordinate, 0)
  583. for i := nhccDate.StartDate; i.Before(nhccDate.EndDate) || i.Equal(nhccDate.EndDate); i = i.AddDate(0, 0, 1) {
  584. dateStr := i.Format(utils.FormatDate)
  585. xValue, ok := aDataMap[dateStr]
  586. if !ok {
  587. err = errors.New("指标A日期:" + dateStr + "数据异常,导致计算线性方程公式失败")
  588. return
  589. }
  590. yValue, ok := bDataMap[dateStr]
  591. if !ok {
  592. err = errors.New("指标B日期:" + dateStr + "数据异常,导致计算线性方程公式失败")
  593. return
  594. }
  595. tmpCoordinate := utils.Coordinate{
  596. X: xValue,
  597. Y: yValue,
  598. }
  599. coordinateData = append(coordinateData, tmpCoordinate)
  600. }
  601. r = utils.ComputeCorrelation(coordinateData)
  602. return
  603. }
  604. // getNhccData 获取拟合残差需要的数据
  605. func getPredictNhccData(firstEdbInfo, secondEdbInfo *EdbInfo, existItemA, existItemB *EdbInfoCalculateMapping, nhccDate NhccDate) (aDataList []EdbInfoSearchData, secondDataList []*EdbInfoSearchData, aDataMap, bDataMap map[string]float64, err error) {
  606. //第一个指标
  607. aDataList = make([]EdbInfoSearchData, 0)
  608. aDataMap = make(map[string]float64)
  609. {
  610. var firstDataList []*EdbInfoSearchData
  611. firstDataList, err = GetPredictEdbDataListAllByStartDate(firstEdbInfo, 0, "")
  612. if err != nil {
  613. return
  614. }
  615. aDataList, aDataMap = handleNhccData(firstDataList, existItemA.MoveValue)
  616. }
  617. //第二个指标
  618. bDataList := make([]EdbInfoSearchData, 0)
  619. secondDataList = make([]*EdbInfoSearchData, 0)
  620. bDataMap = make(map[string]float64)
  621. {
  622. secondDataList, err = GetPredictEdbDataListAllByStartDate(secondEdbInfo, 0, "")
  623. if err != nil {
  624. return
  625. }
  626. bDataList, bDataMap = handleNhccData(secondDataList, existItemB.MoveValue)
  627. }
  628. if len(aDataList) <= 0 {
  629. err = errors.New("指标A没有数据")
  630. return
  631. }
  632. if len(bDataList) <= 0 {
  633. err = errors.New("指标B没有数据")
  634. return
  635. }
  636. // 拟合残差计算的结束日期判断
  637. {
  638. endAData := aDataList[len(aDataList)-1]
  639. tmpEndDate, tmpErr := time.ParseInLocation(utils.FormatDate, endAData.DataTime, time.Local)
  640. if tmpErr != nil {
  641. err = tmpErr
  642. return
  643. }
  644. // 如果A指标的最新数据日期早于拟合残差的结束日期,那么就用A指标的最新数据日期
  645. if tmpEndDate.Before(nhccDate.EndDate) {
  646. nhccDate.EndDate = tmpEndDate
  647. }
  648. endBData := bDataList[len(bDataList)-1]
  649. tmpEndDate, tmpErr = time.ParseInLocation(utils.FormatDate, endBData.DataTime, time.Local)
  650. if tmpErr != nil {
  651. err = tmpErr
  652. return
  653. }
  654. // 如果B指标的最新数据日期早于拟合残差的结束日期,那么就用B指标的最新数据日期
  655. if tmpEndDate.Before(nhccDate.EndDate) {
  656. nhccDate.EndDate = tmpEndDate
  657. }
  658. }
  659. return
  660. }