predict_edb_data_calculate_nhcc.go 23 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730
  1. package models
  2. import (
  3. "errors"
  4. "eta/eta_index_lib/utils"
  5. "fmt"
  6. "github.com/beego/beego/v2/client/orm"
  7. "github.com/shopspring/decimal"
  8. "math"
  9. "strings"
  10. "time"
  11. )
  12. // EdbDataPredictCalculateNhcc 拟合残差数据结构体
  13. type EdbDataPredictCalculateNhcc struct {
  14. EdbDataId int `orm:"column(edb_data_id);pk"`
  15. EdbInfoId int
  16. EdbCode string
  17. DataTime string
  18. Value float64
  19. Status int
  20. CreateTime time.Time
  21. ModifyTime time.Time
  22. DataTimestamp int64
  23. }
  24. // SavePredictCalculateNhcc 新增拟合残差数据
  25. func SavePredictCalculateNhcc(req *EdbInfoCalculateBatchSaveReq, firstEdbInfo, secondEdbInfo *EdbInfo, edbCode, uniqueCode string, nhccDate NhccDate, sysUserId int, sysUserRealName string) (edbInfo *EdbInfo, latestDateStr string, latestValue float64, err error) {
  26. o := orm.NewOrm()
  27. to, err := o.Begin()
  28. if err != nil {
  29. return
  30. }
  31. defer func() {
  32. if err != nil {
  33. fmt.Println("SavePredictCalculateNhcc,Err:" + err.Error())
  34. _ = to.Rollback()
  35. } else {
  36. _ = to.Commit()
  37. }
  38. }()
  39. var existItemA, existItemB *EdbInfoCalculateMapping
  40. if req.EdbInfoId <= 0 {
  41. edbInfo = &EdbInfo{
  42. EdbInfoType: 1,
  43. SourceName: "预测拟合残差",
  44. Source: utils.DATA_SOURCE_PREDICT_CALCULATE_NHCC,
  45. EdbCode: edbCode,
  46. EdbName: req.EdbName,
  47. EdbNameSource: req.EdbName,
  48. Frequency: req.Frequency,
  49. Unit: req.Unit,
  50. StartDate: firstEdbInfo.StartDate,
  51. EndDate: firstEdbInfo.EndDate,
  52. ClassifyId: req.ClassifyId,
  53. SysUserId: sysUserId,
  54. SysUserRealName: sysUserRealName,
  55. UniqueCode: uniqueCode,
  56. CreateTime: time.Now(),
  57. ModifyTime: time.Now(),
  58. CalculateFormula: req.Formula,
  59. EdbType: 2,
  60. }
  61. newEdbInfoId, tmpErr := to.Insert(edbInfo)
  62. if tmpErr != nil {
  63. err = tmpErr
  64. return
  65. }
  66. edbInfo.EdbInfoId = int(newEdbInfoId)
  67. //第一个指标
  68. {
  69. existItemA = &EdbInfoCalculateMapping{
  70. EdbInfoCalculateMappingId: 0,
  71. EdbInfoId: edbInfo.EdbInfoId,
  72. Source: edbInfo.Source,
  73. SourceName: edbInfo.SourceName,
  74. EdbCode: edbInfo.EdbCode,
  75. FromEdbInfoId: firstEdbInfo.EdbInfoId,
  76. FromEdbCode: firstEdbInfo.EdbCode,
  77. FromEdbName: firstEdbInfo.EdbName,
  78. FromSource: firstEdbInfo.Source,
  79. FromSourceName: firstEdbInfo.SourceName,
  80. FromTag: "A",
  81. MoveValue: req.EdbInfoIdArr[0].MoveValue,
  82. Sort: 1,
  83. CreateTime: time.Now(),
  84. ModifyTime: time.Now(),
  85. }
  86. insertId, tmpErr := to.Insert(existItemA)
  87. if tmpErr != nil {
  88. err = tmpErr
  89. return
  90. }
  91. existItemA.EdbInfoCalculateMappingId = int(insertId)
  92. }
  93. //第二个指标
  94. {
  95. existItemB = &EdbInfoCalculateMapping{
  96. EdbInfoCalculateMappingId: 0,
  97. EdbInfoId: edbInfo.EdbInfoId,
  98. Source: edbInfo.Source,
  99. SourceName: edbInfo.SourceName,
  100. EdbCode: edbInfo.EdbCode,
  101. FromEdbInfoId: secondEdbInfo.EdbInfoId,
  102. FromEdbCode: secondEdbInfo.EdbCode,
  103. FromEdbName: secondEdbInfo.EdbName,
  104. FromSource: secondEdbInfo.Source,
  105. FromSourceName: secondEdbInfo.SourceName,
  106. FromTag: "B",
  107. MoveValue: req.EdbInfoIdArr[1].MoveValue,
  108. Sort: 1,
  109. CreateTime: time.Now(),
  110. ModifyTime: time.Now(),
  111. }
  112. insertId, tmpErr := to.Insert(existItemB)
  113. if tmpErr != nil {
  114. err = tmpErr
  115. return
  116. }
  117. existItemB.EdbInfoCalculateMappingId = int(insertId)
  118. }
  119. } else {
  120. edbInfo, err = GetEdbInfoById(req.EdbInfoId)
  121. if err != nil {
  122. return
  123. }
  124. latestDateStr = edbInfo.LatestDate
  125. latestValue = edbInfo.LatestValue
  126. nowEdbInfo := *edbInfo // 现在的指标信息
  127. //修改指标信息
  128. edbInfo.EdbName = req.EdbName
  129. edbInfo.EdbNameSource = req.EdbName
  130. edbInfo.Frequency = req.Frequency
  131. edbInfo.Unit = req.Unit
  132. edbInfo.ClassifyId = req.ClassifyId
  133. edbInfo.CalculateFormula = req.Formula
  134. edbInfo.ModifyTime = time.Now()
  135. _, err = to.Update(edbInfo, "EdbName", "EdbNameSource", "Frequency", "Unit", "ClassifyId", "CalculateFormula", "ModifyTime")
  136. if err != nil {
  137. return
  138. }
  139. var existCondition string
  140. var existPars []interface{}
  141. existCondition += " AND edb_info_id=? "
  142. existPars = append(existPars, edbInfo.EdbInfoId)
  143. //查询出所有的关联指标
  144. var existList []*EdbInfoCalculateMapping
  145. existList, err = GetEdbInfoCalculateListByCondition(existCondition, existPars)
  146. if err != nil {
  147. err = fmt.Errorf("判断指标是否改变失败,Err:" + err.Error())
  148. return
  149. }
  150. for _, existItem := range existList {
  151. if existItem.FromTag == "A" {
  152. existItemA = existItem
  153. } else if existItem.FromTag == "B" {
  154. existItemB = existItem
  155. }
  156. }
  157. if existItemA == nil {
  158. err = errors.New("原自变量指标不存在")
  159. return
  160. }
  161. if existItemB == nil {
  162. err = errors.New("原因变量指标不存在")
  163. return
  164. }
  165. // 是否需要删除数据重新计算
  166. isNeedCalculateData := false
  167. // 如果截止日期变更,那么需要重新计算
  168. if req.Formula != nowEdbInfo.CalculateFormula {
  169. isNeedCalculateData = true
  170. }
  171. var isDeleteA, isDeleteB bool
  172. // 如果指标变了,那么需要删除关系
  173. {
  174. if existItemA.FromEdbInfoId != firstEdbInfo.EdbInfoId {
  175. //删除之前的A指标关联关系
  176. sql := ` DELETE FROM edb_info_calculate_mapping WHERE edb_info_id = ? and from_edb_info_id = ?`
  177. _, err = to.Raw(sql, edbInfo.EdbInfoId, existItemA.FromEdbInfoId).Exec()
  178. if err != nil {
  179. err = fmt.Errorf("删除指标A关联关系失败,Err:" + err.Error())
  180. return
  181. }
  182. isDeleteA = true
  183. }
  184. //并重新计算
  185. if existItemB.FromEdbInfoId != secondEdbInfo.EdbInfoId {
  186. //删除之前的B指标关联关系
  187. sql := ` DELETE FROM edb_info_calculate_mapping WHERE edb_info_id = ? and from_edb_info_id = ?`
  188. _, err = to.Raw(sql, edbInfo.EdbInfoId, existItemB.FromEdbInfoId).Exec()
  189. if err != nil {
  190. err = fmt.Errorf("删除指标B关联关系失败,Err:" + err.Error())
  191. return
  192. }
  193. isDeleteB = true
  194. }
  195. }
  196. //第一个指标数据
  197. {
  198. // 如果指标变了,那么需要删除关系,并重新计算
  199. if isDeleteA {
  200. //添加新的指标关系
  201. {
  202. existItemA = &EdbInfoCalculateMapping{
  203. EdbInfoCalculateMappingId: 0,
  204. EdbInfoId: edbInfo.EdbInfoId,
  205. Source: edbInfo.Source,
  206. SourceName: edbInfo.SourceName,
  207. EdbCode: edbInfo.EdbCode,
  208. FromEdbInfoId: firstEdbInfo.EdbInfoId,
  209. FromEdbCode: firstEdbInfo.EdbCode,
  210. FromEdbName: firstEdbInfo.EdbName,
  211. FromSource: firstEdbInfo.Source,
  212. FromSourceName: firstEdbInfo.SourceName,
  213. FromTag: "A",
  214. MoveValue: req.EdbInfoIdArr[0].MoveValue,
  215. Sort: 1,
  216. CreateTime: time.Now(),
  217. ModifyTime: time.Now(),
  218. }
  219. insertId, tmpErr := to.Insert(existItemA)
  220. if tmpErr != nil {
  221. err = tmpErr
  222. return
  223. }
  224. existItemA.EdbInfoCalculateMappingId = int(insertId)
  225. isNeedCalculateData = true
  226. }
  227. } else if existItemA.MoveValue != req.EdbInfoIdArr[0].MoveValue {
  228. // 如果平移天数不一致的话,
  229. existItemA.ModifyTime = time.Now()
  230. existItemA.MoveValue = req.EdbInfoIdArr[0].MoveValue
  231. _, err = to.Update(existItemA, "ModifyTime", "MoveValue")
  232. if err != nil {
  233. return
  234. }
  235. isNeedCalculateData = true
  236. }
  237. }
  238. //第二个指标数据
  239. {
  240. // 如果指标变了,那么需要删除关系,并重新计算
  241. if isDeleteB {
  242. // 添加新的指标关联关系
  243. existItemB = &EdbInfoCalculateMapping{
  244. EdbInfoCalculateMappingId: 0,
  245. EdbInfoId: edbInfo.EdbInfoId,
  246. Source: edbInfo.Source,
  247. SourceName: edbInfo.SourceName,
  248. EdbCode: edbInfo.EdbCode,
  249. FromEdbInfoId: secondEdbInfo.EdbInfoId,
  250. FromEdbCode: secondEdbInfo.EdbCode,
  251. FromEdbName: secondEdbInfo.EdbName,
  252. FromSource: secondEdbInfo.Source,
  253. FromSourceName: secondEdbInfo.SourceName,
  254. FromTag: "B",
  255. MoveValue: req.EdbInfoIdArr[1].MoveValue,
  256. Sort: 2,
  257. CreateTime: time.Now(),
  258. ModifyTime: time.Now(),
  259. }
  260. insertId, tmpErr := to.Insert(existItemB)
  261. if tmpErr != nil {
  262. err = tmpErr
  263. return
  264. }
  265. existItemB.EdbInfoCalculateMappingId = int(insertId)
  266. isNeedCalculateData = true
  267. } else if existItemB.MoveValue != req.EdbInfoIdArr[1].MoveValue {
  268. // 如果平移天数不一致的话,
  269. existItemB.ModifyTime = time.Now()
  270. existItemB.MoveValue = req.EdbInfoIdArr[1].MoveValue
  271. _, err = to.Update(existItemB, "ModifyTime", "MoveValue")
  272. if err != nil {
  273. return
  274. }
  275. isNeedCalculateData = true
  276. }
  277. }
  278. // 如果需要重新计算,那么先删除所有的指标数据,然后再重新计算
  279. if isNeedCalculateData {
  280. // 删除之前所有的指标数据
  281. tableName := GetEdbDataTableName(edbInfo.Source, edbInfo.SubSource)
  282. sql := fmt.Sprintf(` DELETE FROM %s WHERE edb_info_id = ? `, tableName)
  283. _, err = to.Raw(sql, edbInfo.EdbInfoId).Exec()
  284. if err != nil {
  285. err = fmt.Errorf("删除历史数据失败,Err:" + err.Error())
  286. return
  287. }
  288. } else {
  289. return
  290. }
  291. }
  292. //拼接数据
  293. latestDateStr, latestValue, err = refreshAllPredictCalculateNhcc(to, edbInfo, firstEdbInfo, secondEdbInfo, existItemA, existItemB, nhccDate)
  294. return
  295. }
  296. // RefreshAllPredictCalculateNhcc 刷新所有 拟合残差 数据
  297. func RefreshAllPredictCalculateNhcc(edbInfo *EdbInfo) (latestDateStr string, latestValue float64, err error) {
  298. o := orm.NewOrm()
  299. to, err := o.Begin()
  300. defer func() {
  301. if err != nil {
  302. fmt.Println("RefreshAllPredictCalculateNhcc,Err:" + err.Error())
  303. _ = to.Rollback()
  304. } else {
  305. _ = to.Commit()
  306. }
  307. }()
  308. //查询关联指标信息
  309. var existCondition string
  310. var existPars []interface{}
  311. existCondition += " AND edb_info_id=? "
  312. existPars = append(existPars, edbInfo.EdbInfoId)
  313. existList, err := GetEdbInfoCalculateListByCondition(existCondition, existPars)
  314. if err != nil {
  315. err = fmt.Errorf("判断指标是否改变失败,Err:" + err.Error())
  316. return
  317. }
  318. var existItemA, existItemB *EdbInfoCalculateMapping
  319. for _, existItem := range existList {
  320. if existItem.FromTag == "A" {
  321. existItemA = existItem
  322. } else if existItem.FromTag == "B" {
  323. existItemB = existItem
  324. }
  325. }
  326. if existItemA == nil {
  327. err = errors.New("原自变量指标不存在")
  328. return
  329. }
  330. if existItemB == nil {
  331. err = errors.New("原因变量指标不存在")
  332. return
  333. }
  334. fromEdbInfo, err := GetEdbInfoById(existItemA.FromEdbInfoId)
  335. if err != nil {
  336. err = fmt.Errorf("GetEdbInfoById Err:" + err.Error())
  337. return
  338. }
  339. secondEdbInfo, err := GetEdbInfoById(existItemB.FromEdbInfoId)
  340. if err != nil {
  341. err = fmt.Errorf("GetEdbInfoById Err:" + err.Error())
  342. return
  343. }
  344. timeList := strings.Split(edbInfo.CalculateFormula, ",")
  345. startDate, err := time.ParseInLocation(utils.FormatDate, timeList[0], time.Local)
  346. if err != nil {
  347. return
  348. }
  349. endDate, err := time.ParseInLocation(utils.FormatDate, timeList[1], time.Local)
  350. if err != nil {
  351. return
  352. }
  353. if endDate.Sub(startDate).Hours() < 48 {
  354. return
  355. }
  356. nhccDate := NhccDate{
  357. StartDate: startDate,
  358. EndDate: endDate,
  359. }
  360. // 刷新数据
  361. latestDateStr, latestValue, err = refreshAllPredictCalculateNhcc(to, edbInfo, fromEdbInfo, secondEdbInfo, existItemA, existItemB, nhccDate)
  362. return
  363. }
  364. // refreshAllPredictCalculateNhcc 刷新所有 拟合残差 数据
  365. func refreshAllPredictCalculateNhcc(to orm.TxOrmer, edbInfo, firstEdbInfo, secondEdbInfo *EdbInfo, existItemA, existItemB *EdbInfoCalculateMapping, nhccDate NhccDate) (latestDateStr string, latestValue float64, err error) {
  366. var dataList []*EdbDataPredictCalculateNhcc
  367. sql := ` SELECT * FROM edb_data_predict_calculate_nhcc WHERE edb_info_id=? ORDER BY data_time DESC `
  368. _, err = to.Raw(sql, edbInfo.EdbInfoId).QueryRows(&dataList)
  369. if err != nil {
  370. return
  371. }
  372. // 计算最新的实际值的日期
  373. latestDateA, _ := time.ParseInLocation(utils.FormatDate, firstEdbInfo.LatestDate, time.Local)
  374. if existItemA.MoveValue != 0 {
  375. latestDateA = latestDateA.AddDate(0, 0, existItemA.MoveValue)
  376. }
  377. latestDateB, _ := time.ParseInLocation(utils.FormatDate, secondEdbInfo.LatestDate, time.Local)
  378. if existItemB.MoveValue != 0 {
  379. latestDateB = latestDateA.AddDate(0, 0, existItemB.MoveValue)
  380. }
  381. if latestDateA.Before(latestDateB) {
  382. latestDateStr = latestDateA.Format(utils.FormatDate)
  383. } else {
  384. latestDateStr = latestDateB.Format(utils.FormatDate)
  385. }
  386. var dateArr []string
  387. dataMap := make(map[string]*EdbDataPredictCalculateNhcc)
  388. removeDataTimeMap := make(map[string]int) //需要移除的日期数据
  389. for _, v := range dataList {
  390. dateArr = append(dateArr, v.DataTime)
  391. dataMap[v.DataTime] = v
  392. removeDataTimeMap[v.DataTime] = 1
  393. }
  394. addDataList := make([]*EdbDataPredictCalculateNhcc, 0)
  395. aDataList, secondDataList, aDataMap, bDataMap, err := getPredictNhccData(firstEdbInfo, secondEdbInfo, existItemA, existItemB, nhccDate)
  396. // 计算线性方程公式
  397. var a, b float64
  398. {
  399. coordinateData := make([]utils.Coordinate, 0)
  400. for i := nhccDate.StartDate; i.Before(nhccDate.EndDate) || i.Equal(nhccDate.EndDate); i = i.AddDate(0, 0, 1) {
  401. dateStr := i.Format(utils.FormatDate)
  402. xValue, ok := aDataMap[dateStr]
  403. if !ok {
  404. err = errors.New("指标A日期:" + dateStr + "数据异常,导致计算线性方程公式失败")
  405. return
  406. }
  407. yValue, ok := bDataMap[dateStr]
  408. if !ok {
  409. err = errors.New("指标B日期:" + dateStr + "数据异常,导致计算线性方程公式失败")
  410. return
  411. }
  412. tmpCoordinate := utils.Coordinate{
  413. X: xValue,
  414. Y: yValue,
  415. }
  416. coordinateData = append(coordinateData, tmpCoordinate)
  417. }
  418. a, b = utils.GetLinearResult(coordinateData)
  419. }
  420. if math.IsNaN(a) || math.IsNaN(b) {
  421. err = errors.New("线性方程公式生成失败")
  422. return
  423. }
  424. //fmt.Println("a:", a, ";======b:", b)
  425. //计算B’
  426. newBDataMap := make(map[string]float64)
  427. {
  428. //B’=aA+b
  429. aDecimal := decimal.NewFromFloat(a)
  430. bDecimal := decimal.NewFromFloat(b)
  431. for _, aData := range aDataList {
  432. xDecimal := decimal.NewFromFloat(aData.Value)
  433. val, _ := aDecimal.Mul(xDecimal).Add(bDecimal).RoundCeil(4).Float64()
  434. newBDataMap[aData.DataTime] = val
  435. }
  436. }
  437. // Delta=B-B‘
  438. for _, bData := range secondDataList {
  439. currDate := bData.DataTime
  440. //校验待删除日期数据里面是否存在该元素,如果存在的话,那么移除该元素
  441. if _, ok := removeDataTimeMap[currDate]; ok {
  442. delete(removeDataTimeMap, currDate)
  443. }
  444. b2Val, ok := newBDataMap[currDate]
  445. if !ok {
  446. // 如果B`不存在数据,那么就退出当前循环
  447. continue
  448. }
  449. bDecimal := decimal.NewFromFloat(bData.Value)
  450. b2Decimal := decimal.NewFromFloat(b2Val)
  451. val, _ := bDecimal.Sub(b2Decimal).RoundCeil(4).Float64()
  452. // 判断之前有没有该数据
  453. existData, ok := dataMap[currDate]
  454. if !ok { //不存在那么就添加吧
  455. currentDate, _ := time.ParseInLocation(utils.FormatDate, currDate, time.Local)
  456. timestamp := currentDate.UnixNano() / 1e6
  457. edbDataNhcc := &EdbDataPredictCalculateNhcc{
  458. EdbInfoId: edbInfo.EdbInfoId,
  459. EdbCode: edbInfo.EdbCode,
  460. DataTime: currDate,
  461. Value: val,
  462. Status: 1,
  463. CreateTime: time.Now(),
  464. ModifyTime: time.Now(),
  465. DataTimestamp: timestamp,
  466. }
  467. addDataList = append(addDataList, edbDataNhcc)
  468. } else {
  469. // 如果有的话,还需要判断值是否一致,一致则不处理,不一致则修改
  470. if existData.Value != val {
  471. existData.Value = val
  472. _, err = to.Update(existData, "Value")
  473. if err != nil {
  474. return
  475. }
  476. }
  477. }
  478. }
  479. //删除已经不存在的拟合残差指标数据(由于该指标当日的数据删除了)
  480. {
  481. removeDateList := make([]string, 0)
  482. for dateTime := range removeDataTimeMap {
  483. removeDateList = append(removeDateList, dateTime)
  484. }
  485. removeNum := len(removeDateList)
  486. if removeNum > 0 {
  487. //如果拼接指标变更了,那么需要删除所有的指标数据
  488. tableName := GetEdbDataTableName(edbInfo.Source, edbInfo.SubSource)
  489. sql := fmt.Sprintf(` DELETE FROM %s WHERE edb_info_id = ? and data_time in (`+utils.GetOrmInReplace(removeNum)+`) `, tableName)
  490. _, err = to.Raw(sql, edbInfo.EdbInfoId, removeDateList).Exec()
  491. if err != nil {
  492. err = fmt.Errorf("删除不存在的拟合残差指标数据失败,Err:" + err.Error())
  493. return
  494. }
  495. }
  496. }
  497. //数据入库
  498. if len(addDataList) > 0 {
  499. tmpAddDataList := make([]*EdbDataPredictCalculateNhcc, 0)
  500. i := 0
  501. for _, v := range addDataList {
  502. tmpAddDataList = append(tmpAddDataList, v)
  503. i++
  504. if i >= 500 {
  505. _, err = to.InsertMulti(len(tmpAddDataList), tmpAddDataList)
  506. if err != nil {
  507. return
  508. }
  509. i = 0
  510. tmpAddDataList = make([]*EdbDataPredictCalculateNhcc, 0)
  511. }
  512. }
  513. if len(tmpAddDataList) > 0 {
  514. _, err = to.InsertMulti(len(tmpAddDataList), tmpAddDataList)
  515. if err != nil {
  516. return
  517. }
  518. }
  519. }
  520. //确定最终值
  521. var finalLast EdbInfoSearchData
  522. sql = ` SELECT data_time , value FROM edb_data_predict_calculate_nhcc WHERE edb_info_id=? and data_time<=? ORDER BY data_time DESC `
  523. tmpErr := to.Raw(sql, edbInfo.EdbInfoId, latestDateStr).QueryRow(&finalLast)
  524. if tmpErr != nil {
  525. if tmpErr.Error() != utils.ErrNoRow() {
  526. err = tmpErr
  527. }
  528. return
  529. } else {
  530. latestDateStr = finalLast.DataTime
  531. latestValue = finalLast.Value
  532. }
  533. return
  534. }
  535. // CalculatePredictComputeCorrelation 计算相关性结果
  536. func CalculatePredictComputeCorrelation(req *EdbInfoCalculateBatchSaveReq, firstEdbInfo, secondEdbInfo *EdbInfo, nhccDate NhccDate) (r float64, err error) {
  537. var existItemA, existItemB *EdbInfoCalculateMapping
  538. //第一个指标
  539. {
  540. existItemA = &EdbInfoCalculateMapping{
  541. EdbInfoCalculateMappingId: 0,
  542. FromEdbInfoId: firstEdbInfo.EdbInfoId,
  543. FromEdbCode: firstEdbInfo.EdbCode,
  544. FromEdbName: firstEdbInfo.EdbName,
  545. FromSource: firstEdbInfo.Source,
  546. FromSourceName: firstEdbInfo.SourceName,
  547. FromTag: "A",
  548. MoveValue: req.EdbInfoIdArr[0].MoveValue,
  549. Sort: 1,
  550. CreateTime: time.Now(),
  551. ModifyTime: time.Now(),
  552. }
  553. }
  554. //第二个指标
  555. {
  556. existItemB = &EdbInfoCalculateMapping{
  557. EdbInfoCalculateMappingId: 0,
  558. FromEdbInfoId: secondEdbInfo.EdbInfoId,
  559. FromEdbCode: secondEdbInfo.EdbCode,
  560. FromEdbName: secondEdbInfo.EdbName,
  561. FromSource: secondEdbInfo.Source,
  562. FromSourceName: secondEdbInfo.SourceName,
  563. FromTag: "B",
  564. MoveValue: req.EdbInfoIdArr[1].MoveValue,
  565. Sort: 1,
  566. CreateTime: time.Now(),
  567. ModifyTime: time.Now(),
  568. }
  569. }
  570. o := orm.NewOrm()
  571. to, err := o.Begin()
  572. defer func() {
  573. if err != nil {
  574. fmt.Println("RefreshAllCalculateNhcc,Err:" + err.Error())
  575. _ = to.Rollback()
  576. } else {
  577. _ = to.Commit()
  578. }
  579. }()
  580. // 获取相关数据
  581. _, _, aDataMap, bDataMap, err := getPredictNhccData(firstEdbInfo, secondEdbInfo, existItemA, existItemB, nhccDate)
  582. // 计算相关性
  583. coordinateData := make([]utils.Coordinate, 0)
  584. for i := nhccDate.StartDate; i.Before(nhccDate.EndDate) || i.Equal(nhccDate.EndDate); i = i.AddDate(0, 0, 1) {
  585. dateStr := i.Format(utils.FormatDate)
  586. xValue, ok := aDataMap[dateStr]
  587. if !ok {
  588. err = errors.New("指标A日期:" + dateStr + "数据异常,导致计算线性方程公式失败")
  589. return
  590. }
  591. yValue, ok := bDataMap[dateStr]
  592. if !ok {
  593. err = errors.New("指标B日期:" + dateStr + "数据异常,导致计算线性方程公式失败")
  594. return
  595. }
  596. tmpCoordinate := utils.Coordinate{
  597. X: xValue,
  598. Y: yValue,
  599. }
  600. coordinateData = append(coordinateData, tmpCoordinate)
  601. }
  602. r = utils.ComputeCorrelation(coordinateData)
  603. return
  604. }
  605. // getNhccData 获取拟合残差需要的数据
  606. func getPredictNhccData(firstEdbInfo, secondEdbInfo *EdbInfo, existItemA, existItemB *EdbInfoCalculateMapping, nhccDate NhccDate) (aDataList []EdbInfoSearchData, secondDataList []*EdbInfoSearchData, aDataMap, bDataMap map[string]float64, err error) {
  607. //第一个指标
  608. aDataList = make([]EdbInfoSearchData, 0)
  609. aDataMap = make(map[string]float64)
  610. {
  611. /*var condition string
  612. var pars []interface{}
  613. condition += " AND edb_info_id=? "
  614. pars = append(pars, existItemA.FromEdbInfoId)
  615. //第一个指标的数据列表
  616. firstDataList, tmpErr := GetEdbDataListAllByTo(to, condition, pars, existItemA.FromSource, 0)
  617. if tmpErr != nil {
  618. return tmpErr
  619. }*/
  620. var firstDataList []*EdbInfoSearchData
  621. firstDataList, err = GetPredictEdbDataListAllByStartDate(firstEdbInfo, 0, "")
  622. if err != nil {
  623. return
  624. }
  625. aDataList, aDataMap = handleNhccData(firstDataList, existItemA.MoveValue)
  626. }
  627. //第二个指标
  628. bDataList := make([]EdbInfoSearchData, 0)
  629. secondDataList = make([]*EdbInfoSearchData, 0)
  630. bDataMap = make(map[string]float64)
  631. {
  632. /*condition = ``
  633. pars = make([]interface{}, 0)
  634. condition += " AND edb_info_id = ? "
  635. pars = append(pars, existItemB.FromEdbInfoId)
  636. //第二个指标的数据列表
  637. secondDataList, err = GetEdbDataListAllByTo(to, condition, pars, existItemB.FromSource, 0)
  638. if err != nil {
  639. return
  640. }*/
  641. secondDataList, err = GetPredictEdbDataListAllByStartDate(secondEdbInfo, 0, "")
  642. if err != nil {
  643. return
  644. }
  645. bDataList, bDataMap = handleNhccData(secondDataList, existItemB.MoveValue)
  646. }
  647. if len(aDataList) <= 0 {
  648. err = errors.New("指标A没有数据")
  649. return
  650. }
  651. if len(bDataList) <= 0 {
  652. err = errors.New("指标B没有数据")
  653. return
  654. }
  655. // 拟合残差计算的结束日期判断
  656. {
  657. endAData := aDataList[len(aDataList)-1]
  658. tmpEndDate, tmpErr := time.ParseInLocation(utils.FormatDate, endAData.DataTime, time.Local)
  659. if tmpErr != nil {
  660. err = tmpErr
  661. return
  662. }
  663. // 如果A指标的最新数据日期早于拟合残差的结束日期,那么就用A指标的最新数据日期
  664. if tmpEndDate.Before(nhccDate.EndDate) {
  665. nhccDate.EndDate = tmpEndDate
  666. }
  667. endBData := bDataList[len(bDataList)-1]
  668. tmpEndDate, tmpErr = time.ParseInLocation(utils.FormatDate, endBData.DataTime, time.Local)
  669. if tmpErr != nil {
  670. err = tmpErr
  671. return
  672. }
  673. // 如果B指标的最新数据日期早于拟合残差的结束日期,那么就用B指标的最新数据日期
  674. if tmpEndDate.Before(nhccDate.EndDate) {
  675. nhccDate.EndDate = tmpEndDate
  676. }
  677. }
  678. return
  679. }