predict_edb_data_calculate_nhcc.go 23 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739
  1. package models
  2. import (
  3. "errors"
  4. "eta/eta_index_lib/utils"
  5. "fmt"
  6. "github.com/beego/beego/v2/client/orm"
  7. "github.com/shopspring/decimal"
  8. "math"
  9. "strings"
  10. "time"
  11. )
  12. // EdbDataPredictCalculateNhcc 拟合残差数据结构体
  13. type EdbDataPredictCalculateNhcc struct {
  14. EdbDataId int `orm:"column(edb_data_id);pk"`
  15. EdbInfoId int
  16. EdbCode string
  17. DataTime string
  18. Value float64
  19. Status int
  20. CreateTime time.Time
  21. ModifyTime time.Time
  22. DataTimestamp int64
  23. }
  24. // SavePredictCalculateNhcc 新增拟合残差数据
  25. func SavePredictCalculateNhcc(req *EdbInfoCalculateBatchSaveReq, firstEdbInfo, secondEdbInfo *EdbInfo, edbCode, uniqueCode string, nhccDate NhccDate, sysUserId int, sysUserRealName, lang string) (edbInfo *EdbInfo, latestDateStr string, latestValue float64, err error) {
  26. o := orm.NewOrm()
  27. to, err := o.Begin()
  28. if err != nil {
  29. return
  30. }
  31. defer func() {
  32. if err != nil {
  33. fmt.Println("SavePredictCalculateNhcc,Err:" + err.Error())
  34. _ = to.Rollback()
  35. } else {
  36. _ = to.Commit()
  37. }
  38. }()
  39. var existItemA, existItemB *EdbInfoCalculateMapping
  40. if req.EdbInfoId <= 0 {
  41. edbInfo = &EdbInfo{
  42. EdbInfoType: 1,
  43. SourceName: "预测拟合残差",
  44. Source: utils.DATA_SOURCE_PREDICT_CALCULATE_NHCC,
  45. EdbCode: edbCode,
  46. EdbName: req.EdbName,
  47. EdbNameSource: req.EdbName,
  48. Frequency: req.Frequency,
  49. Unit: req.Unit,
  50. StartDate: firstEdbInfo.StartDate,
  51. EndDate: firstEdbInfo.EndDate,
  52. ClassifyId: req.ClassifyId,
  53. SysUserId: sysUserId,
  54. SysUserRealName: sysUserRealName,
  55. UniqueCode: uniqueCode,
  56. CreateTime: time.Now(),
  57. ModifyTime: time.Now(),
  58. CalculateFormula: req.Formula,
  59. EdbNameEn: req.EdbName,
  60. UnitEn: req.Unit,
  61. EdbType: 2,
  62. Sort: GetAddEdbMaxSortByClassifyId(req.ClassifyId, utils.PREDICT_EDB_INFO_TYPE),
  63. }
  64. newEdbInfoId, tmpErr := to.Insert(edbInfo)
  65. if tmpErr != nil {
  66. err = tmpErr
  67. return
  68. }
  69. edbInfo.EdbInfoId = int(newEdbInfoId)
  70. //第一个指标
  71. {
  72. existItemA = &EdbInfoCalculateMapping{
  73. EdbInfoCalculateMappingId: 0,
  74. EdbInfoId: edbInfo.EdbInfoId,
  75. Source: edbInfo.Source,
  76. SourceName: edbInfo.SourceName,
  77. EdbCode: edbInfo.EdbCode,
  78. FromEdbInfoId: firstEdbInfo.EdbInfoId,
  79. FromEdbCode: firstEdbInfo.EdbCode,
  80. FromEdbName: firstEdbInfo.EdbName,
  81. FromSource: firstEdbInfo.Source,
  82. FromSourceName: firstEdbInfo.SourceName,
  83. FromTag: "A",
  84. MoveValue: req.EdbInfoIdArr[0].MoveValue,
  85. Sort: 1,
  86. CreateTime: time.Now(),
  87. ModifyTime: time.Now(),
  88. }
  89. insertId, tmpErr := to.Insert(existItemA)
  90. if tmpErr != nil {
  91. err = tmpErr
  92. return
  93. }
  94. existItemA.EdbInfoCalculateMappingId = int(insertId)
  95. }
  96. //第二个指标
  97. {
  98. existItemB = &EdbInfoCalculateMapping{
  99. EdbInfoCalculateMappingId: 0,
  100. EdbInfoId: edbInfo.EdbInfoId,
  101. Source: edbInfo.Source,
  102. SourceName: edbInfo.SourceName,
  103. EdbCode: edbInfo.EdbCode,
  104. FromEdbInfoId: secondEdbInfo.EdbInfoId,
  105. FromEdbCode: secondEdbInfo.EdbCode,
  106. FromEdbName: secondEdbInfo.EdbName,
  107. FromSource: secondEdbInfo.Source,
  108. FromSourceName: secondEdbInfo.SourceName,
  109. FromTag: "B",
  110. MoveValue: req.EdbInfoIdArr[1].MoveValue,
  111. Sort: 1,
  112. CreateTime: time.Now(),
  113. ModifyTime: time.Now(),
  114. }
  115. insertId, tmpErr := to.Insert(existItemB)
  116. if tmpErr != nil {
  117. err = tmpErr
  118. return
  119. }
  120. existItemB.EdbInfoCalculateMappingId = int(insertId)
  121. }
  122. } else {
  123. edbInfo, err = GetEdbInfoById(req.EdbInfoId)
  124. if err != nil {
  125. return
  126. }
  127. latestDateStr = edbInfo.LatestDate
  128. latestValue = edbInfo.LatestValue
  129. nowEdbInfo := *edbInfo // 现在的指标信息
  130. //修改指标信息
  131. switch lang {
  132. case utils.EnLangVersion:
  133. edbInfo.EdbNameEn = req.EdbName
  134. edbInfo.UnitEn = req.Unit
  135. default:
  136. edbInfo.EdbName = req.EdbName
  137. edbInfo.Unit = req.Unit
  138. edbInfo.EdbNameSource = req.EdbName
  139. }
  140. edbInfo.Frequency = req.Frequency
  141. edbInfo.ClassifyId = req.ClassifyId
  142. edbInfo.CalculateFormula = req.Formula
  143. edbInfo.ModifyTime = time.Now()
  144. _, err = to.Update(edbInfo, "EdbName", "EdbNameSource", "Frequency", "Unit", "ClassifyId", "CalculateFormula", "ModifyTime", "EdbNameEn", "UnitEn")
  145. if err != nil {
  146. return
  147. }
  148. var existCondition string
  149. var existPars []interface{}
  150. existCondition += " AND edb_info_id=? "
  151. existPars = append(existPars, edbInfo.EdbInfoId)
  152. //查询出所有的关联指标
  153. var existList []*EdbInfoCalculateMapping
  154. existList, err = GetEdbInfoCalculateListByCondition(existCondition, existPars)
  155. if err != nil {
  156. err = fmt.Errorf("判断指标是否改变失败,Err:" + err.Error())
  157. return
  158. }
  159. for _, existItem := range existList {
  160. if existItem.FromTag == "A" {
  161. existItemA = existItem
  162. } else if existItem.FromTag == "B" {
  163. existItemB = existItem
  164. }
  165. }
  166. if existItemA == nil {
  167. err = errors.New("原自变量指标不存在")
  168. return
  169. }
  170. if existItemB == nil {
  171. err = errors.New("原因变量指标不存在")
  172. return
  173. }
  174. // 是否需要删除数据重新计算
  175. isNeedCalculateData := false
  176. // 如果截止日期变更,那么需要重新计算
  177. if req.Formula != nowEdbInfo.CalculateFormula {
  178. isNeedCalculateData = true
  179. }
  180. var isDeleteA, isDeleteB bool
  181. // 如果指标变了,那么需要删除关系
  182. {
  183. if existItemA.FromEdbInfoId != firstEdbInfo.EdbInfoId {
  184. //删除之前的A指标关联关系
  185. sql := ` DELETE FROM edb_info_calculate_mapping WHERE edb_info_id = ? and from_edb_info_id = ?`
  186. _, err = to.Raw(sql, edbInfo.EdbInfoId, existItemA.FromEdbInfoId).Exec()
  187. if err != nil {
  188. err = fmt.Errorf("删除指标A关联关系失败,Err:" + err.Error())
  189. return
  190. }
  191. isDeleteA = true
  192. }
  193. //并重新计算
  194. if existItemB.FromEdbInfoId != secondEdbInfo.EdbInfoId {
  195. //删除之前的B指标关联关系
  196. sql := ` DELETE FROM edb_info_calculate_mapping WHERE edb_info_id = ? and from_edb_info_id = ?`
  197. _, err = to.Raw(sql, edbInfo.EdbInfoId, existItemB.FromEdbInfoId).Exec()
  198. if err != nil {
  199. err = fmt.Errorf("删除指标B关联关系失败,Err:" + err.Error())
  200. return
  201. }
  202. isDeleteB = true
  203. }
  204. }
  205. //第一个指标数据
  206. {
  207. // 如果指标变了,那么需要删除关系,并重新计算
  208. if isDeleteA {
  209. //添加新的指标关系
  210. {
  211. existItemA = &EdbInfoCalculateMapping{
  212. EdbInfoCalculateMappingId: 0,
  213. EdbInfoId: edbInfo.EdbInfoId,
  214. Source: edbInfo.Source,
  215. SourceName: edbInfo.SourceName,
  216. EdbCode: edbInfo.EdbCode,
  217. FromEdbInfoId: firstEdbInfo.EdbInfoId,
  218. FromEdbCode: firstEdbInfo.EdbCode,
  219. FromEdbName: firstEdbInfo.EdbName,
  220. FromSource: firstEdbInfo.Source,
  221. FromSourceName: firstEdbInfo.SourceName,
  222. FromTag: "A",
  223. MoveValue: req.EdbInfoIdArr[0].MoveValue,
  224. Sort: 1,
  225. CreateTime: time.Now(),
  226. ModifyTime: time.Now(),
  227. }
  228. insertId, tmpErr := to.Insert(existItemA)
  229. if tmpErr != nil {
  230. err = tmpErr
  231. return
  232. }
  233. existItemA.EdbInfoCalculateMappingId = int(insertId)
  234. isNeedCalculateData = true
  235. }
  236. } else if existItemA.MoveValue != req.EdbInfoIdArr[0].MoveValue {
  237. // 如果平移天数不一致的话,
  238. existItemA.ModifyTime = time.Now()
  239. existItemA.MoveValue = req.EdbInfoIdArr[0].MoveValue
  240. _, err = to.Update(existItemA, "ModifyTime", "MoveValue")
  241. if err != nil {
  242. return
  243. }
  244. isNeedCalculateData = true
  245. }
  246. }
  247. //第二个指标数据
  248. {
  249. // 如果指标变了,那么需要删除关系,并重新计算
  250. if isDeleteB {
  251. // 添加新的指标关联关系
  252. existItemB = &EdbInfoCalculateMapping{
  253. EdbInfoCalculateMappingId: 0,
  254. EdbInfoId: edbInfo.EdbInfoId,
  255. Source: edbInfo.Source,
  256. SourceName: edbInfo.SourceName,
  257. EdbCode: edbInfo.EdbCode,
  258. FromEdbInfoId: secondEdbInfo.EdbInfoId,
  259. FromEdbCode: secondEdbInfo.EdbCode,
  260. FromEdbName: secondEdbInfo.EdbName,
  261. FromSource: secondEdbInfo.Source,
  262. FromSourceName: secondEdbInfo.SourceName,
  263. FromTag: "B",
  264. MoveValue: req.EdbInfoIdArr[1].MoveValue,
  265. Sort: 2,
  266. CreateTime: time.Now(),
  267. ModifyTime: time.Now(),
  268. }
  269. insertId, tmpErr := to.Insert(existItemB)
  270. if tmpErr != nil {
  271. err = tmpErr
  272. return
  273. }
  274. existItemB.EdbInfoCalculateMappingId = int(insertId)
  275. isNeedCalculateData = true
  276. } else if existItemB.MoveValue != req.EdbInfoIdArr[1].MoveValue {
  277. // 如果平移天数不一致的话,
  278. existItemB.ModifyTime = time.Now()
  279. existItemB.MoveValue = req.EdbInfoIdArr[1].MoveValue
  280. _, err = to.Update(existItemB, "ModifyTime", "MoveValue")
  281. if err != nil {
  282. return
  283. }
  284. isNeedCalculateData = true
  285. }
  286. }
  287. // 如果需要重新计算,那么先删除所有的指标数据,然后再重新计算
  288. if isNeedCalculateData {
  289. // 删除之前所有的指标数据
  290. tableName := GetEdbDataTableName(edbInfo.Source, edbInfo.SubSource)
  291. sql := fmt.Sprintf(` DELETE FROM %s WHERE edb_info_id = ? `, tableName)
  292. _, err = to.Raw(sql, edbInfo.EdbInfoId).Exec()
  293. if err != nil {
  294. err = fmt.Errorf("删除历史数据失败,Err:" + err.Error())
  295. return
  296. }
  297. } else {
  298. return
  299. }
  300. }
  301. //拼接数据
  302. latestDateStr, latestValue, err = refreshAllPredictCalculateNhcc(to, edbInfo, firstEdbInfo, secondEdbInfo, existItemA, existItemB, nhccDate)
  303. return
  304. }
  305. // RefreshAllPredictCalculateNhcc 刷新所有 拟合残差 数据
  306. func RefreshAllPredictCalculateNhcc(edbInfo *EdbInfo) (latestDateStr string, latestValue float64, err error) {
  307. o := orm.NewOrm()
  308. to, err := o.Begin()
  309. defer func() {
  310. if err != nil {
  311. fmt.Println("RefreshAllPredictCalculateNhcc,Err:" + err.Error())
  312. _ = to.Rollback()
  313. } else {
  314. _ = to.Commit()
  315. }
  316. }()
  317. //查询关联指标信息
  318. var existCondition string
  319. var existPars []interface{}
  320. existCondition += " AND edb_info_id=? "
  321. existPars = append(existPars, edbInfo.EdbInfoId)
  322. existList, err := GetEdbInfoCalculateListByCondition(existCondition, existPars)
  323. if err != nil {
  324. err = fmt.Errorf("判断指标是否改变失败,Err:" + err.Error())
  325. return
  326. }
  327. var existItemA, existItemB *EdbInfoCalculateMapping
  328. for _, existItem := range existList {
  329. if existItem.FromTag == "A" {
  330. existItemA = existItem
  331. } else if existItem.FromTag == "B" {
  332. existItemB = existItem
  333. }
  334. }
  335. if existItemA == nil {
  336. err = errors.New("原自变量指标不存在")
  337. return
  338. }
  339. if existItemB == nil {
  340. err = errors.New("原因变量指标不存在")
  341. return
  342. }
  343. fromEdbInfo, err := GetEdbInfoById(existItemA.FromEdbInfoId)
  344. if err != nil {
  345. err = fmt.Errorf("GetEdbInfoById Err:" + err.Error())
  346. return
  347. }
  348. secondEdbInfo, err := GetEdbInfoById(existItemB.FromEdbInfoId)
  349. if err != nil {
  350. err = fmt.Errorf("GetEdbInfoById Err:" + err.Error())
  351. return
  352. }
  353. timeList := strings.Split(edbInfo.CalculateFormula, ",")
  354. startDate, err := time.ParseInLocation(utils.FormatDate, timeList[0], time.Local)
  355. if err != nil {
  356. return
  357. }
  358. endDate, err := time.ParseInLocation(utils.FormatDate, timeList[1], time.Local)
  359. if err != nil {
  360. return
  361. }
  362. if endDate.Sub(startDate).Hours() < 48 {
  363. return
  364. }
  365. nhccDate := NhccDate{
  366. StartDate: startDate,
  367. EndDate: endDate,
  368. }
  369. // 刷新数据
  370. latestDateStr, latestValue, err = refreshAllPredictCalculateNhcc(to, edbInfo, fromEdbInfo, secondEdbInfo, existItemA, existItemB, nhccDate)
  371. return
  372. }
  373. // refreshAllPredictCalculateNhcc 刷新所有 拟合残差 数据
  374. func refreshAllPredictCalculateNhcc(to orm.TxOrmer, edbInfo, firstEdbInfo, secondEdbInfo *EdbInfo, existItemA, existItemB *EdbInfoCalculateMapping, nhccDate NhccDate) (latestDateStr string, latestValue float64, err error) {
  375. var dataList []*EdbDataPredictCalculateNhcc
  376. sql := ` SELECT * FROM edb_data_predict_calculate_nhcc WHERE edb_info_id=? ORDER BY data_time DESC `
  377. _, err = to.Raw(sql, edbInfo.EdbInfoId).QueryRows(&dataList)
  378. if err != nil {
  379. return
  380. }
  381. // 计算最新的实际值的日期
  382. latestDateA, _ := time.ParseInLocation(utils.FormatDate, firstEdbInfo.LatestDate, time.Local)
  383. if existItemA.MoveValue != 0 {
  384. latestDateA = latestDateA.AddDate(0, 0, existItemA.MoveValue)
  385. }
  386. latestDateB, _ := time.ParseInLocation(utils.FormatDate, secondEdbInfo.LatestDate, time.Local)
  387. if existItemB.MoveValue != 0 {
  388. latestDateB = latestDateA.AddDate(0, 0, existItemB.MoveValue)
  389. }
  390. if latestDateA.Before(latestDateB) {
  391. latestDateStr = latestDateA.Format(utils.FormatDate)
  392. } else {
  393. latestDateStr = latestDateB.Format(utils.FormatDate)
  394. }
  395. var dateArr []string
  396. dataMap := make(map[string]*EdbDataPredictCalculateNhcc)
  397. removeDataTimeMap := make(map[string]int) //需要移除的日期数据
  398. for _, v := range dataList {
  399. dateArr = append(dateArr, v.DataTime)
  400. dataMap[v.DataTime] = v
  401. removeDataTimeMap[v.DataTime] = 1
  402. }
  403. addDataList := make([]*EdbDataPredictCalculateNhcc, 0)
  404. aDataList, secondDataList, aDataMap, bDataMap, err := getPredictNhccData(firstEdbInfo, secondEdbInfo, existItemA, existItemB, nhccDate)
  405. // 计算线性方程公式
  406. var a, b float64
  407. {
  408. coordinateData := make([]utils.Coordinate, 0)
  409. for i := nhccDate.StartDate; i.Before(nhccDate.EndDate) || i.Equal(nhccDate.EndDate); i = i.AddDate(0, 0, 1) {
  410. dateStr := i.Format(utils.FormatDate)
  411. xValue, ok := aDataMap[dateStr]
  412. if !ok {
  413. err = errors.New("指标A日期:" + dateStr + "数据异常,导致计算线性方程公式失败")
  414. return
  415. }
  416. yValue, ok := bDataMap[dateStr]
  417. if !ok {
  418. err = errors.New("指标B日期:" + dateStr + "数据异常,导致计算线性方程公式失败")
  419. return
  420. }
  421. tmpCoordinate := utils.Coordinate{
  422. X: xValue,
  423. Y: yValue,
  424. }
  425. coordinateData = append(coordinateData, tmpCoordinate)
  426. }
  427. a, b = utils.GetLinearResult(coordinateData)
  428. }
  429. if math.IsNaN(a) || math.IsNaN(b) {
  430. err = errors.New("线性方程公式生成失败")
  431. return
  432. }
  433. //fmt.Println("a:", a, ";======b:", b)
  434. //计算B’
  435. newBDataMap := make(map[string]float64)
  436. {
  437. //B’=aA+b
  438. aDecimal := decimal.NewFromFloat(a)
  439. bDecimal := decimal.NewFromFloat(b)
  440. for _, aData := range aDataList {
  441. xDecimal := decimal.NewFromFloat(aData.Value)
  442. val, _ := aDecimal.Mul(xDecimal).Add(bDecimal).RoundCeil(4).Float64()
  443. newBDataMap[aData.DataTime] = val
  444. }
  445. }
  446. // Delta=B-B‘
  447. for _, bData := range secondDataList {
  448. currDate := bData.DataTime
  449. //校验待删除日期数据里面是否存在该元素,如果存在的话,那么移除该元素
  450. if _, ok := removeDataTimeMap[currDate]; ok {
  451. delete(removeDataTimeMap, currDate)
  452. }
  453. b2Val, ok := newBDataMap[currDate]
  454. if !ok {
  455. // 如果B`不存在数据,那么就退出当前循环
  456. continue
  457. }
  458. bDecimal := decimal.NewFromFloat(bData.Value)
  459. b2Decimal := decimal.NewFromFloat(b2Val)
  460. val, _ := bDecimal.Sub(b2Decimal).RoundCeil(4).Float64()
  461. // 判断之前有没有该数据
  462. existData, ok := dataMap[currDate]
  463. if !ok { //不存在那么就添加吧
  464. currentDate, _ := time.ParseInLocation(utils.FormatDate, currDate, time.Local)
  465. timestamp := currentDate.UnixNano() / 1e6
  466. edbDataNhcc := &EdbDataPredictCalculateNhcc{
  467. EdbInfoId: edbInfo.EdbInfoId,
  468. EdbCode: edbInfo.EdbCode,
  469. DataTime: currDate,
  470. Value: val,
  471. Status: 1,
  472. CreateTime: time.Now(),
  473. ModifyTime: time.Now(),
  474. DataTimestamp: timestamp,
  475. }
  476. addDataList = append(addDataList, edbDataNhcc)
  477. } else {
  478. // 如果有的话,还需要判断值是否一致,一致则不处理,不一致则修改
  479. if existData.Value != val {
  480. existData.Value = val
  481. _, err = to.Update(existData, "Value")
  482. if err != nil {
  483. return
  484. }
  485. }
  486. }
  487. }
  488. //删除已经不存在的拟合残差指标数据(由于该指标当日的数据删除了)
  489. {
  490. removeDateList := make([]string, 0)
  491. for dateTime := range removeDataTimeMap {
  492. removeDateList = append(removeDateList, dateTime)
  493. }
  494. removeNum := len(removeDateList)
  495. if removeNum > 0 {
  496. //如果拼接指标变更了,那么需要删除所有的指标数据
  497. tableName := GetEdbDataTableName(edbInfo.Source, edbInfo.SubSource)
  498. sql := fmt.Sprintf(` DELETE FROM %s WHERE edb_info_id = ? and data_time in (`+utils.GetOrmInReplace(removeNum)+`) `, tableName)
  499. _, err = to.Raw(sql, edbInfo.EdbInfoId, removeDateList).Exec()
  500. if err != nil {
  501. err = fmt.Errorf("删除不存在的拟合残差指标数据失败,Err:" + err.Error())
  502. return
  503. }
  504. }
  505. }
  506. //数据入库
  507. if len(addDataList) > 0 {
  508. tmpAddDataList := make([]*EdbDataPredictCalculateNhcc, 0)
  509. i := 0
  510. for _, v := range addDataList {
  511. tmpAddDataList = append(tmpAddDataList, v)
  512. i++
  513. if i >= 500 {
  514. _, err = to.InsertMulti(len(tmpAddDataList), tmpAddDataList)
  515. if err != nil {
  516. return
  517. }
  518. i = 0
  519. tmpAddDataList = make([]*EdbDataPredictCalculateNhcc, 0)
  520. }
  521. }
  522. if len(tmpAddDataList) > 0 {
  523. _, err = to.InsertMulti(len(tmpAddDataList), tmpAddDataList)
  524. if err != nil {
  525. return
  526. }
  527. }
  528. }
  529. //确定最终值
  530. var finalLast EdbInfoSearchData
  531. sql = ` SELECT data_time , value FROM edb_data_predict_calculate_nhcc WHERE edb_info_id=? and data_time<=? ORDER BY data_time DESC `
  532. tmpErr := to.Raw(sql, edbInfo.EdbInfoId, latestDateStr).QueryRow(&finalLast)
  533. if tmpErr != nil {
  534. if tmpErr.Error() != utils.ErrNoRow() {
  535. err = tmpErr
  536. }
  537. return
  538. } else {
  539. latestDateStr = finalLast.DataTime
  540. latestValue = finalLast.Value
  541. }
  542. return
  543. }
  544. // CalculatePredictComputeCorrelation 计算相关性结果
  545. func CalculatePredictComputeCorrelation(req *EdbInfoCalculateBatchSaveReq, firstEdbInfo, secondEdbInfo *EdbInfo, nhccDate NhccDate) (r float64, err error) {
  546. var existItemA, existItemB *EdbInfoCalculateMapping
  547. //第一个指标
  548. {
  549. existItemA = &EdbInfoCalculateMapping{
  550. EdbInfoCalculateMappingId: 0,
  551. FromEdbInfoId: firstEdbInfo.EdbInfoId,
  552. FromEdbCode: firstEdbInfo.EdbCode,
  553. FromEdbName: firstEdbInfo.EdbName,
  554. FromSource: firstEdbInfo.Source,
  555. FromSourceName: firstEdbInfo.SourceName,
  556. FromTag: "A",
  557. MoveValue: req.EdbInfoIdArr[0].MoveValue,
  558. Sort: 1,
  559. CreateTime: time.Now(),
  560. ModifyTime: time.Now(),
  561. }
  562. }
  563. //第二个指标
  564. {
  565. existItemB = &EdbInfoCalculateMapping{
  566. EdbInfoCalculateMappingId: 0,
  567. FromEdbInfoId: secondEdbInfo.EdbInfoId,
  568. FromEdbCode: secondEdbInfo.EdbCode,
  569. FromEdbName: secondEdbInfo.EdbName,
  570. FromSource: secondEdbInfo.Source,
  571. FromSourceName: secondEdbInfo.SourceName,
  572. FromTag: "B",
  573. MoveValue: req.EdbInfoIdArr[1].MoveValue,
  574. Sort: 1,
  575. CreateTime: time.Now(),
  576. ModifyTime: time.Now(),
  577. }
  578. }
  579. o := orm.NewOrm()
  580. to, err := o.Begin()
  581. defer func() {
  582. if err != nil {
  583. fmt.Println("RefreshAllCalculateNhcc,Err:" + err.Error())
  584. _ = to.Rollback()
  585. } else {
  586. _ = to.Commit()
  587. }
  588. }()
  589. // 获取相关数据
  590. _, _, aDataMap, bDataMap, err := getPredictNhccData(firstEdbInfo, secondEdbInfo, existItemA, existItemB, nhccDate)
  591. // 计算相关性
  592. coordinateData := make([]utils.Coordinate, 0)
  593. for i := nhccDate.StartDate; i.Before(nhccDate.EndDate) || i.Equal(nhccDate.EndDate); i = i.AddDate(0, 0, 1) {
  594. dateStr := i.Format(utils.FormatDate)
  595. xValue, ok := aDataMap[dateStr]
  596. if !ok {
  597. err = errors.New("指标A日期:" + dateStr + "数据异常,导致计算线性方程公式失败")
  598. return
  599. }
  600. yValue, ok := bDataMap[dateStr]
  601. if !ok {
  602. err = errors.New("指标B日期:" + dateStr + "数据异常,导致计算线性方程公式失败")
  603. return
  604. }
  605. tmpCoordinate := utils.Coordinate{
  606. X: xValue,
  607. Y: yValue,
  608. }
  609. coordinateData = append(coordinateData, tmpCoordinate)
  610. }
  611. r = utils.ComputeCorrelation(coordinateData)
  612. return
  613. }
  614. // getNhccData 获取拟合残差需要的数据
  615. func getPredictNhccData(firstEdbInfo, secondEdbInfo *EdbInfo, existItemA, existItemB *EdbInfoCalculateMapping, nhccDate NhccDate) (aDataList []EdbInfoSearchData, secondDataList []*EdbInfoSearchData, aDataMap, bDataMap map[string]float64, err error) {
  616. //第一个指标
  617. aDataList = make([]EdbInfoSearchData, 0)
  618. aDataMap = make(map[string]float64)
  619. {
  620. /*var condition string
  621. var pars []interface{}
  622. condition += " AND edb_info_id=? "
  623. pars = append(pars, existItemA.FromEdbInfoId)
  624. //第一个指标的数据列表
  625. firstDataList, tmpErr := GetEdbDataListAllByTo(to, condition, pars, existItemA.FromSource, 0)
  626. if tmpErr != nil {
  627. return tmpErr
  628. }*/
  629. var firstDataList []*EdbInfoSearchData
  630. firstDataList, err = GetPredictEdbDataListAllByStartDate(firstEdbInfo, 0, "")
  631. if err != nil {
  632. return
  633. }
  634. aDataList, aDataMap = handleNhccData(firstDataList, existItemA.MoveValue)
  635. }
  636. //第二个指标
  637. bDataList := make([]EdbInfoSearchData, 0)
  638. secondDataList = make([]*EdbInfoSearchData, 0)
  639. bDataMap = make(map[string]float64)
  640. {
  641. /*condition = ``
  642. pars = make([]interface{}, 0)
  643. condition += " AND edb_info_id = ? "
  644. pars = append(pars, existItemB.FromEdbInfoId)
  645. //第二个指标的数据列表
  646. secondDataList, err = GetEdbDataListAllByTo(to, condition, pars, existItemB.FromSource, 0)
  647. if err != nil {
  648. return
  649. }*/
  650. secondDataList, err = GetPredictEdbDataListAllByStartDate(secondEdbInfo, 0, "")
  651. if err != nil {
  652. return
  653. }
  654. bDataList, bDataMap = handleNhccData(secondDataList, existItemB.MoveValue)
  655. }
  656. if len(aDataList) <= 0 {
  657. err = errors.New("指标A没有数据")
  658. return
  659. }
  660. if len(bDataList) <= 0 {
  661. err = errors.New("指标B没有数据")
  662. return
  663. }
  664. // 拟合残差计算的结束日期判断
  665. {
  666. endAData := aDataList[len(aDataList)-1]
  667. tmpEndDate, tmpErr := time.ParseInLocation(utils.FormatDate, endAData.DataTime, time.Local)
  668. if tmpErr != nil {
  669. err = tmpErr
  670. return
  671. }
  672. // 如果A指标的最新数据日期早于拟合残差的结束日期,那么就用A指标的最新数据日期
  673. if tmpEndDate.Before(nhccDate.EndDate) {
  674. nhccDate.EndDate = tmpEndDate
  675. }
  676. endBData := bDataList[len(bDataList)-1]
  677. tmpEndDate, tmpErr = time.ParseInLocation(utils.FormatDate, endBData.DataTime, time.Local)
  678. if tmpErr != nil {
  679. err = tmpErr
  680. return
  681. }
  682. // 如果B指标的最新数据日期早于拟合残差的结束日期,那么就用B指标的最新数据日期
  683. if tmpEndDate.Before(nhccDate.EndDate) {
  684. nhccDate.EndDate = tmpEndDate
  685. }
  686. }
  687. return
  688. }