edb_info_calculate.go 27 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894
  1. package data
  2. import (
  3. "encoding/json"
  4. "errors"
  5. "eta/eta_api/models/data_manage"
  6. "eta/eta_api/utils"
  7. "fmt"
  8. "github.com/shopspring/decimal"
  9. "github.com/yidane/formula"
  10. "math"
  11. "regexp"
  12. "sort"
  13. "strconv"
  14. "strings"
  15. "time"
  16. )
  17. func CheckFormula(formula string) map[string]string {
  18. mathFormula := []string{"MAX", "MIN", "ABS", "ACOS", "ASIN", "CEIL", "MOD", "POW", "ROUND", "SIGN", "SIN", "TAN", "LOG10", "LOG2", "LOG", "LN"}
  19. str := strings.ToUpper(formula)
  20. for _, v := range mathFormula {
  21. str = strings.Replace(str, v, "", -1)
  22. }
  23. str = strings.Replace(str, "(", "", -1)
  24. str = strings.Replace(str, ")", "", -1)
  25. byteMap := make(map[string]string)
  26. for i := 0; i < len(str); i++ {
  27. byteInt := str[i]
  28. if byteInt >= 65 && byteInt <= 90 {
  29. byteStr := string(byteInt)
  30. if _, ok := byteMap[byteStr]; !ok {
  31. byteMap[byteStr] = byteStr
  32. }
  33. }
  34. }
  35. return byteMap
  36. }
  37. // CheckFormula2 校验公式是否正常(比如说除法的分母不能为0之类的,实际上就是用预设的字段数据做一次计算)
  38. func CheckFormula2(edbInfoArr []*data_manage.EdbInfo, formulaMap map[string]string, formulaStr string, edbInfoIdBytes []string) (ok bool, err error) {
  39. valArr := make(map[int]float64)
  40. for _, v := range edbInfoArr {
  41. valArr[v.EdbInfoId] = 100
  42. }
  43. formulaStr = strings.ToUpper(formulaStr)
  44. formulaFormStr := ReplaceFormula(edbInfoArr, valArr, formulaMap, formulaStr, edbInfoIdBytes)
  45. if formulaFormStr == "" {
  46. return
  47. }
  48. expression := formula.NewExpression(formulaFormStr)
  49. _, err = expression.Evaluate()
  50. if err != nil {
  51. } else {
  52. ok = true
  53. }
  54. return
  55. }
  56. type FormulaListItem struct {
  57. Formula string `json:"f"`
  58. Date string `json:"d"`
  59. }
  60. // HandleFormulaJson 处理计算公式json串是否异常
  61. func HandleFormulaJson(formula string, startDate string) (dateSlice []string, formulaMap map[string]string, err error) {
  62. list := make([]FormulaListItem, 0)
  63. err = json.Unmarshal([]byte(formula), &list)
  64. if err != nil {
  65. err = fmt.Errorf("公式串解析失败: json.Unmarshal Err: %v", err)
  66. return
  67. }
  68. formulaMap = make(map[string]string)
  69. dateSlice = make([]string, 0)
  70. // 日期排序
  71. for k, v := range list {
  72. if k == 0 { // 首个日期均为起始日
  73. v.Date = startDate
  74. }
  75. formulaMap[v.Date] = v.Formula
  76. dateSlice = append(dateSlice, v.Date)
  77. }
  78. sort.Slice(dateSlice, func(i, j int) bool {
  79. return dateSlice[i] > dateSlice[j]
  80. })
  81. return
  82. }
  83. // CheckFormulaJson 检测计算公式json串是否异常
  84. func CheckFormulaJson(formula string) (formulaSlice []string, err error) {
  85. list := make([]FormulaListItem, 0)
  86. err = json.Unmarshal([]byte(formula), &list)
  87. if err != nil {
  88. err = fmt.Errorf("公式串解析失败: json.Unmarshal Err: %v", err)
  89. return
  90. }
  91. formulaSlice = make([]string, 0)
  92. // 日期排序
  93. for _, v := range list {
  94. formulaSlice = append(formulaSlice, v.Formula)
  95. }
  96. return
  97. }
  98. type CalculateItems struct {
  99. EdbInfoId int
  100. DataMap map[string]float64
  101. }
  102. func Calculate(edbInfoIdArr []*data_manage.EdbInfo, edbInfoId int, edbCode, formulaStr string, edbInfoIdBytes []string) (err error) {
  103. defer func() {
  104. if err != nil {
  105. utils.FileLog.Info("Calculate Err:%s" + err.Error())
  106. }
  107. }()
  108. saveDataMap := make(map[string]map[int]float64)
  109. for _, v := range edbInfoIdArr {
  110. var condition string
  111. var pars []interface{}
  112. condition += " AND edb_info_id=? "
  113. pars = append(pars, v.EdbInfoId)
  114. dataList, err := data_manage.GetEdbDataListAll(condition, pars, v.Source, v.SubSource, 1)
  115. if err != nil {
  116. return err
  117. }
  118. dataMap := make(map[string]float64)
  119. for _, dv := range dataList {
  120. if val, ok := saveDataMap[dv.DataTime]; ok {
  121. if _, ok := val[v.EdbInfoId]; !ok {
  122. val[v.EdbInfoId] = dv.Value
  123. }
  124. } else {
  125. temp := make(map[int]float64)
  126. temp[v.EdbInfoId] = dv.Value
  127. saveDataMap[dv.DataTime] = temp
  128. }
  129. }
  130. item := new(CalculateItems)
  131. item.EdbInfoId = v.EdbInfoId
  132. item.DataMap = dataMap
  133. }
  134. formulaMap := CheckFormula(formulaStr)
  135. addSql := ` INSERT INTO edb_data_calculate(edb_info_id,edb_code,data_time,value,create_time,modify_time,status,data_timestamp) values `
  136. nowStr := time.Now().Format(utils.FormatDateTime)
  137. var isAdd bool
  138. for sk, sv := range saveDataMap {
  139. formulaStr = strings.ToUpper(formulaStr)
  140. formulaFormStr := ReplaceFormula(edbInfoIdArr, sv, formulaMap, formulaStr, edbInfoIdBytes)
  141. if formulaStr == "" {
  142. return
  143. }
  144. if formulaFormStr != "" {
  145. expression := formula.NewExpression(formulaFormStr)
  146. calResult, err := expression.Evaluate()
  147. if err != nil {
  148. err = errors.New("计算失败:Err:" + err.Error() + ";formulaStr:" + formulaFormStr)
  149. fmt.Println(err)
  150. return err
  151. }
  152. calVal, err := calResult.Float64()
  153. if err != nil {
  154. err = errors.New("计算失败:获取计算值失败 Err:" + err.Error() + ";formulaStr:" + formulaFormStr)
  155. fmt.Println(err)
  156. return err
  157. }
  158. //需要存入的数据
  159. {
  160. dataTime, _ := time.Parse(utils.FormatDate, sk)
  161. timestamp := dataTime.UnixNano() / 1e6
  162. timeStr := fmt.Sprintf("%d", timestamp)
  163. addSql += "("
  164. addSql += strconv.Itoa(edbInfoId) + "," + "'" + edbCode + "'" + "," + "'" + sk + "'" + "," + utils.SubFloatToString(calVal, 4) + "," + "'" + nowStr + "'" +
  165. "," + "'" + nowStr + "'" + "," + "1"
  166. addSql += "," + "'" + timeStr + "'"
  167. addSql += "),"
  168. isAdd = true
  169. }
  170. } else {
  171. fmt.Println("formulaFormStr is empty")
  172. }
  173. }
  174. if isAdd {
  175. addSql = strings.TrimRight(addSql, ",")
  176. data_manage.AddEdbDataCalculateBySql(addSql)
  177. if err != nil {
  178. fmt.Println("AddEdbDataCalculate Err:" + err.Error())
  179. return err
  180. }
  181. }
  182. return
  183. }
  184. func ReplaceFormula(edbInfoIdArr []*data_manage.EdbInfo, valArr map[int]float64, formulaMap map[string]string, formulaStr string, edbInfoIdBytes []string) string {
  185. // todo 处理min和max
  186. funMap := GetFormulaMap()
  187. for k, v := range funMap {
  188. formulaStr = strings.Replace(formulaStr, k, v, -1)
  189. }
  190. replaceCount := 0
  191. for dk, dv := range edbInfoIdArr {
  192. if dk == 0 {
  193. dKey := edbInfoIdBytes[dk]
  194. if _, ok := formulaMap[dKey]; ok { //公式中存在
  195. if val, valOk := valArr[dv.EdbInfoId]; valOk { //值存在
  196. dvStr := fmt.Sprintf("%v", val)
  197. formulaStr = strings.Replace(formulaStr, dKey, dvStr, -1)
  198. replaceCount++
  199. } else {
  200. fmt.Println("valArr not found:", valArr, valOk)
  201. }
  202. } else {
  203. fmt.Println("formulaMap not found:", dKey, dk)
  204. }
  205. }
  206. if dk == 1 {
  207. dKey := edbInfoIdBytes[dk]
  208. if _, ok := formulaMap[dKey]; ok { //公式中存在
  209. if val, valOk := valArr[dv.EdbInfoId]; valOk { //值存在
  210. dvStr := fmt.Sprintf("%v", val)
  211. formulaStr = strings.Replace(formulaStr, dKey, dvStr, -1)
  212. replaceCount++
  213. } else {
  214. fmt.Println("valArr not found:", valArr, valOk)
  215. }
  216. } else {
  217. fmt.Println("formulaMap not found:", dKey, dk)
  218. }
  219. }
  220. if dk == 2 {
  221. dKey := edbInfoIdBytes[dk]
  222. if _, ok := formulaMap[dKey]; ok { //公式中存在
  223. if val, valOk := valArr[dv.EdbInfoId]; valOk { //值存在
  224. dvStr := fmt.Sprintf("%v", val)
  225. formulaStr = strings.Replace(formulaStr, dKey, dvStr, -1)
  226. replaceCount++
  227. }
  228. }
  229. }
  230. if dk == 3 {
  231. dKey := edbInfoIdBytes[dk]
  232. if _, ok := formulaMap[dKey]; ok { //公式中存在
  233. if val, valOk := valArr[dv.EdbInfoId]; valOk { //值存在
  234. dvStr := fmt.Sprintf("%v", val)
  235. formulaStr = strings.Replace(formulaStr, dKey, dvStr, -1)
  236. replaceCount++
  237. }
  238. }
  239. }
  240. if dk == 4 {
  241. dKey := edbInfoIdBytes[dk]
  242. if _, ok := formulaMap[dKey]; ok { //公式中存在
  243. if val, valOk := valArr[dv.EdbInfoId]; valOk { //值存在
  244. dvStr := fmt.Sprintf("%v", val)
  245. formulaStr = strings.Replace(formulaStr, dKey, dvStr, -1)
  246. replaceCount++
  247. }
  248. }
  249. }
  250. if dk == 5 {
  251. dKey := edbInfoIdBytes[dk]
  252. if _, ok := formulaMap[dKey]; ok { //公式中存在
  253. if val, valOk := valArr[dv.EdbInfoId]; valOk { //值存在
  254. dvStr := fmt.Sprintf("%v", val)
  255. formulaStr = strings.Replace(formulaStr, dKey, dvStr, -1)
  256. replaceCount++
  257. }
  258. }
  259. }
  260. if dk == 6 {
  261. dKey := edbInfoIdBytes[dk]
  262. if _, ok := formulaMap[dKey]; ok { //公式中存在
  263. if val, valOk := valArr[dv.EdbInfoId]; valOk { //值存在
  264. dvStr := fmt.Sprintf("%v", val)
  265. formulaStr = strings.Replace(formulaStr, dKey, dvStr, -1)
  266. replaceCount++
  267. }
  268. }
  269. }
  270. if dk == 7 {
  271. dKey := edbInfoIdBytes[dk]
  272. if _, ok := formulaMap[dKey]; ok { //公式中存在
  273. if val, valOk := valArr[dv.EdbInfoId]; valOk { //值存在
  274. dvStr := fmt.Sprintf("%v", val)
  275. formulaStr = strings.Replace(formulaStr, dKey, dvStr, -1)
  276. replaceCount++
  277. }
  278. }
  279. }
  280. if dk == 8 {
  281. dKey := edbInfoIdBytes[dk]
  282. if _, ok := formulaMap[dKey]; ok { //公式中存在
  283. if val, valOk := valArr[dv.EdbInfoId]; valOk { //值存在
  284. dvStr := fmt.Sprintf("%v", val)
  285. formulaStr = strings.Replace(formulaStr, dKey, dvStr, -1)
  286. replaceCount++
  287. }
  288. }
  289. }
  290. if dk == 9 {
  291. dKey := edbInfoIdBytes[dk]
  292. if _, ok := formulaMap[dKey]; ok { //公式中存在
  293. if val, valOk := valArr[dv.EdbInfoId]; valOk { //值存在
  294. dvStr := fmt.Sprintf("%v", val)
  295. formulaStr = strings.Replace(formulaStr, dKey, dvStr, -1)
  296. replaceCount++
  297. }
  298. }
  299. }
  300. if dk == 10 {
  301. dKey := edbInfoIdBytes[dk]
  302. if _, ok := formulaMap[dKey]; ok { //公式中存在
  303. if val, valOk := valArr[dv.EdbInfoId]; valOk { //值存在
  304. dvStr := fmt.Sprintf("%v", val)
  305. formulaStr = strings.Replace(formulaStr, dKey, dvStr, -1)
  306. replaceCount++
  307. }
  308. }
  309. }
  310. if dk == 11 {
  311. dKey := edbInfoIdBytes[dk]
  312. if _, ok := formulaMap[dKey]; ok { //公式中存在
  313. if val, valOk := valArr[dv.EdbInfoId]; valOk { //值存在
  314. dvStr := fmt.Sprintf("%v", val)
  315. formulaStr = strings.Replace(formulaStr, dKey, dvStr, -1)
  316. replaceCount++
  317. }
  318. }
  319. }
  320. if dk == 12 {
  321. dKey := edbInfoIdBytes[dk]
  322. if _, ok := formulaMap[dKey]; ok { //公式中存在
  323. if val, valOk := valArr[dv.EdbInfoId]; valOk { //值存在
  324. dvStr := fmt.Sprintf("%v", val)
  325. formulaStr = strings.Replace(formulaStr, dKey, dvStr, -1)
  326. replaceCount++
  327. }
  328. }
  329. }
  330. if dk == 13 {
  331. dKey := edbInfoIdBytes[dk]
  332. if _, ok := formulaMap[dKey]; ok { //公式中存在
  333. if val, valOk := valArr[dv.EdbInfoId]; valOk { //值存在
  334. dvStr := fmt.Sprintf("%v", val)
  335. formulaStr = strings.Replace(formulaStr, dKey, dvStr, -1)
  336. replaceCount++
  337. }
  338. }
  339. }
  340. if dk == 14 {
  341. dKey := edbInfoIdBytes[dk]
  342. if _, ok := formulaMap[dKey]; ok { //公式中存在
  343. if val, valOk := valArr[dv.EdbInfoId]; valOk { //值存在
  344. dvStr := fmt.Sprintf("%v", val)
  345. formulaStr = strings.Replace(formulaStr, dKey, dvStr, -1)
  346. replaceCount++
  347. }
  348. }
  349. }
  350. if dk == 15 {
  351. dKey := edbInfoIdBytes[dk]
  352. if _, ok := formulaMap[dKey]; ok { //公式中存在
  353. if val, valOk := valArr[dv.EdbInfoId]; valOk { //值存在
  354. dvStr := fmt.Sprintf("%v", val)
  355. formulaStr = strings.Replace(formulaStr, dKey, dvStr, -1)
  356. replaceCount++
  357. }
  358. }
  359. }
  360. if dk == 16 {
  361. dKey := edbInfoIdBytes[dk]
  362. if _, ok := formulaMap[dKey]; ok { //公式中存在
  363. if val, valOk := valArr[dv.EdbInfoId]; valOk { //值存在
  364. dvStr := fmt.Sprintf("%v", val)
  365. formulaStr = strings.Replace(formulaStr, dKey, dvStr, -1)
  366. replaceCount++
  367. }
  368. }
  369. }
  370. if dk == 17 {
  371. dKey := edbInfoIdBytes[dk]
  372. if _, ok := formulaMap[dKey]; ok { //公式中存在
  373. if val, valOk := valArr[dv.EdbInfoId]; valOk { //值存在
  374. dvStr := fmt.Sprintf("%v", val)
  375. formulaStr = strings.Replace(formulaStr, dKey, dvStr, -1)
  376. replaceCount++
  377. }
  378. }
  379. }
  380. if dk == 18 {
  381. dKey := edbInfoIdBytes[dk]
  382. if _, ok := formulaMap[dKey]; ok { //公式中存在
  383. if val, valOk := valArr[dv.EdbInfoId]; valOk { //值存在
  384. dvStr := fmt.Sprintf("%v", val)
  385. formulaStr = strings.Replace(formulaStr, dKey, dvStr, -1)
  386. replaceCount++
  387. }
  388. }
  389. }
  390. if dk == 19 {
  391. dKey := edbInfoIdBytes[dk]
  392. if _, ok := formulaMap[dKey]; ok { //公式中存在
  393. if val, valOk := valArr[dv.EdbInfoId]; valOk { //值存在
  394. dvStr := fmt.Sprintf("%v", val)
  395. formulaStr = strings.Replace(formulaStr, dKey, dvStr, -1)
  396. replaceCount++
  397. }
  398. }
  399. }
  400. if dk == 20 {
  401. dKey := edbInfoIdBytes[dk]
  402. if _, ok := formulaMap[dKey]; ok { //公式中存在
  403. if val, valOk := valArr[dv.EdbInfoId]; valOk { //值存在
  404. dvStr := fmt.Sprintf("%v", val)
  405. formulaStr = strings.Replace(formulaStr, dKey, dvStr, -1)
  406. replaceCount++
  407. }
  408. }
  409. }
  410. if dk == 21 {
  411. dKey := edbInfoIdBytes[dk]
  412. if _, ok := formulaMap[dKey]; ok { //公式中存在
  413. if val, valOk := valArr[dv.EdbInfoId]; valOk { //值存在
  414. dvStr := fmt.Sprintf("%v", val)
  415. formulaStr = strings.Replace(formulaStr, dKey, dvStr, -1)
  416. replaceCount++
  417. }
  418. }
  419. }
  420. if dk == 22 {
  421. dKey := edbInfoIdBytes[dk]
  422. if _, ok := formulaMap[dKey]; ok { //公式中存在
  423. if val, valOk := valArr[dv.EdbInfoId]; valOk { //值存在
  424. dvStr := fmt.Sprintf("%v", val)
  425. formulaStr = strings.Replace(formulaStr, dKey, dvStr, -1)
  426. replaceCount++
  427. }
  428. }
  429. }
  430. if dk == 23 {
  431. dKey := edbInfoIdBytes[dk]
  432. if _, ok := formulaMap[dKey]; ok { //公式中存在
  433. if val, valOk := valArr[dv.EdbInfoId]; valOk { //值存在
  434. dvStr := fmt.Sprintf("%v", val)
  435. formulaStr = strings.Replace(formulaStr, dKey, dvStr, -1)
  436. replaceCount++
  437. }
  438. }
  439. }
  440. if dk == 24 {
  441. dKey := edbInfoIdBytes[dk]
  442. if _, ok := formulaMap[dKey]; ok { //公式中存在
  443. if val, valOk := valArr[dv.EdbInfoId]; valOk { //值存在
  444. dvStr := fmt.Sprintf("%v", val)
  445. formulaStr = strings.Replace(formulaStr, dKey, dvStr, -1)
  446. replaceCount++
  447. }
  448. }
  449. }
  450. if dk == 25 {
  451. dKey := edbInfoIdBytes[dk]
  452. if _, ok := formulaMap[dKey]; ok { //公式中存在
  453. if val, valOk := valArr[dv.EdbInfoId]; valOk { //值存在
  454. dvStr := fmt.Sprintf("%v", val)
  455. formulaStr = strings.Replace(formulaStr, dKey, dvStr, -1)
  456. replaceCount++
  457. }
  458. }
  459. }
  460. }
  461. for k, v := range funMap {
  462. formulaStr = strings.Replace(formulaStr, v, k, -1)
  463. }
  464. if replaceCount == len(formulaMap) {
  465. return formulaStr
  466. } else {
  467. return ""
  468. }
  469. }
  470. func GetFormulaMap() map[string]string {
  471. funMap := make(map[string]string)
  472. funMap["MAX"] = "[@@]"
  473. funMap["MIN"] = "[@!]"
  474. funMap["ABS"] = "[@#]"
  475. funMap["CEIL"] = "[@$]"
  476. funMap["COS"] = "[@%]"
  477. funMap["FLOOR"] = "[@^]"
  478. funMap["MOD"] = "[@&]"
  479. funMap["POW"] = "[@*]"
  480. funMap["ROUND"] = "[@(]"
  481. return funMap
  482. }
  483. // 刷新数据
  484. func RefreshCalculate(edbInfoIdArr []*data_manage.EdbInfo, edbInfoId int, edbCode, formulaStr, startDate, endDate string, edbInfoIdBytes []string) (err error) {
  485. defer func() {
  486. if err != nil {
  487. fmt.Println("RefreshCalculate:Err " + err.Error())
  488. utils.FileLog.Info("Calculate Err:%s" + err.Error())
  489. }
  490. }()
  491. saveDataMap := make(map[string]map[int]float64)
  492. for _, v := range edbInfoIdArr {
  493. var condition string
  494. var pars []interface{}
  495. condition += " AND edb_info_id=? "
  496. pars = append(pars, v.EdbInfoId)
  497. if startDate != "" {
  498. condition += " AND data_time>=? "
  499. pars = append(pars, startDate)
  500. }
  501. if endDate != "" {
  502. condition += " AND data_time<=? "
  503. pars = append(pars, endDate)
  504. }
  505. dataList, err := data_manage.GetEdbDataListAll(condition, pars, v.Source, v.SubSource, 1)
  506. if err != nil {
  507. return err
  508. }
  509. dataMap := make(map[string]float64)
  510. for _, dv := range dataList {
  511. if val, ok := saveDataMap[dv.DataTime]; ok {
  512. if _, ok := val[v.EdbInfoId]; !ok {
  513. val[v.EdbInfoId] = dv.Value
  514. }
  515. } else {
  516. temp := make(map[int]float64)
  517. temp[v.EdbInfoId] = dv.Value
  518. saveDataMap[dv.DataTime] = temp
  519. }
  520. }
  521. item := new(CalculateItems)
  522. item.EdbInfoId = v.EdbInfoId
  523. item.DataMap = dataMap
  524. }
  525. formulaMap := CheckFormula(formulaStr)
  526. addSql := ` INSERT INTO edb_data_calculate(edb_info_id,edb_code,data_time,value,create_time,modify_time,status,data_timestamp) values `
  527. nowStr := time.Now().Format(utils.FormatDateTime)
  528. var isAdd bool
  529. for sk, sv := range saveDataMap {
  530. formulaStr = strings.ToUpper(formulaStr)
  531. formulaFormStr := ReplaceFormula(edbInfoIdArr, sv, formulaMap, formulaStr, edbInfoIdBytes)
  532. if formulaFormStr != "" {
  533. utils.FileLog.Info("formulaFormStr:%s", formulaFormStr)
  534. expression := formula.NewExpression(formulaFormStr)
  535. calResult, err := expression.Evaluate()
  536. if err != nil {
  537. err = errors.New("计算失败:Err:" + err.Error() + ";formulaStr:" + formulaFormStr)
  538. fmt.Println(err)
  539. return err
  540. }
  541. calVal, err := calResult.Float64()
  542. if err != nil {
  543. err = errors.New("计算失败:获取计算值失败 Err:" + err.Error() + ";formulaStr:" + formulaFormStr)
  544. fmt.Println(err)
  545. return err
  546. }
  547. count, err := data_manage.GetEdbDataCalculateByCodeAndDate(edbCode, sk)
  548. if err != nil && err.Error() != utils.ErrNoRow() {
  549. return err
  550. }
  551. if count <= 0 { //需要存入的数据
  552. dataTime, _ := time.Parse(utils.FormatDate, sk)
  553. timestamp := dataTime.UnixNano() / 1e6
  554. timeStr := fmt.Sprintf("%d", timestamp)
  555. addSql += "("
  556. addSql += strconv.Itoa(edbInfoId) + "," + "'" + edbCode + "'" + "," + "'" + sk + "'" + "," + utils.SubFloatToString(calVal, 4) + "," + "'" + nowStr + "'" +
  557. "," + "'" + nowStr + "'" + "," + "1"
  558. addSql += "," + "'" + timeStr + "'"
  559. addSql += "),"
  560. isAdd = true
  561. } else {
  562. calVal = utils.FixFloat(calVal, 4)
  563. err = data_manage.ModifyEdbDataCalculate(int64(edbInfoId), sk, calVal)
  564. if err != nil {
  565. return err
  566. }
  567. }
  568. }
  569. }
  570. if isAdd {
  571. addSql = strings.TrimRight(addSql, ",")
  572. data_manage.AddEdbDataCalculateBySql(addSql)
  573. if err != nil {
  574. fmt.Println("AddEdbDataCalculate Err:" + err.Error())
  575. return err
  576. }
  577. }
  578. return
  579. }
  580. // 处理整个数据
  581. func handleDateSaveDataMap(dateList []string, realSaveDataMap, saveDataMap map[string]map[int]float64, edbInfoIdArr []*data_manage.EdbInfo, emptyType int) {
  582. var startDate, endDate string
  583. var startDateT, endDateT time.Time
  584. if emptyType == 2 || emptyType == 3 {
  585. for k, _ := range realSaveDataMap {
  586. if k > endDate {
  587. endDate = k
  588. }
  589. if k < startDate || startDate == "" {
  590. startDate = k
  591. }
  592. }
  593. startDateT, _ = time.ParseInLocation(utils.FormatDate, startDate, time.Local)
  594. endDateT, _ = time.ParseInLocation(utils.FormatDate, endDate, time.Local)
  595. }
  596. for _, date := range dateList {
  597. tmpDataMap := realSaveDataMap[date]
  598. for _, edbInfo := range edbInfoIdArr {
  599. tmpEdbInfoId := edbInfo.EdbInfoId // 当前指标id
  600. // 如果该日期不存在该指标数据,那么需要找寻前后日期的数据,进行填补
  601. if _, ok := tmpDataMap[tmpEdbInfoId]; !ok {
  602. //day := 0
  603. //switch edbInfo.Frequency {
  604. //case "周度":
  605. // day = 7
  606. //case "旬度":
  607. // day = 15
  608. //case "月度":
  609. // day = 30
  610. //case "季度":
  611. // day = 90
  612. //case "年度":
  613. // day = 365
  614. //}
  615. // 需求池 255 指标运算文案修改,补数据遍历区间修改(2023-3-7 09:37:23修改)
  616. switch emptyType {
  617. case 0:
  618. handleDateDataMap(realSaveDataMap, saveDataMap, date, tmpEdbInfoId, 35)
  619. case 2:
  620. handleDateDataMapBefore(realSaveDataMap, saveDataMap, date, tmpEdbInfoId, startDateT, endDateT)
  621. case 3:
  622. handleDateDataMapAfter(realSaveDataMap, saveDataMap, date, tmpEdbInfoId, startDateT, endDateT)
  623. case 4:
  624. handleDateDataMapZero(saveDataMap, date, tmpEdbInfoId)
  625. }
  626. }
  627. }
  628. }
  629. }
  630. // handleDataMap 处理单个日期的数据
  631. func handleDateDataMap(realSaveDataMap, saveDataMap map[string]map[int]float64, date string, edbInfoId, day int) {
  632. currDate, _ := time.ParseInLocation(utils.FormatDate, date, time.Local)
  633. // 后一天
  634. nextDateDayStr := currDate.AddDate(0, 0, 1).Format(utils.FormatDate)
  635. // 前一天
  636. preDateDayStr := currDate.AddDate(0, 0, -1).Format(utils.FormatDate)
  637. for i := 1; i <= day; i++ {
  638. // 下个日期的数据
  639. {
  640. if i >= 1 {
  641. nextDateDayStr = currDate.AddDate(0, 0, i).Format(utils.FormatDate)
  642. }
  643. if findDataMap, hasFindDataMap := realSaveDataMap[nextDateDayStr]; hasFindDataMap { // 下一个日期有数据
  644. if val, hasFindItem := findDataMap[edbInfoId]; hasFindItem {
  645. saveDataMap[date][edbInfoId] = val
  646. return
  647. }
  648. }
  649. }
  650. // 上个日期的数据
  651. {
  652. if i >= 1 {
  653. preDateDayStr = currDate.AddDate(0, 0, -i).Format(utils.FormatDate)
  654. }
  655. if findDataMap, hasFindDataMap := realSaveDataMap[preDateDayStr]; hasFindDataMap { // 下一个日期有数据
  656. if val, hasFindItem := findDataMap[edbInfoId]; hasFindItem {
  657. saveDataMap[date][edbInfoId] = val
  658. return
  659. }
  660. }
  661. }
  662. }
  663. }
  664. // handleDataByLinearRegression 插值法补充数据(线性方程式)
  665. func handleDataByLinearRegression(edbInfoDataList []*data_manage.EdbDataList, handleDataMap map[string]float64) (err error) {
  666. if len(edbInfoDataList) < 2 {
  667. return
  668. }
  669. var startEdbInfoData *data_manage.EdbDataList
  670. for _, v := range edbInfoDataList {
  671. handleDataMap[v.DataTime] = v.Value
  672. // 第一个数据就给过滤了,给后面的试用
  673. if startEdbInfoData == nil {
  674. startEdbInfoData = v
  675. continue
  676. }
  677. // 获取两条数据之间相差的天数
  678. startDataTime, _ := time.ParseInLocation(utils.FormatDate, startEdbInfoData.DataTime, time.Local)
  679. currDataTime, _ := time.ParseInLocation(utils.FormatDate, v.DataTime, time.Local)
  680. betweenHour := int(currDataTime.Sub(startDataTime).Hours())
  681. betweenDay := betweenHour / 24
  682. // 如果相差一天,那么过滤
  683. if betweenDay <= 1 {
  684. startEdbInfoData = v
  685. continue
  686. }
  687. // 生成线性方程式
  688. var a, b float64
  689. {
  690. coordinateData := make([]utils.Coordinate, 0)
  691. tmpCoordinate1 := utils.Coordinate{
  692. X: 1,
  693. Y: startEdbInfoData.Value,
  694. }
  695. coordinateData = append(coordinateData, tmpCoordinate1)
  696. tmpCoordinate2 := utils.Coordinate{
  697. X: float64(betweenDay) + 1,
  698. Y: v.Value,
  699. }
  700. coordinateData = append(coordinateData, tmpCoordinate2)
  701. a, b = utils.GetLinearResult(coordinateData)
  702. if math.IsNaN(a) || math.IsNaN(b) {
  703. err = errors.New("线性方程公式生成失败")
  704. return
  705. }
  706. }
  707. // 生成对应的值
  708. {
  709. for i := 1; i < betweenDay; i++ {
  710. tmpDataTime := startDataTime.AddDate(0, 0, i)
  711. aDecimal := decimal.NewFromFloat(a)
  712. xDecimal := decimal.NewFromInt(int64(i) + 1)
  713. bDecimal := decimal.NewFromFloat(b)
  714. val, _ := aDecimal.Mul(xDecimal).Add(bDecimal).Round(4).Float64()
  715. handleDataMap[tmpDataTime.Format(utils.FormatDate)] = val
  716. }
  717. }
  718. startEdbInfoData = v
  719. }
  720. return
  721. }
  722. // HandleDataByLinearRegression 插值法补充数据(线性方程式)
  723. func HandleDataByLinearRegression(edbInfoDataList []*data_manage.EdbDataList, handleDataMap map[string]float64) (err error) {
  724. return handleDataByLinearRegression(edbInfoDataList, handleDataMap)
  725. }
  726. // CallCalculateComputeCorrelation 调用计算拟合残差的相关系数
  727. func CallCalculateComputeCorrelation(data *data_manage.EdbInfoCalculateBatchSaveReqByEdbLib) (val string, err error, errMsg string) {
  728. errMsg = "计算失败"
  729. // 调用指标库去更新
  730. reqJson, err := json.Marshal(data)
  731. if err != nil {
  732. errMsg = "计算相关系数参数解析异常!"
  733. err = errors.New("参数解析失败,Err:" + err.Error())
  734. return
  735. }
  736. respItem, err := CalculateComputeCorrelation(string(reqJson))
  737. if err != nil {
  738. return
  739. }
  740. if respItem.Ret == 200 {
  741. val = respItem.Data
  742. }
  743. return
  744. }
  745. // handleDateDataMapBefore 前值填充:空值优先以最近的前值填充,没有前值时,用后值填充
  746. func handleDateDataMapBefore(realSaveDataMap, saveDataMap map[string]map[int]float64, date string, edbInfoId int, startDateT, endDateT time.Time) {
  747. currDate, _ := time.ParseInLocation(utils.FormatDate, date, time.Local)
  748. // 后一天
  749. nextDateDay := currDate
  750. // 前一天
  751. preDateDay := currDate
  752. for i := 1; preDateDay.After(startDateT) || preDateDay == startDateT; i++ {
  753. // 上个日期的数据
  754. {
  755. preDateDay = currDate.AddDate(0, 0, -i)
  756. preDateDayStr := preDateDay.Format(utils.FormatDate)
  757. if findDataMap, hasFindDataMap := realSaveDataMap[preDateDayStr]; hasFindDataMap { // 下一个日期有数据
  758. if val, hasFindItem := findDataMap[edbInfoId]; hasFindItem {
  759. fmt.Println(fmt.Sprintf("date:%s, 无值,取%s的值%.4f", date, preDateDayStr, val))
  760. saveDataMap[date][edbInfoId] = val
  761. return
  762. }
  763. }
  764. }
  765. }
  766. for i := 1; nextDateDay.Before(endDateT) || nextDateDay == endDateT; i++ {
  767. // 下个日期的数据
  768. {
  769. nextDateDay = currDate.AddDate(0, 0, i)
  770. nextDateDayStr := nextDateDay.Format(utils.FormatDate)
  771. if findDataMap, hasFindDataMap := realSaveDataMap[nextDateDayStr]; hasFindDataMap { // 下一个日期有数据
  772. if val, hasFindItem := findDataMap[edbInfoId]; hasFindItem {
  773. fmt.Println(fmt.Sprintf("date:%s, 无值,取%s的值%.4f", date, nextDateDayStr, val))
  774. saveDataMap[date][edbInfoId] = val
  775. return
  776. }
  777. }
  778. }
  779. }
  780. return
  781. }
  782. // handleDateDataMapAfter 后值填充:空值优先以最近的后值填充,没有后值时,用前值填充
  783. func handleDateDataMapAfter(realSaveDataMap, saveDataMap map[string]map[int]float64, date string, edbInfoId int, startDateT, endDateT time.Time) {
  784. currDate, _ := time.ParseInLocation(utils.FormatDate, date, time.Local)
  785. // 后一天
  786. nextDateDay := currDate
  787. // 前一天
  788. preDateDay := currDate
  789. for i := 1; nextDateDay.Before(endDateT) || nextDateDay == endDateT; i++ {
  790. // 下个日期的数据
  791. {
  792. nextDateDay = currDate.AddDate(0, 0, i)
  793. nextDateDayStr := nextDateDay.Format(utils.FormatDate)
  794. if findDataMap, hasFindDataMap := realSaveDataMap[nextDateDayStr]; hasFindDataMap { // 下一个日期有数据
  795. if val, hasFindItem := findDataMap[edbInfoId]; hasFindItem {
  796. fmt.Println(fmt.Sprintf("date:%s, 无值,取%s的值%.4f", date, nextDateDayStr, val))
  797. saveDataMap[date][edbInfoId] = val
  798. return
  799. }
  800. }
  801. }
  802. }
  803. for i := 1; preDateDay.After(startDateT) || preDateDay == startDateT; i++ {
  804. // 上个日期的数据
  805. {
  806. preDateDay = currDate.AddDate(0, 0, -i)
  807. preDateDayStr := preDateDay.Format(utils.FormatDate)
  808. if findDataMap, hasFindDataMap := realSaveDataMap[preDateDayStr]; hasFindDataMap { // 下一个日期有数据
  809. if val, hasFindItem := findDataMap[edbInfoId]; hasFindItem {
  810. fmt.Println(fmt.Sprintf("date:%s, 无值,取%s的值%.4f", date, preDateDayStr, val))
  811. saveDataMap[date][edbInfoId] = val
  812. return
  813. }
  814. }
  815. }
  816. }
  817. return
  818. }
  819. // handleDateDataMapZero 等于0
  820. func handleDateDataMapZero(saveDataMap map[string]map[int]float64, date string, edbInfoId int) {
  821. saveDataMap[date][edbInfoId] = 0
  822. return
  823. }
  824. func GetMaxMinEdbInfo(formula string) string {
  825. //formula := "A+min(A,B,max(A,C))"
  826. // todo 无法处理max里嵌套max或者min的情况
  827. // 使用正则表达式匹配MAX和MIN函数及其参数
  828. regex := regexp.MustCompile(`(?i)(MAX|MIN)\((.*?)\)`)
  829. matches := regex.FindAllStringSubmatch(formula, -1)
  830. // 遍历匹配结果,输出MAX和MIN函数及其参数
  831. for _, match := range matches {
  832. if len(match) == 3 {
  833. parameter := strings.ToLower(match[0]) // 参数
  834. formula = strings.ReplaceAll(formula, match[0], parameter)
  835. fmt.Printf("formula: %s\n", formula)
  836. }
  837. }
  838. formula = strings.ReplaceAll(formula, "max", "MAX")
  839. formula = strings.ReplaceAll(formula, "min", "MIN")
  840. return formula
  841. }