edb_data_calculate_sum.go 14 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478
  1. package models
  2. import (
  3. "encoding/json"
  4. "errors"
  5. "eta/eta_index_lib/utils"
  6. "fmt"
  7. "github.com/beego/beego/v2/client/orm"
  8. "github.com/shopspring/decimal"
  9. "reflect"
  10. "strconv"
  11. "strings"
  12. "time"
  13. )
  14. type CalculateSum struct {
  15. }
  16. // Add 新增
  17. func (obj CalculateSum) Add(params AddCalculateBatchParams) (edbInfo *EdbInfo, err error, errMsg string) {
  18. req := params.Req
  19. edbCode := params.EdbCode
  20. o := orm.NewOrm()
  21. to, err := o.Begin()
  22. if err != nil {
  23. return
  24. }
  25. defer func() {
  26. if err != nil {
  27. fmt.Println("CalculateSum.Add,Err:" + err.Error())
  28. fmt.Println("CalculateSum.Add,ErrMsg:" + errMsg)
  29. _ = to.Rollback()
  30. } else {
  31. _ = to.Commit()
  32. }
  33. }()
  34. if req.EdbInfoId > 0 {
  35. err = errors.New("无法新增")
  36. return
  37. }
  38. edbInfo = new(EdbInfo)
  39. edbInfo.Source = obj.GetSource()
  40. edbInfo.SourceName = obj.GetSourceName()
  41. edbInfo.EdbCode = edbCode
  42. edbInfo.EdbName = req.EdbName
  43. edbInfo.EdbNameSource = req.EdbName
  44. edbInfo.Frequency = req.Frequency
  45. edbInfo.Unit = req.Unit
  46. edbInfo.ClassifyId = req.ClassifyId
  47. edbInfo.SysUserId = params.SysUserId
  48. edbInfo.SysUserRealName = params.SysUserRealName
  49. edbInfo.CreateTime = time.Now()
  50. edbInfo.ModifyTime = time.Now()
  51. edbInfo.UniqueCode = params.UniqueCode
  52. edbInfo.EdbType = obj.GetEdbType()
  53. edbInfo.Sort = GetAddEdbMaxSortByClassifyId(req.ClassifyId, utils.EDB_INFO_TYPE)
  54. edbInfo.EmptyType = req.EmptyType
  55. edbInfo.MaxEmptyType = req.MaxEmptyType
  56. edbInfo.Extra = req.Extra
  57. edbInfo.EdbNameEn = req.EdbName
  58. edbInfo.UnitEn = req.Unit
  59. newEdbInfoId, tmpErr := to.Insert(edbInfo)
  60. if tmpErr != nil {
  61. errMsg = "新增失败"
  62. err = tmpErr
  63. return
  64. }
  65. edbInfo.EdbInfoId = int(newEdbInfoId)
  66. //关联关系
  67. tagMap := make(map[string]int)
  68. relationEdbInfoList := make([]*EdbInfo, 0)
  69. calculateMappingItemList := make([]*EdbInfoCalculateMapping, 0)
  70. for k, v := range req.EdbInfoIdArr {
  71. tmpEdbInfo, e := GetEdbInfoById(v.EdbInfoId)
  72. if e != nil {
  73. errMsg = "查不到原指标"
  74. err = e
  75. return
  76. }
  77. relationEdbInfoList = append(relationEdbInfoList, tmpEdbInfo)
  78. calculateMappingItem := new(EdbInfoCalculateMapping)
  79. calculateMappingItem.CreateTime = time.Now()
  80. calculateMappingItem.ModifyTime = time.Now()
  81. calculateMappingItem.Sort = k
  82. calculateMappingItem.EdbCode = edbCode
  83. calculateMappingItem.EdbInfoId = edbInfo.EdbInfoId
  84. calculateMappingItem.FromEdbInfoId = tmpEdbInfo.EdbInfoId
  85. calculateMappingItem.FromEdbCode = tmpEdbInfo.EdbCode
  86. calculateMappingItem.FromEdbName = tmpEdbInfo.EdbName
  87. calculateMappingItem.FromSource = tmpEdbInfo.Source
  88. calculateMappingItem.FromSourceName = tmpEdbInfo.SourceName
  89. calculateMappingItem.FromTag = v.FromTag
  90. calculateMappingItem.Source = edbInfo.Source
  91. calculateMappingItem.SourceName = edbInfo.SourceName
  92. calculateMappingItem.FromSubSource = edbInfo.SubSource
  93. calculateMappingItemList = append(calculateMappingItemList, calculateMappingItem)
  94. tagMap[v.FromTag] = v.EdbInfoId
  95. }
  96. _, err = to.InsertMulti(len(calculateMappingItemList), calculateMappingItemList)
  97. if err != nil {
  98. errMsg = "保存失败"
  99. return
  100. }
  101. //计算数据
  102. err = obj.refresh(to, edbInfo.EdbInfoId, edbInfo.Source, edbInfo.SubSource, relationEdbInfoList, edbInfo.EdbCode, edbInfo.Extra, tagMap, edbInfo.EmptyType)
  103. return
  104. }
  105. func (obj CalculateSum) Edit(params EditCalculateBatchParams) (err error, errMsg string) {
  106. req := params.Req
  107. edbInfo := params.EdbInfo
  108. o := orm.NewOrm()
  109. to, err := o.Begin()
  110. if err != nil {
  111. return
  112. }
  113. defer func() {
  114. if err != nil {
  115. fmt.Println("EditCalculateKszs,Err:" + err.Error())
  116. _ = to.Rollback()
  117. } else {
  118. _ = to.Commit()
  119. }
  120. }()
  121. if len(req.EdbInfoIdArr) <= 0 {
  122. err = fmt.Errorf("请选择要关联的指标")
  123. return
  124. }
  125. needCalculate := false
  126. if edbInfo.EmptyType != req.EmptyType || edbInfo.Extra != req.Extra {
  127. needCalculate = true
  128. }
  129. //修改指标信息
  130. edbInfo.EdbName = req.EdbName
  131. edbInfo.EdbNameSource = req.EdbName
  132. edbInfo.Frequency = req.Frequency
  133. edbInfo.Unit = req.Unit
  134. edbInfo.ClassifyId = req.ClassifyId
  135. edbInfo.CalculateFormula = req.Formula
  136. edbInfo.Extra = req.Extra
  137. edbInfo.EmptyType = req.EmptyType
  138. edbInfo.MaxEmptyType = req.MaxEmptyType
  139. edbInfo.EdbNameEn = req.EdbNameEn
  140. edbInfo.UnitEn = req.UnitEn
  141. edbInfo.ModifyTime = time.Now()
  142. _, err = to.Update(edbInfo, "EdbName", "EdbNameSource", "Frequency", "Unit", "ClassifyId", "CalculateFormula", "Extra", "EmptyType", "MaxEmptyType", "ModifyTime", "EdbNameEn", "UnitEn")
  143. if err != nil {
  144. return
  145. }
  146. //判断是否重复指标
  147. //edbInfoMap := make(map[int]string)
  148. if !needCalculate {
  149. //用到的指标
  150. newEdbInfoIdArr := make([]string, 0)
  151. for _, tmpEdbInfo := range req.EdbInfoIdArr {
  152. /*_, ok := edbInfoMap[tmpEdbInfo.EdbInfoId]
  153. if ok {
  154. br.Msg = "选择指标失败,请勿选择重复指标!"
  155. return
  156. }
  157. edbInfoMap[tmpEdbInfo.EdbInfoId] = tmpEdbInfo.FromTag*/
  158. newEdbInfoIdArr = append(newEdbInfoIdArr, strconv.Itoa(tmpEdbInfo.EdbInfoId))
  159. }
  160. edbInfoIdStr := strings.Join(newEdbInfoIdArr, ",")
  161. oldEdbInfoIdStr, e := GetCalculateEdbInfo(req.EdbInfoId)
  162. if e != nil {
  163. errMsg = "获取数据失败,GetCalculateEdbInfo:" + e.Error()
  164. err = fmt.Errorf("查询关联指标失败")
  165. return
  166. }
  167. if edbInfoIdStr != oldEdbInfoIdStr { // 不需要重复计算
  168. fmt.Printf("edbInfoIdStr:%s,oldEdbInfoIdStr:%s\n", edbInfoIdStr, oldEdbInfoIdStr)
  169. needCalculate = true
  170. }
  171. }
  172. fmt.Printf("needCalculate:%v\n", needCalculate)
  173. if !needCalculate {
  174. return
  175. }
  176. //删除,计算指标关联的,基础指标的关联关系
  177. sql := ` DELETE FROM edb_info_calculate_mapping WHERE edb_info_id = ? `
  178. _, err = to.Raw(sql, edbInfo.EdbInfoId).Exec()
  179. if err != nil {
  180. err = errors.New("删除计算指标关联关系失败,Err:" + err.Error())
  181. return
  182. }
  183. //清空原有数据
  184. tableName := GetEdbDataTableName(edbInfo.Source, edbInfo.SubSource)
  185. sql = ` DELETE FROM ` + tableName + ` WHERE edb_info_id = ? `
  186. _, err = to.Raw(sql, edbInfo.EdbInfoId).Exec()
  187. if err != nil {
  188. return
  189. }
  190. //关联关系
  191. tagMap := make(map[string]int)
  192. relationEdbInfoList := make([]*EdbInfo, 0)
  193. calculateMappingItemList := make([]*EdbInfoCalculateMapping, 0)
  194. for k, v := range req.EdbInfoIdArr {
  195. tmpEdbInfo, tmpErr := GetEdbInfoById(v.EdbInfoId)
  196. if tmpErr != nil {
  197. errMsg = "获取数据失败,GetEdbInfoById:" + tmpErr.Error()
  198. err = tmpErr
  199. return
  200. }
  201. relationEdbInfoList = append(relationEdbInfoList, tmpEdbInfo)
  202. calculateMappingItem := new(EdbInfoCalculateMapping)
  203. calculateMappingItem.CreateTime = time.Now()
  204. calculateMappingItem.ModifyTime = time.Now()
  205. calculateMappingItem.Sort = k
  206. calculateMappingItem.EdbCode = edbInfo.EdbCode
  207. calculateMappingItem.EdbInfoId = edbInfo.EdbInfoId
  208. calculateMappingItem.FromEdbInfoId = tmpEdbInfo.EdbInfoId
  209. calculateMappingItem.FromEdbCode = tmpEdbInfo.EdbCode
  210. calculateMappingItem.FromEdbName = tmpEdbInfo.EdbName
  211. calculateMappingItem.FromSource = tmpEdbInfo.Source
  212. calculateMappingItem.FromSourceName = tmpEdbInfo.SourceName
  213. calculateMappingItem.FromTag = v.FromTag
  214. calculateMappingItem.Source = edbInfo.Source
  215. calculateMappingItem.SourceName = edbInfo.SourceName
  216. calculateMappingItem.FromSubSource = edbInfo.SubSource
  217. calculateMappingItemList = append(calculateMappingItemList, calculateMappingItem)
  218. tagMap[v.FromTag] = v.EdbInfoId
  219. }
  220. _, err = to.InsertMulti(len(calculateMappingItemList), calculateMappingItemList)
  221. if err != nil {
  222. return
  223. }
  224. //计算数据
  225. err = obj.refresh(to, edbInfo.EdbInfoId, edbInfo.Source, edbInfo.SubSource, relationEdbInfoList, edbInfo.EdbCode, edbInfo.Extra, tagMap, edbInfo.EmptyType)
  226. return
  227. }
  228. func (obj CalculateSum) Refresh(params RefreshParams) (err error, errMsg string) {
  229. edbInfo := params.EdbInfo
  230. edbInfoCalculateDetailList, err := GetEdbInfoCalculateDetailList(edbInfo.EdbInfoId)
  231. if err != nil {
  232. return
  233. }
  234. tagMap := make(map[string]int)
  235. relationEdbInfoList := make([]*EdbInfo, 0)
  236. for _, v := range edbInfoCalculateDetailList {
  237. tagMap[v.FromTag] = v.FromEdbInfoId
  238. fromEdbInfo, _ := GetEdbInfoById(v.FromEdbInfoId)
  239. relationEdbInfoList = append(relationEdbInfoList, fromEdbInfo)
  240. }
  241. o := orm.NewOrm()
  242. to, err := o.Begin()
  243. if err != nil {
  244. return
  245. }
  246. defer func() {
  247. if err != nil {
  248. fmt.Println(reflect.TypeOf(obj).Name() + ",Err:" + err.Error())
  249. _ = to.Rollback()
  250. } else {
  251. _ = to.Commit()
  252. }
  253. }()
  254. // 计算数据
  255. err = obj.refresh(to, edbInfo.EdbInfoId, edbInfo.Source, edbInfo.SubSource, relationEdbInfoList, edbInfo.EdbCode, edbInfo.Extra, tagMap, edbInfo.EmptyType)
  256. return
  257. }
  258. func (obj CalculateSum) refresh(to orm.TxOrmer, edbInfoId, source, subSource int, edbInfoIdArr []*EdbInfo, edbCode, extra string, edbInfoTag map[string]int, emptyType int) (err error) {
  259. //to orm.TxOrmer, edbInfoId, source, subSource int, edbInfoIdArr []*EdbInfo, edbCode, extra string, edbInfoTag map[string]int, emptyType int
  260. realSaveDataMap := make(map[string]map[int]float64)
  261. saveDataMap := make(map[string]map[int]float64)
  262. // 最小的结束日期 , 最晚的数据开始日期
  263. var minLatestDate, maxStartDate time.Time
  264. dateList := make([]string, 0) // 最终的日期数据
  265. dateMap := make(map[string]struct{}) // 最终的日期数据
  266. edbInfoIdStr := strconv.Itoa(edbInfoId)
  267. tableName := GetEdbDataTableName(source, subSource)
  268. // 获取多指标求和关联的指标id
  269. dateTagConfig := ""
  270. if extra != "" {
  271. var dateConfig CalculateEdbExtra
  272. err = json.Unmarshal([]byte(extra), &dateConfig)
  273. if err != nil {
  274. err = fmt.Errorf("refreshAllCalculate,extra解析失败,Err:%s", err.Error())
  275. return
  276. }
  277. dateTagConfig = dateConfig.DateTag
  278. }
  279. for edbInfoIndex, v := range edbInfoIdArr {
  280. dataList, err := GetEdbDataListAllByTo(to, v.Source, v.SubSource, FindEdbDataListAllCond{
  281. EdbInfoId: v.EdbInfoId,
  282. }, 1)
  283. if err != nil {
  284. return err
  285. }
  286. for _, dv := range dataList {
  287. if val, ok := realSaveDataMap[dv.DataTime]; ok {
  288. if _, ok := val[v.EdbInfoId]; !ok {
  289. val[v.EdbInfoId] = dv.Value
  290. }
  291. } else {
  292. temp := make(map[int]float64)
  293. temp[v.EdbInfoId] = dv.Value
  294. realSaveDataMap[dv.DataTime] = temp
  295. }
  296. // saveDataMap
  297. if val, ok := saveDataMap[dv.DataTime]; ok {
  298. if _, ok := val[v.EdbInfoId]; !ok {
  299. val[v.EdbInfoId] = dv.Value
  300. }
  301. } else {
  302. temp2 := make(map[int]float64)
  303. temp2[v.EdbInfoId] = dv.Value
  304. saveDataMap[dv.DataTime] = temp2
  305. }
  306. if dateTagConfig == "all" {
  307. if _, ok := dateMap[dv.DataTime]; !ok {
  308. dateList = append(dateList, dv.DataTime)
  309. dateMap[dv.DataTime] = struct{}{}
  310. }
  311. } else if dateTagConfig == "" { // 默认取第一个指标的时间序列
  312. if edbInfoIndex == 0 {
  313. if _, ok := dateMap[dv.DataTime]; !ok {
  314. dateList = append(dateList, dv.DataTime)
  315. dateMap[dv.DataTime] = struct{}{}
  316. }
  317. }
  318. } else {
  319. if eId, ok := edbInfoTag[dateTagConfig]; ok {
  320. if v.EdbInfoId == eId {
  321. if _, ok1 := dateMap[dv.DataTime]; !ok1 {
  322. dateList = append(dateList, dv.DataTime)
  323. dateMap[dv.DataTime] = struct{}{}
  324. }
  325. }
  326. }
  327. }
  328. }
  329. }
  330. // 处理最大日期和最小日期
  331. for _, v := range dateList {
  332. tmpDate, _ := time.ParseInLocation(utils.FormatDate, v, time.Local)
  333. if minLatestDate.IsZero() || tmpDate.After(minLatestDate) {
  334. minLatestDate = tmpDate
  335. }
  336. if maxStartDate.IsZero() || tmpDate.Before(maxStartDate) {
  337. maxStartDate = tmpDate
  338. }
  339. }
  340. //数据处理,将日期内不全的数据做补全
  341. HandleDateSaveDataMap(dateList, maxStartDate, minLatestDate, realSaveDataMap, saveDataMap, edbInfoIdArr, emptyType)
  342. addSql := ` INSERT INTO ` + tableName + ` (edb_info_id,edb_code,data_time,value,create_time,modify_time,data_timestamp) values `
  343. var isAdd bool
  344. //获取指标所有数据
  345. dataList := make([]*EdbData, 0)
  346. dataTableName := GetEdbDataTableName(source, subSource)
  347. sql := `SELECT * FROM %s WHERE edb_info_id=? `
  348. sql = fmt.Sprintf(sql, dataTableName)
  349. _, err = to.Raw(sql, edbInfoId).QueryRows(&dataList)
  350. if err != nil {
  351. return err
  352. }
  353. dataMap := make(map[string]string)
  354. removeDateMap := make(map[string]int) //需要移除的日期
  355. for _, v := range dataList {
  356. dataMap[v.DataTime] = v.Value
  357. removeDateMap[v.DataTime] = 1
  358. }
  359. existDataMap := make(map[string]string)
  360. for sk, sv := range saveDataMap {
  361. if _, ok := dateMap[sk]; !ok {
  362. continue
  363. }
  364. // 当空值处理类型选择了不计算时,只要有一个指标在某个日期没有值(即空值),则计算指标在该日期没有值
  365. if len(sv) != len(edbInfoIdArr) {
  366. continue
  367. }
  368. var calVal float64
  369. for _, value := range sv {
  370. calVal += value
  371. }
  372. // 有计算出来值,那么就从待删除指标中移除
  373. delete(removeDateMap, sk)
  374. saveValue := decimal.NewFromFloat(calVal).Round(4).String()
  375. if existVal, ok := dataMap[sk]; !ok {
  376. dataTime, _ := time.ParseInLocation(utils.FormatDate, sk, time.Local)
  377. timestamp := dataTime.UnixNano() / 1e6
  378. timeStr := fmt.Sprintf("%d", timestamp)
  379. if _, existOk := existDataMap[sk]; !existOk {
  380. addSql += GetAddSql(edbInfoIdStr, edbCode, sk, timeStr, saveValue)
  381. isAdd = true
  382. }
  383. existDataMap[sk] = sk
  384. } else {
  385. existValDecimal, err := decimal.NewFromString(existVal)
  386. existStr := existValDecimal.String()
  387. if existStr != saveValue {
  388. sql := ` UPDATE %s SET value=?,modify_time=NOW() WHERE edb_info_id=? AND data_time=? `
  389. sql = fmt.Sprintf(sql, dataTableName)
  390. _, err = to.Raw(sql, saveValue, edbInfoId, sk).Exec()
  391. if err != nil {
  392. return err
  393. }
  394. }
  395. }
  396. }
  397. if isAdd {
  398. addSql = strings.TrimRight(addSql, ",")
  399. _, err = to.Raw(addSql).Exec()
  400. if err != nil {
  401. fmt.Println("RefreshAllCalculate add Err", err.Error())
  402. return
  403. }
  404. }
  405. if len(removeDateMap) > 0 {
  406. removeDateList := make([]string, 0) //需要移除的日期
  407. for k := range removeDateMap {
  408. removeDateList = append(removeDateList, k)
  409. }
  410. removeDateStr := strings.Join(removeDateList, `","`)
  411. removeDateStr = `"` + removeDateStr + `"`
  412. //如果拼接指标变更了,那么需要删除所有的指标数据
  413. sql := fmt.Sprintf(` DELETE FROM %s WHERE edb_info_id = ? and data_time in (%s) `, tableName, removeDateStr)
  414. _, err = to.Raw(sql, edbInfoId).Exec()
  415. if err != nil {
  416. err = fmt.Errorf("删除计算失败的计算指标数据失败,Err:" + err.Error())
  417. return
  418. }
  419. }
  420. return
  421. }
  422. // GetSource 获取来源编码id
  423. func (obj CalculateSum) GetSource() int {
  424. return utils.DATA_SOURCE_CALCULATE_SUM
  425. }
  426. // GetSourceName 获取来源名称
  427. func (obj CalculateSum) GetSourceName() string {
  428. return utils.DATA_SOURCE_NAME_CALCULATE_SUM
  429. }
  430. // GetEdbType 获取指标类型
  431. func (obj CalculateSum) GetEdbType() int {
  432. return utils.CALCULATE_EDB_TYPE
  433. }