base_from_adjust.go 12 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427
  1. package models
  2. import (
  3. "errors"
  4. "eta_gn/eta_index_lib/global"
  5. "eta_gn/eta_index_lib/utils"
  6. "fmt"
  7. "github.com/beego/beego/v2/client/orm"
  8. "strconv"
  9. "strings"
  10. "time"
  11. )
  12. // EdbDataAdjust 数据调整指标数据结构体
  13. type EdbDataAdjust struct {
  14. EdbDataId int `orm:"column(edb_data_id);pk"`
  15. EdbInfoId int
  16. EdbCode string
  17. DataTime string
  18. Value float64
  19. CreateTime time.Time
  20. ModifyTime time.Time
  21. DataTimestamp int64
  22. }
  23. // AddAdjustEdbData 保存数据调整请求的数据的参数
  24. type AddAdjustEdbData struct {
  25. Date string `description:"数据日期"`
  26. TimestampStr string `description:"日期时间戳"`
  27. Value float64 `description:"数据值"`
  28. }
  29. // SaveAdjustEdb 保存数据调整指标
  30. func SaveAdjustEdb(req SaveAdjustEdbReq) (edbInfo *EdbInfo, err error, errMsg string) {
  31. errMsg = `添加指标失败`
  32. //o := orm.NewOrm()
  33. //to, err := o.Begin()
  34. to := global.DEFAULT_DmSQL.Begin()
  35. if err != nil {
  36. return
  37. }
  38. defer func() {
  39. if err != nil {
  40. _ = to.Rollback()
  41. } else {
  42. _ = to.Commit()
  43. }
  44. }()
  45. var edbCode string
  46. if req.EdbInfoId <= 0 {
  47. //指标code生成
  48. //生成指标编码
  49. edbCode, err = utils.GenerateEdbCode(1, "")
  50. if err != nil {
  51. err = errors.New("指标编码生成失败,Err:" + err.Error())
  52. return
  53. }
  54. timestamp := strconv.FormatInt(time.Now().UnixNano(), 10)
  55. uniqueCode := utils.MD5(utils.DATA_PREFIX + "_" + timestamp)
  56. edbInfo = &EdbInfo{
  57. Source: utils.DATA_SOURCE_CALCULATE_ADJUST,
  58. SourceName: "数据调整",
  59. EdbCode: edbCode,
  60. EdbName: utils.TrimStr(req.EdbName),
  61. EdbNameSource: utils.TrimStr(req.EdbName),
  62. Frequency: utils.TrimStr(req.Frequency),
  63. Unit: utils.TrimStr(req.Unit),
  64. ClassifyId: req.ClassifyId,
  65. SysUserId: req.AdminId,
  66. SysUserRealName: req.AdminName,
  67. CreateTime: time.Now(),
  68. ModifyTime: time.Now(),
  69. UniqueCode: uniqueCode,
  70. EdbType: 2,
  71. Sort: GetAddEdbMaxSortByClassifyId(req.ClassifyId, utils.EDB_INFO_TYPE),
  72. }
  73. //newEdbInfoId, tmpErr := to.Insert(edbInfo)
  74. tmpErr := to.Create(edbInfo).Error
  75. if tmpErr != nil {
  76. err = tmpErr
  77. return
  78. }
  79. //edbInfo.EdbInfoId = int(newEdbInfoId)
  80. // 获取来源指标
  81. fromEdbInfo, tmpErr := GetEdbInfoById(req.FromEdbInfoId)
  82. if tmpErr != nil {
  83. err = tmpErr
  84. errMsg = "获取来源指标信息失败"
  85. return
  86. }
  87. //关联关系
  88. {
  89. calculateMappingItem := new(EdbInfoCalculateMapping)
  90. calculateMappingItem.CreateTime = time.Now()
  91. calculateMappingItem.ModifyTime = time.Now()
  92. calculateMappingItem.Sort = 1
  93. calculateMappingItem.EdbCode = edbCode
  94. calculateMappingItem.EdbInfoId = edbInfo.EdbInfoId
  95. calculateMappingItem.FromEdbInfoId = fromEdbInfo.EdbInfoId
  96. calculateMappingItem.FromEdbCode = fromEdbInfo.EdbCode
  97. calculateMappingItem.FromEdbName = fromEdbInfo.EdbName
  98. calculateMappingItem.FromSource = fromEdbInfo.Source
  99. calculateMappingItem.FromSourceName = fromEdbInfo.SourceName
  100. calculateMappingItem.FromTag = ""
  101. calculateMappingItem.Source = edbInfo.Source
  102. calculateMappingItem.SourceName = edbInfo.SourceName
  103. //_, err = to.Insert(calculateMappingItem)
  104. err = to.Create(calculateMappingItem).Error
  105. if err != nil {
  106. return
  107. }
  108. }
  109. //相关配置
  110. {
  111. var fromEdbEndDate time.Time
  112. if fromEdbInfo.EndDate != `0000-00-00` {
  113. fromEdbEndDate, _ = time.Parse(utils.FormatDate, fromEdbInfo.EndDate)
  114. }
  115. edbAdjustConf := &EdbAdjustConf{
  116. EdbInfoId: edbInfo.EdbInfoId,
  117. SourceEndDate: fromEdbEndDate,
  118. ModifyTime: time.Now(),
  119. CreateTime: time.Now(),
  120. }
  121. //_, err = to.Insert(edbAdjustConf)
  122. err = to.Create(edbAdjustConf).Error
  123. if err != nil {
  124. return
  125. }
  126. }
  127. } else {
  128. edbInfo, err = GetEdbInfoById(req.EdbInfoId)
  129. if err != nil {
  130. return
  131. }
  132. // 更新指标信息
  133. edbInfo.EdbName = utils.TrimStr(req.EdbName)
  134. edbInfo.EdbNameSource = utils.TrimStr(req.EdbName)
  135. edbInfo.Frequency = utils.TrimStr(req.Frequency)
  136. edbInfo.Unit = utils.TrimStr(req.Unit)
  137. edbInfo.ClassifyId = req.ClassifyId
  138. edbInfo.ModifyTime = time.Now()
  139. //_, err = to.Update(edbInfo, "EdbName", "EdbNameSource", "Frequency", "Unit", "ClassifyId", "ModifyTime")
  140. err = to.Model(edbInfo).Updates(edbInfo).Error
  141. if err != nil {
  142. return
  143. }
  144. // 指标code
  145. edbCode = edbInfo.EdbCode
  146. // 清除数据
  147. dataTableName := GetEdbDataTableName(utils.DATA_SOURCE_CALCULATE_ADJUST, edbInfo.SubSource)
  148. deleteSql := ` DELETE FROM %s WHERE edb_info_id=? `
  149. deleteSql = fmt.Sprintf(deleteSql, dataTableName)
  150. //_, err = to.Raw(deleteSql, req.EdbInfoId).Exec()
  151. err = to.Exec(deleteSql, req.EdbInfoId).Error
  152. if err != nil {
  153. return
  154. }
  155. //相关配置
  156. {
  157. // 获取来源指标
  158. calculateMapping, tmpErr := GetEdbInfoCalculateMappingDetail(edbInfo.EdbInfoId)
  159. if tmpErr != nil {
  160. errMsg = `关联指标获取异常`
  161. err = tmpErr
  162. return
  163. }
  164. fromEdbInfo, tmpErr := GetEdbInfoById(calculateMapping.FromEdbInfoId)
  165. if tmpErr != nil {
  166. errMsg = `来源指标不存在`
  167. err = tmpErr
  168. return
  169. }
  170. var fromEdbEndDate time.Time
  171. if fromEdbInfo.EndDate != `0000-00-00` {
  172. fromEdbEndDate, _ = time.Parse(utils.FormatDate, fromEdbInfo.EndDate)
  173. }
  174. // 获取之前的配置
  175. var edbAdjustConf *EdbAdjustConf
  176. sql := ` SELECT * FROM edb_adjust_conf WHERE edb_info_id=? `
  177. //err = o.Raw(sql, edbInfo.EdbInfoId).QueryRow(&edbAdjustConf)
  178. err = to.Raw(sql, edbInfo.EdbInfoId).Scan(&edbAdjustConf).Error
  179. if err != nil {
  180. errMsg = "获取指标配置失败"
  181. return
  182. }
  183. // 修改配置的最晚日期
  184. edbAdjustConf.SourceEndDate = fromEdbEndDate
  185. edbAdjustConf.ModifyTime = time.Now()
  186. //_, err = to.Update(edbAdjustConf, "SourceEndDate", "ModifyTime")
  187. err = to.Model(edbAdjustConf).Updates(edbAdjustConf).Error
  188. if err != nil {
  189. errMsg = "更新指标配置失败"
  190. return
  191. }
  192. }
  193. }
  194. dataList := req.DataList
  195. dataObj := make(map[string]AddAdjustEdbData)
  196. for _, item := range dataList {
  197. currentDate, tmpErr := time.ParseInLocation(utils.FormatDate, item.Date, time.Local)
  198. if tmpErr != nil {
  199. err = tmpErr
  200. return
  201. }
  202. timestamp := currentDate.UnixNano() / 1e6
  203. timestampStr := fmt.Sprintf("%d", timestamp)
  204. tmpVal, ok := dataObj[item.Date]
  205. if !ok {
  206. tmpVal = AddAdjustEdbData{
  207. Date: item.Date,
  208. TimestampStr: timestampStr,
  209. Value: item.Value,
  210. }
  211. } else {
  212. tmpVal.Value = item.Value
  213. }
  214. dataObj[item.Date] = tmpVal
  215. }
  216. var isAdd bool
  217. dataTableName := GetEdbDataTableName(edbInfo.Source, edbInfo.SubSource)
  218. addSql := ` INSERT INTO ` + dataTableName + `(edb_info_id,edb_code,data_time,value,create_time,modify_time,data_timestamp) values `
  219. for _, item := range dataObj {
  220. //值
  221. val := item.Value
  222. saveVal := utils.SubFloatToString(val, 20)
  223. addSql += GetAddSql(fmt.Sprint(edbInfo.EdbInfoId), edbCode, item.Date, item.TimestampStr, saveVal)
  224. isAdd = true
  225. }
  226. if isAdd {
  227. addSql = strings.TrimRight(addSql, ",")
  228. //_, err = to.Raw(addSql).Exec()
  229. err = to.Exec(addSql).Error
  230. if err != nil {
  231. return
  232. }
  233. }
  234. return
  235. }
  236. // RefreshAllAdjustEdb 刷新所有 数据调整指标
  237. func RefreshAllAdjustEdb(edbInfo *EdbInfo, fromEdbInfo *EdbInfo) (err error) {
  238. //o := orm.NewOrm()
  239. //to, err := o.Begin()
  240. to := global.DEFAULT_DmSQL
  241. if err != nil {
  242. return
  243. }
  244. defer func() {
  245. if err != nil {
  246. fmt.Println("RefreshAllPythonEdb,Err:" + err.Error())
  247. _ = to.Rollback()
  248. } else {
  249. _ = to.Commit()
  250. }
  251. }()
  252. // 当前数据表名
  253. dataTableName := GetEdbDataTableName(edbInfo.Source, edbInfo.SubSource)
  254. // 获取之前的配置
  255. var edbAdjustConf *EdbAdjustConf
  256. sql := ` SELECT * FROM edb_adjust_conf WHERE edb_info_id=? `
  257. //err = o.Raw(sql, edbInfo.EdbInfoId).QueryRow(&edbAdjustConf)
  258. err = to.Raw(sql, edbInfo.EdbInfoId).Scan(&edbAdjustConf).Error
  259. if err != nil {
  260. err = errors.New("获取指标配置失败")
  261. return
  262. }
  263. // 查询关联指标的数据
  264. dataList, err := GetEdbDataListAllByTo(to, fromEdbInfo.Source, fromEdbInfo.SubSource, FindEdbDataListAllCond{
  265. EdbInfoId: fromEdbInfo.EdbInfoId,
  266. StartDataTime: edbAdjustConf.SourceEndDate.Format(utils.FormatDate),
  267. StartDataTimeCond: ">",
  268. }, 1)
  269. if err != nil {
  270. return err
  271. }
  272. // 已经入库的日期数据
  273. existDataMap := make(map[string]float64)
  274. {
  275. existDataList, tmpErr := GetEdbDataListAllByTo(to, edbInfo.Source, edbInfo.SubSource, FindEdbDataListAllCond{
  276. EdbInfoId: edbInfo.EdbInfoId,
  277. StartDataTime: edbAdjustConf.SourceEndDate.Format(utils.FormatDate),
  278. StartDataTimeCond: ">",
  279. }, 1)
  280. if tmpErr != nil {
  281. err = tmpErr
  282. return
  283. }
  284. for _, v := range existDataList {
  285. existDataMap[v.DataTime] = v.Value
  286. }
  287. }
  288. var isAdd bool
  289. addSql := ` INSERT INTO ` + dataTableName + `(edb_info_id,edb_code,data_time,value,create_time,modify_time,data_timestamp) values `
  290. for _, item := range dataList {
  291. currDay := item.DataTime
  292. //值
  293. val := item.Value
  294. saveVal := utils.SubFloatToString(val, 20)
  295. if existVal, ok := existDataMap[currDay]; !ok {
  296. //格式化时间
  297. currentDate, tmpErr := time.Parse(utils.FormatDate, item.DataTime)
  298. if tmpErr != nil {
  299. err = tmpErr
  300. return
  301. }
  302. timestamp := currentDate.UnixNano() / 1e6
  303. timestampStr := fmt.Sprintf("%d", timestamp)
  304. addSql += GetAddSql(fmt.Sprint(edbInfo.EdbInfoId), edbInfo.EdbCode, item.DataTime, timestampStr, saveVal)
  305. isAdd = true
  306. } else {
  307. if existVal != item.Value {
  308. sql = ` UPDATE %s SET value=?,modify_time=NOW() WHERE edb_info_id=? AND data_time=? `
  309. sql = fmt.Sprintf(sql, dataTableName)
  310. //_, err = to.Raw(sql, saveVal, edbInfo.EdbInfoId, currDay).Exec()
  311. err = to.Exec(sql, saveVal, edbInfo.EdbInfoId, currDay).Error
  312. if err != nil {
  313. return err
  314. }
  315. }
  316. }
  317. }
  318. if isAdd {
  319. addSql = strings.TrimRight(addSql, ",")
  320. //_, err = to.Raw(addSql).Exec()
  321. err = to.Exec(addSql).Error
  322. if err != nil {
  323. return
  324. }
  325. // 变更数据调整配置表的实际结束日期
  326. {
  327. var fromEdbEndDate time.Time
  328. if fromEdbInfo.EndDate != `0000-00-00` {
  329. fromEdbEndDate, _ = time.Parse(utils.FormatDate, fromEdbInfo.EndDate)
  330. }
  331. edbAdjustConf.SourceEndDate = fromEdbEndDate
  332. edbAdjustConf.ModifyTime = time.Now()
  333. //_, err = to.Update(edbAdjustConf, "SourceEndDate", "ModifyTime")
  334. err = to.Model(edbAdjustConf).Updates(edbAdjustConf).Error
  335. }
  336. }
  337. return
  338. }
  339. // SaveAdjustEdbReq 保存数据调整请求参数(请求指标服务)
  340. type SaveAdjustEdbReq struct {
  341. AdminId int `description:"添加人id"`
  342. AdminName string `description:"添加人名称"`
  343. EdbInfoId int `description:"指标id"`
  344. FromEdbInfoId int `description:"来源指标id"`
  345. EdbName string `description:"指标名称"`
  346. Frequency string `description:"频度"`
  347. Unit string `description:"单位"`
  348. ClassifyId int `description:"分类id"`
  349. DataList []SaveAdjustEdbDataReq `description:"指标对应的数据值"`
  350. }
  351. // SaveAdjustEdbDataReq 保存数据调整请求的数据的参数
  352. type SaveAdjustEdbDataReq struct {
  353. Date string `description:"数据日期"`
  354. Value float64 `description:"数据值"`
  355. }
  356. // FixData 修复数据
  357. func FixData() {
  358. var list []*EdbInfo
  359. o := orm.NewOrm()
  360. sql := ` SELECT * FROM edb_info WHERE source=? `
  361. _, err := o.Raw(sql, 40).QueryRows(&list)
  362. for _, edbInfo := range list {
  363. // 获取来源指标
  364. calculateMapping, tmpErr := GetEdbInfoCalculateMappingDetail(edbInfo.EdbInfoId)
  365. if tmpErr != nil {
  366. fmt.Println(edbInfo.EdbInfoId, "关联指标获取异常;err:", tmpErr)
  367. continue
  368. }
  369. fromEdbInfo, tmpErr := GetEdbInfoById(calculateMapping.FromEdbInfoId)
  370. if tmpErr != nil {
  371. fmt.Println(edbInfo.EdbInfoId, "来源指标不存在;err:", tmpErr)
  372. continue
  373. }
  374. var fromEdbEndDate time.Time
  375. if fromEdbInfo.EndDate != `0000-00-00` {
  376. fromEdbEndDate, _ = time.Parse(utils.FormatDate, fromEdbInfo.EndDate)
  377. }
  378. // 获取之前的配置
  379. edbAdjustConf := &EdbAdjustConf{
  380. EdbInfoId: edbInfo.EdbInfoId,
  381. SourceEndDate: fromEdbEndDate,
  382. ModifyTime: time.Now(),
  383. CreateTime: time.Now(),
  384. }
  385. _, err = o.Insert(edbAdjustConf)
  386. if err != nil {
  387. fmt.Println(edbInfo.EdbInfoId, "添加失败;err:", tmpErr)
  388. continue
  389. }
  390. }
  391. fmt.Println("end")
  392. }