base_from_jiayue.go 8.2 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228
  1. package models
  2. import (
  3. "eta/eta_index_lib/utils"
  4. "fmt"
  5. "github.com/beego/beego/v2/client/orm"
  6. "strconv"
  7. "strings"
  8. "time"
  9. )
  10. // BridgeJiaYueIndexDataParams 桥接服务-获取嘉悦指标数据入参
  11. type BridgeJiaYueIndexDataParams struct {
  12. IndexCode string `json:"index_code" form:"index_code" description:"指标编码"`
  13. SourceExtend string `json:"source_extend" form:"source_extend" description:"来源"`
  14. StartDate string `json:"start_date" form:"start_date" description:"开始日期"`
  15. EndDate string `json:"end_date" form:"end_date" description:"结束日期"`
  16. IndexCodeRequired int `json:"index_code_required" form:"index_code_required" description:"指标编码是否必填: 0-否; 1-是"`
  17. }
  18. // BridgeJiaYueResultIndexData 桥接服务-获取嘉悦指标数据响应体
  19. type BridgeJiaYueResultIndexData struct {
  20. Code int `json:"code" description:"状态码"`
  21. Msg string `json:"msg" description:"提示信息"`
  22. Data BridgeJiaYueIndexAndData `json:"data" description:"返回数据"`
  23. }
  24. // BridgeJiaYueIndexAndData 桥接服务-嘉悦指标和数据
  25. type BridgeJiaYueIndexAndData struct {
  26. Id int `description:"指标自增ID" json:"id"`
  27. IndexCode string `description:"指标编码" json:"index_code"`
  28. IndexName string `description:"指标名称" json:"index_name"`
  29. SourceType string `description:"指标来源" json:"source_type"`
  30. Unit string `description:"单位" json:"unit"`
  31. Frequency string `description:"频度" json:"frequency"`
  32. LastDate time.Time `description:"指标最新时间" json:"last_date"`
  33. LastUpdateTime time.Time `description:"最新更新时间" json:"last_update_time"`
  34. Status int `description:"指标状态" json:"status"`
  35. IndexData []BridgeJiaYueIndexData `description:"指标数据" json:"index_data"`
  36. MenuData BridgeJiaYueIndexMenuData `description:"指标目录信息" json:"menu_data"`
  37. }
  38. // BridgeJiaYueIndexData 桥接服务-嘉悦指标数据
  39. type BridgeJiaYueIndexData struct {
  40. Val float64 `json:"val"`
  41. DataTime time.Time `json:"data_time"`
  42. UpdateTime time.Time `json:"update_time"`
  43. }
  44. // BridgeJiaYueIndexMenuData 桥接服务-嘉悦指标目录信息
  45. type BridgeJiaYueIndexMenuData struct {
  46. Id int `description:"目录ID" json:"id"`
  47. Type string `description:"目录类型" json:"type"`
  48. Code string `description:"目录编码" json:"code"`
  49. Name string `description:"目录名称" json:"name"`
  50. Icon string `description:"目录图标" json:"icon"`
  51. Sort int `description:"排序" json:"sort"`
  52. ParentId int `description:"父级目录ID" json:"parent_id"`
  53. ParentName string `description:"父级目录名称" json:"parent_name"`
  54. Path string `description:"目录全路径" json:"path"`
  55. }
  56. // BridgeJiaYueIndexMenuWithLevel 桥接服务-嘉悦指标目录带层级
  57. type BridgeJiaYueIndexMenuWithLevel struct {
  58. Level int `description:"层级"`
  59. Menu BridgeJiaYueIndexMenuData
  60. }
  61. // BridgeJiaYueResultNewIndexData 桥接服务-获取嘉悦增量指标数据响应体
  62. type BridgeJiaYueResultNewIndexData struct {
  63. Code int `json:"code" description:"状态码"`
  64. Msg string `json:"msg" description:"提示信息"`
  65. Data []BridgeJiaYueIndexAndData `json:"data" description:"返回数据"`
  66. }
  67. // BridgeJiaYueResultMenuListData 桥接服务-获取嘉悦指标目录数据响应体
  68. type BridgeJiaYueResultMenuListData struct {
  69. Code int `json:"code" description:"状态码"`
  70. Msg string `json:"msg" description:"提示信息"`
  71. Data []BridgeJiaYueIndexMenuData `json:"data" description:"返回数据"`
  72. }
  73. // AddEdbDataFromJiaYue 新增嘉悦指标数据
  74. func AddEdbDataFromJiaYue(tableName, edbCode string, dataList []BridgeJiaYueIndexData) (err error) {
  75. if tableName == "" {
  76. err = fmt.Errorf("数据表名为空")
  77. return
  78. }
  79. if edbCode == "" {
  80. err = fmt.Errorf("指标编码为空")
  81. return
  82. }
  83. if len(dataList) == 0 {
  84. return
  85. }
  86. sql := fmt.Sprintf(`INSERT INTO %s(edb_info_id, edb_code, data_time, value, create_time, modify_time, data_timestamp) VALUES `, tableName)
  87. for _, v := range dataList {
  88. val := utils.SubFloatToString(v.Val, 20)
  89. stamp := fmt.Sprint(v.DataTime.UnixMilli())
  90. sql += GetAddSql("0", edbCode, v.DataTime.Format(utils.FormatDate), stamp, val)
  91. }
  92. sql = strings.TrimRight(sql, ",")
  93. // 新增入库
  94. o := orm.NewOrm()
  95. _, e := o.Raw(sql).Exec()
  96. if e != nil {
  97. err = fmt.Errorf("insert data err: %s", e.Error())
  98. }
  99. return
  100. }
  101. // RefreshEdbDataFromJiaYue 刷新嘉悦指标数据
  102. func RefreshEdbDataFromJiaYue(source, subSource, edbInfoId int, tableName, edbCode, startDate string, dataList []BridgeJiaYueIndexData) (err error) {
  103. if source <= 0 {
  104. err = fmt.Errorf("指标来源有误")
  105. return
  106. }
  107. if edbInfoId <= 0 {
  108. err = fmt.Errorf("指标ID有误")
  109. return
  110. }
  111. if tableName == "" {
  112. err = fmt.Errorf("数据表名为空")
  113. return
  114. }
  115. // 真实数据的最大日期, 插入规则配置的日期
  116. var realDataMaxDate, edbDataInsertConfigDate time.Time
  117. var edbDataInsertConfig *EdbDataInsertConfig
  118. var isFindConfigDateRealData bool //是否找到配置日期的实际数据的值
  119. {
  120. conf, e := GetEdbDataInsertConfigByEdbId(edbInfoId)
  121. if e != nil && e.Error() != utils.ErrNoRow() {
  122. return
  123. }
  124. edbDataInsertConfig = conf
  125. if edbDataInsertConfig != nil {
  126. edbDataInsertConfigDate = edbDataInsertConfig.Date
  127. }
  128. }
  129. // 获取已有数据
  130. cond := ` AND edb_info_id = ?`
  131. pars := make([]interface{}, 0)
  132. pars = append(pars, edbInfoId)
  133. var startDateTime time.Time
  134. if startDate != "" {
  135. cond += ` AND data_time >= ?`
  136. pars = append(pars, startDate)
  137. startDateTime, _ = time.ParseInLocation(utils.FormatDate, startDate, time.Local)
  138. }
  139. existList, e := GetEdbDataByCondition(source, subSource, cond, pars)
  140. if e != nil {
  141. err = fmt.Errorf("获取指标已有数据失败, Err: %s", e.Error())
  142. return
  143. }
  144. existMap := make(map[string]*EdbInfoSearchData)
  145. for _, v := range existList {
  146. existMap[v.DataTime] = v
  147. }
  148. // 比对数据
  149. hasNew := false
  150. strEdbInfoId := strconv.Itoa(edbInfoId)
  151. addExists := make(map[string]bool)
  152. sqlInsert := fmt.Sprintf(`INSERT INTO %s(edb_info_id, edb_code, data_time, value, create_time, modify_time, data_timestamp) VALUES `, tableName)
  153. for _, v := range dataList {
  154. val := utils.SubFloatToString(v.Val, 30)
  155. stamp := fmt.Sprint(v.DataTime.UnixMilli())
  156. dataTime := v.DataTime.Format(utils.FormatDate)
  157. // 如果传入的开始时间是空的, 且当前数据日期早于传入的开始日期, 那么需要判断下当前日期的数据是否存在
  158. if !startDateTime.IsZero() && v.DataTime.Before(startDateTime) {
  159. t, e := GetEdbDataByDate(source, subSource, edbCode, dataTime)
  160. if e == nil && t != nil {
  161. existMap[t.DataTime] = t
  162. }
  163. }
  164. // 下面代码主要目的是处理掉手动插入的数据判断
  165. {
  166. if realDataMaxDate.IsZero() || v.DataTime.After(realDataMaxDate) {
  167. realDataMaxDate = v.DataTime
  168. }
  169. if edbDataInsertConfigDate.IsZero() || v.DataTime.Equal(edbDataInsertConfigDate) {
  170. isFindConfigDateRealData = true
  171. }
  172. }
  173. // 新增数据
  174. exist, ok := existMap[dataTime]
  175. if !ok {
  176. // 不在历史数据中且与新增中的数据不重复
  177. if _, o := addExists[dataTime]; !o {
  178. hasNew = true
  179. sqlInsert += GetAddSql(strEdbInfoId, edbCode, dataTime, stamp, val)
  180. addExists[dataTime] = true
  181. }
  182. continue
  183. }
  184. // 更新数据
  185. if exist != nil && utils.SubFloatToString(exist.Value, 30) != val {
  186. if e = ModifyEdbDataById(source, subSource, exist.EdbDataId, val); e != nil {
  187. err = fmt.Errorf("modify edb data err: %s", e.Error())
  188. return
  189. }
  190. }
  191. }
  192. // 处理手工数据补充的配置
  193. HandleConfigInsertEdbData(realDataMaxDate, edbDataInsertConfig, edbInfoId, source, subSource, existMap, isFindConfigDateRealData)
  194. // 执行新增
  195. if !hasNew {
  196. return
  197. }
  198. o := orm.NewOrm()
  199. sqlInsert = strings.TrimRight(sqlInsert, ",")
  200. _, e = o.Raw(sqlInsert).Exec()
  201. if e != nil {
  202. err = fmt.Errorf("insert edb data err: %s", e.Error())
  203. return
  204. }
  205. return
  206. }