edb_data_rzd.go 7.1 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240
  1. // @Author gmy 2024/9/14 16:13:00
  2. package models
  3. import (
  4. "eta/eta_index_lib/global"
  5. "eta/eta_index_lib/utils"
  6. "fmt"
  7. "strconv"
  8. "strings"
  9. "time"
  10. )
  11. type EdbDataRzd struct {
  12. EdbDataId int `gorm:"column:edb_data_id;type:int(10) UNSIGNED;primaryKey;not null;"`
  13. CreateTime string `gorm:"column:create_time;type:datetime;comment:创建时间;not null;default:CURRENT_TIMESTAMP;"` // 创建时间
  14. ModifyTime string `gorm:"column:modify_time;type:datetime;comment:修改时间;not null;default:CURRENT_TIMESTAMP;"` // 修改时间
  15. EdbInfoId int `gorm:"column:edb_info_id;type:int(10) UNSIGNED;comment:指标id;not null;"` // 指标id
  16. EdbCode string `gorm:"column:edb_code;type:varchar(128);comment:指标编码;not null;"` // 指标编码
  17. DataTime string `gorm:"column:data_time;type:date;comment:数据日期;default:NULL;"` // 数据日期
  18. Value float64 `gorm:"column:value;type:double;comment:数据值;default:NULL;"` // 数据值
  19. DataTimestamp uint64 `gorm:"column:data_timestamp;type:bigint(20) UNSIGNED;comment:数据日期时间戳;not null;default:0;"` // 数据日期时间戳
  20. }
  21. // ConvertTimeStr
  22. // @Description: 转成需要输出的格式
  23. // @receiver m
  24. func (m *EdbDataRzd) ConvertTimeStr() {
  25. m.DataTime = utils.GormDateStrToDateStr(m.DataTime)
  26. m.CreateTime = utils.GormDateStrToDateTimeStr(m.DataTime)
  27. m.CreateTime = utils.GormDateStrToDateTimeStr(m.DataTime)
  28. return
  29. }
  30. // GetRzdEdbDataByIndexCodeAndDataTime 根据指标编码和时间获取指标库数据
  31. func GetRzdEdbDataByIndexCodeAndDataTime(indexCode string, dataTime string) (items []EdbDataRzd, err error) {
  32. sql := `SELECT * FROM edb_data_rzd WHERE edb_code=? AND data_time=?`
  33. err = global.DEFAULT_DB.Raw(sql, indexCode, dataTime).Find(&items).Error
  34. if utils.IsErrNoRow(err) {
  35. return nil, nil
  36. }
  37. for _, v := range items {
  38. v.ConvertTimeStr()
  39. }
  40. return
  41. }
  42. // UpdateRzdEdbDataById 更新指标库数据 须根据指标编码和日期更新 仅适合月度数据
  43. func UpdateRzdEdbDataById(id int, value float64) (err error) {
  44. sql := `UPDATE edb_data_rzd SET value=? WHERE edb_data_id=?`
  45. err = global.DEFAULT_DB.Exec(sql, value, id).Error
  46. return
  47. }
  48. // UpdateRzdEdbDataByIndexCodeAndDataTime 根据指标编码和时间更新数据
  49. func UpdateRzdEdbDataByIndexCodeAndDataTime(indexCode int, dataTime string, value float64) (err error) {
  50. sql := `UPDATE edb_data_rzd SET value=? WHERE edb_info_id=? AND data_time=?`
  51. err = global.DEFAULT_DB.Exec(sql, value, indexCode, dataTime).Error
  52. return
  53. }
  54. // AddRzdEdbDataList 新增指标库数据
  55. func AddRzdEdbDataList(items []EdbDataRzd) (err error) {
  56. err = global.DEFAULT_DB.CreateInBatches(items, utils.MultiAddNum).Error
  57. return
  58. }
  59. // AddEdbDataFromRzd 新增指标数据
  60. func AddEdbDataFromRzd(edbCode string) (err error) {
  61. o := global.DEFAULT_DB
  62. var condition string
  63. var pars []interface{}
  64. if edbCode != "" {
  65. condition += " AND index_code = ? "
  66. pars = append(pars, edbCode)
  67. }
  68. dataAll, err := GetBaseFromRzdDataByCondition(condition, pars)
  69. if err != nil {
  70. return err
  71. }
  72. dataLen := len(dataAll)
  73. existMap := make(map[string]string)
  74. if dataLen > 0 {
  75. var isAdd bool
  76. addSql := ` INSERT INTO edb_data_rzd (edb_info_id,edb_code,data_time,value,create_time,modify_time,data_timestamp) values `
  77. for i := 0; i < dataLen; i++ {
  78. item := dataAll[i]
  79. eDate := item.DataTime
  80. sValue := utils.SubFloatToString(item.Value, 4)
  81. if sValue != "" {
  82. if _, ok := existMap[eDate]; !ok {
  83. dataTime, err := time.ParseInLocation(utils.FormatDate, eDate, time.Local)
  84. if err != nil {
  85. return err
  86. }
  87. timestamp := dataTime.UnixNano() / 1e6
  88. timeStr := fmt.Sprintf("%d", timestamp)
  89. addSql += GetAddSql("0", edbCode, eDate, timeStr, sValue)
  90. isAdd = true
  91. }
  92. }
  93. existMap[eDate] = eDate
  94. }
  95. if isAdd {
  96. addSql = strings.TrimRight(addSql, ",")
  97. utils.FileLog.Info("addSql:" + addSql)
  98. err = o.Exec(addSql).Error
  99. if err != nil {
  100. return err
  101. }
  102. }
  103. }
  104. return
  105. }
  106. func RefreshEdbDataFromRzd(edbInfoId int, edbCode, startDate string) (err error) {
  107. source := utils.DATA_SOURCE_RZD
  108. subSource := utils.DATA_SUB_SOURCE_EDB
  109. o := global.DEFAULT_DB
  110. if err != nil {
  111. return
  112. }
  113. edbInfoIdStr := strconv.Itoa(edbInfoId)
  114. //计算数据
  115. var condition string
  116. var pars []interface{}
  117. if edbCode != "" {
  118. condition += " AND index_code=? "
  119. pars = append(pars, edbCode)
  120. }
  121. if startDate != "" {
  122. condition += " AND data_time>=? "
  123. pars = append(pars, startDate)
  124. }
  125. dataList, err := GetBaseFromRzdDataByCondition(condition, pars)
  126. if err != nil {
  127. return
  128. }
  129. // 真实数据的最大日期 , 插入规则配置的日期
  130. var realDataMaxDate, edbDataInsertConfigDate time.Time
  131. var edbDataInsertConfig *EdbDataInsertConfig
  132. var isFindConfigDateRealData bool //是否找到配置日期的实际数据的值
  133. {
  134. edbDataInsertConfig, err = GetEdbDataInsertConfigByEdbId(edbInfoId)
  135. if err != nil && !utils.IsErrNoRow(err) {
  136. return
  137. }
  138. if edbDataInsertConfig != nil {
  139. edbDataInsertConfigDate = edbDataInsertConfig.Date
  140. }
  141. }
  142. var existCondition string
  143. var existPars []interface{}
  144. existCondition += " AND edb_info_id=? "
  145. existPars = append(existPars, edbInfoId)
  146. if startDate != "" {
  147. existCondition += " AND data_time>=? "
  148. existPars = append(existPars, startDate)
  149. }
  150. //获取指标所有数据
  151. existList, err := GetEdbDataByCondition(source, subSource, existCondition, existPars)
  152. if err != nil {
  153. return err
  154. }
  155. existMap := make(map[string]*EdbInfoSearchData)
  156. for _, v := range existList {
  157. existMap[v.DataTime] = v
  158. }
  159. addSql := ` INSERT INTO edb_data_rzd(edb_info_id,edb_code,data_time,value,create_time,modify_time,data_timestamp) values `
  160. var isAdd bool
  161. addMap := make(map[string]string)
  162. for _, v := range dataList {
  163. item := v
  164. eDate := item.DataTime
  165. sValue := utils.SubFloatToString(item.Value, 4)
  166. dataTime, err := time.ParseInLocation(utils.FormatDate, eDate, time.Local)
  167. if err != nil {
  168. return err
  169. }
  170. if findItem, ok := existMap[v.DataTime]; !ok {
  171. if sValue != "" {
  172. timestamp := dataTime.UnixNano() / 1e6
  173. timeStr := fmt.Sprintf("%d", timestamp)
  174. saveValue := sValue
  175. if _, addOk := addMap[eDate]; !addOk {
  176. addSql += GetAddSql(edbInfoIdStr, edbCode, eDate, timeStr, saveValue)
  177. isAdd = true
  178. }
  179. }
  180. } else {
  181. if findItem != nil && utils.SubFloatToString(findItem.Value, 4) != sValue {
  182. err = ModifyEdbDataById(source, subSource, findItem.EdbDataId, sValue)
  183. if err != nil {
  184. return err
  185. }
  186. }
  187. }
  188. addMap[v.DataTime] = v.DataTime
  189. // 下面代码主要目的是处理掉手动插入的数据判断
  190. {
  191. if realDataMaxDate.IsZero() || dataTime.After(realDataMaxDate) {
  192. realDataMaxDate = dataTime
  193. }
  194. if edbDataInsertConfigDate.IsZero() || dataTime.Equal(edbDataInsertConfigDate) {
  195. isFindConfigDateRealData = true
  196. }
  197. }
  198. }
  199. // 处理手工数据补充的配置
  200. HandleConfigInsertEdbData(realDataMaxDate, edbDataInsertConfig, edbInfoId, source, subSource, existMap, isFindConfigDateRealData)
  201. if isAdd {
  202. addSql = strings.TrimRight(addSql, ",")
  203. err = o.Exec(addSql).Error
  204. if err != nil {
  205. return err
  206. }
  207. }
  208. return
  209. }