base_from_sci_hq.go 8.1 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273
  1. package models
  2. import (
  3. "eta/eta_index_lib/utils"
  4. "fmt"
  5. "strconv"
  6. "strings"
  7. "time"
  8. "github.com/beego/beego/v2/client/orm"
  9. )
  10. func GetBaseFromSciHqDataByCondition(condition string, pars []interface{}) (list []*BaseFromSciHqData, err error) {
  11. o := orm.NewOrm()
  12. sql := `SELECT * FROM base_from_sci_hq_data WHERE 1=1 `
  13. if condition != "" {
  14. sql += condition
  15. }
  16. _, err = o.Raw(sql, pars).QueryRows(&list)
  17. return
  18. }
  19. func GetBaseFromSciHqDataByIndexCode(indexCode string) (list []*BaseFromSciHqData, err error) {
  20. o := orm.NewOrm()
  21. sql := `SELECT * FROM base_from_sci_hq_data WHERE 1=1 AND index_code=?`
  22. _, err = o.Raw(sql, indexCode).QueryRows(&list)
  23. return
  24. }
  25. // AddEdbDataFromSciHq 新增卓创红期指标数据
  26. func AddEdbDataFromSciHq(edbCode string) (err error) {
  27. o := orm.NewOrm()
  28. var condition string
  29. var pars []interface{}
  30. if edbCode != "" {
  31. condition += " AND index_code=? "
  32. pars = append(pars, edbCode)
  33. }
  34. sciBaseDataAll, err := GetBaseFromSciHqDataByCondition(condition, pars)
  35. if err != nil && err.Error() != utils.ErrNoRow() {
  36. return
  37. }
  38. var isAdd bool
  39. addSql := ` INSERT INTO edb_data_sci_hq(edb_info_id,edb_code,data_time,value,create_time,modify_time,data_timestamp) values `
  40. existMap := make(map[string]string)
  41. for _, sv := range sciBaseDataAll {
  42. eDate := sv.DataTime
  43. dataTime, err := time.ParseInLocation(utils.FormatDate, eDate, time.Local)
  44. if err != nil {
  45. return err
  46. }
  47. timestamp := dataTime.UnixNano() / 1e6
  48. timeStr := fmt.Sprintf("%d", timestamp)
  49. if _, ok := existMap[eDate]; !ok {
  50. addSql += GetAddSql("0", edbCode, eDate, timeStr, sv.Value)
  51. isAdd = true
  52. }
  53. existMap[eDate] = sv.Value
  54. }
  55. if isAdd {
  56. addSql = strings.TrimRight(addSql, ",")
  57. utils.FileLog.Info("addSql:" + addSql)
  58. _, err = o.Raw(addSql).Exec()
  59. if err != nil {
  60. return err
  61. }
  62. }
  63. return
  64. }
  65. // RefreshEdbDataFromSciHq 刷新卓创红期指标数据
  66. func RefreshEdbDataFromSciHq(edbInfoId int, edbCode, startDate string) (err error) {
  67. source := utils.DATA_SOURCE_SCI_HQ
  68. subSource := utils.DATA_SUB_SOURCE_EDB
  69. o := orm.NewOrm()
  70. edbInfoIdStr := strconv.Itoa(edbInfoId)
  71. //计算数据
  72. var condition string
  73. var pars []interface{}
  74. if edbCode != "" {
  75. condition += " AND index_code=? "
  76. pars = append(pars, edbCode)
  77. }
  78. if startDate != "" {
  79. condition += " AND data_time>=? "
  80. pars = append(pars, startDate)
  81. }
  82. sciDataList, err := GetBaseFromSciDataByCondition(condition, pars)
  83. if err != nil {
  84. return
  85. }
  86. // 真实数据的最大日期 , 插入规则配置的日期
  87. var realDataMaxDate, edbDataInsertConfigDate time.Time
  88. var edbDataInsertConfig *EdbDataInsertConfig
  89. var isFindConfigDateRealData bool //是否找到配置日期的实际数据的值
  90. {
  91. edbDataInsertConfig, err = GetEdbDataInsertConfigByEdbId(edbInfoId)
  92. if err != nil && err.Error() != utils.ErrNoRow() {
  93. return
  94. }
  95. if edbDataInsertConfig != nil {
  96. edbDataInsertConfigDate = edbDataInsertConfig.Date
  97. }
  98. }
  99. var existCondition string
  100. var existPars []interface{}
  101. existCondition += " AND edb_info_id=? "
  102. existPars = append(existPars, edbInfoId)
  103. if startDate != "" {
  104. existCondition += " AND data_time>=? "
  105. existPars = append(existPars, startDate)
  106. }
  107. existList, err := GetEdbDataByCondition(source, subSource, existCondition, existPars)
  108. if err != nil {
  109. return err
  110. }
  111. existMap := make(map[string]*EdbInfoSearchData)
  112. for _, v := range existList {
  113. existMap[v.DataTime] = v
  114. }
  115. addSql := ` INSERT INTO edb_data_sci_hq(edb_info_id,edb_code,data_time,value,create_time,modify_time,data_timestamp) values `
  116. var isAdd bool
  117. for _, v := range sciDataList {
  118. item := v
  119. eDate := item.DataTime
  120. dataTime, err := time.ParseInLocation(utils.FormatDate, eDate, time.Local)
  121. if err != nil {
  122. return err
  123. }
  124. if findItem, ok := existMap[v.DataTime]; !ok {
  125. sValue := item.Value
  126. timestamp := dataTime.UnixNano() / 1e6
  127. timeStr := fmt.Sprintf("%d", timestamp)
  128. addSql += GetAddSql(edbInfoIdStr, edbCode, eDate, timeStr, sValue)
  129. isAdd = true
  130. } else {
  131. if findItem != nil && utils.SubFloatToString(findItem.Value, 30) != item.Value {
  132. err = ModifyEdbDataById(source, subSource, findItem.EdbDataId, item.Value)
  133. if err != nil {
  134. return err
  135. }
  136. }
  137. }
  138. // 下面代码主要目的是处理掉手动插入的数据判断
  139. {
  140. if realDataMaxDate.IsZero() || dataTime.After(realDataMaxDate) {
  141. realDataMaxDate = dataTime
  142. }
  143. if edbDataInsertConfigDate.IsZero() || dataTime.Equal(edbDataInsertConfigDate) {
  144. isFindConfigDateRealData = true
  145. }
  146. }
  147. }
  148. // 处理手工数据补充的配置
  149. HandleConfigInsertEdbData(realDataMaxDate, edbDataInsertConfig, edbInfoId, source, subSource, existMap, isFindConfigDateRealData)
  150. if isAdd {
  151. addSql = strings.TrimRight(addSql, ",")
  152. _, err = o.Raw(addSql).Exec()
  153. if err != nil {
  154. fmt.Println("RefreshEdbDataFromSci add Err", err.Error())
  155. return
  156. }
  157. }
  158. return
  159. }
  160. // HandleSciHqExcelDataReq 卓创红期的excel数据
  161. type HandleSciHqExcelDataReq struct {
  162. IndexName string
  163. ExcelIndexCode string
  164. Frequency string
  165. Unit string
  166. TerminalCode string
  167. FilePath string
  168. Data map[string]string
  169. }
  170. type BaseFromSciHqIndex struct {
  171. BaseFromSciHqIndexId int `orm:"column(base_from_sci_hq_index_id);pk"` //序号
  172. IndexCode string `description:"指标编码"`
  173. IndexName string `description:"指标名称"`
  174. ClassifyId int `description:"分类id"`
  175. Frequency string `description:"频率"`
  176. Unit string `description:"单位"`
  177. StartDate string `description:"开始日期"`
  178. EndDate string `description:"结束日期"`
  179. Sort int `description:"排序"`
  180. TerminalCode string `description:"终端编码"`
  181. FilePath string `description:"文件路径"`
  182. CreateTime time.Time `description:"创建时间"`
  183. ModifyTime time.Time `description:"更新时间"`
  184. }
  185. func GetAllBaseFromSciHqIndex() (items []*BaseFromSciHqIndex, err error) {
  186. o := orm.NewOrm()
  187. sql := `SELECT * FROM base_from_sci_hq_index`
  188. _, err = o.Raw(sql).QueryRows(&items)
  189. return
  190. }
  191. func BatchAddBaseFromSciHqIndex(list []*BaseFromSciHqIndex) (err error) {
  192. o := orm.NewOrm()
  193. _, err = o.InsertMulti(len(list), list)
  194. return
  195. }
  196. type BaseFromSciHqData struct {
  197. SciHqDataId int `orm:"column(sci_hq_data_id);pk"` //序号
  198. BaseFromSciHqIndexId int `orm:"column(base_from_sci_hq_index_id)"` //指标id
  199. IndexCode string `description:"指标编码"`
  200. DataTime string `description:"数据时间"`
  201. Value string `description:"数据值"`
  202. CreateTime time.Time `description:"创建时间"`
  203. ModifyTime time.Time `description:"更新时间"`
  204. DataTimestamp int64 `description:"数据时间戳"`
  205. }
  206. func (r *BaseFromSciHqData) Update(cols []string) (err error) {
  207. o := orm.NewOrm()
  208. _, err = o.Update(r, cols...)
  209. return
  210. }
  211. func BatchAddBaseFromSciHqData(list []*BaseFromSciHqData) (err error) {
  212. o := orm.NewOrm()
  213. _, err = o.InsertMulti(500, list)
  214. return
  215. }
  216. func GetSciHqIndexInfoMaxAndMinInfo(indexCode string) (item *EdbInfoMaxAndMinInfo, err error) {
  217. o := orm.NewOrm()
  218. sql := ` SELECT MIN(data_time) AS min_date,MAX(data_time) AS max_date,MIN(value) AS min_value,MAX(value) AS max_value FROM base_from_sci_hq_data WHERE index_code=? `
  219. err = o.Raw(sql, indexCode).QueryRow(&item)
  220. if err != nil {
  221. return
  222. }
  223. // 获取最新值
  224. var latest_value float64
  225. var latestDate string
  226. sql = ` SELECT value AS latest_value, data_time AS latest_date FROM base_from_sci_hq_data WHERE index_code=? ORDER BY data_time DESC LIMIT 1 `
  227. err = o.Raw(sql, indexCode).QueryRow(&latest_value, &latestDate)
  228. if err != nil {
  229. return
  230. }
  231. item.LatestValue = latest_value
  232. item.LatestDate = latestDate
  233. return
  234. }
  235. func ModifySciHqIndexMaxAndMinInfo(indexCode string, item *EdbInfoMaxAndMinInfo) (err error) {
  236. o := orm.NewOrm()
  237. sql := ` UPDATE base_from_sci_hq_index SET start_date=?,end_date=?,latest_value=?,latest_date=?,modify_time=NOW() WHERE index_code=? `
  238. _, err = o.Raw(sql, item.MinDate, item.MaxDate, item.LatestValue, item.LatestDate, indexCode).Exec()
  239. return
  240. }