base_from_hisugar.go 8.5 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291
  1. package models
  2. import (
  3. "eta/eta_index_lib/global"
  4. "eta/eta_index_lib/utils"
  5. "fmt"
  6. "strconv"
  7. "strings"
  8. "time"
  9. )
  10. type BaseFromHisugarIndex struct {
  11. BaseFromHisugarIndexId int // 主键ID
  12. IndexCode string // 指标编码
  13. IndexName string // 指标名称
  14. ClassifyId uint // 分类ID
  15. Unit string // 单位
  16. Frequency string // 频度
  17. Describe string // 指标描述
  18. Sort int // 排序
  19. CreateTime time.Time // 创建时间
  20. ModifyTime time.Time // 修改时间
  21. }
  22. type BaseFromHisugarData struct {
  23. BaseFromHisugarDataId int // 数据表ID
  24. BaseFromHisugarIndexId int // 指标ID
  25. IndexCode string // 指标编码
  26. DataTime string
  27. Value string
  28. CreateTime time.Time // 创建时间
  29. ModifyTime time.Time // 修改时间
  30. }
  31. // 添加数据
  32. func AddBaseFromHisugarIndexMuti(items []*BaseFromHisugarIndex) (err error) {
  33. //o := orm.NewOrm()
  34. //_, err = o.InsertMulti(500, items)
  35. err = global.DEFAULT_DB.CreateInBatches(items, len(items)).Error
  36. return
  37. }
  38. func AddBaseFromHisugarIndex(item *BaseFromHisugarIndex) (lastId int64, err error) {
  39. //o := orm.NewOrm()
  40. //lastId, err = o.Insert(item)
  41. err = global.DEFAULT_DB.Create(&item).Error
  42. if err != nil {
  43. return
  44. }
  45. lastId = int64(item.BaseFromHisugarIndexId)
  46. return
  47. }
  48. func AddBaseFromHisugarData(item *BaseFromHisugarData) (lastId int64, err error) {
  49. //o := orm.NewOrm()
  50. //lastId, err = o.Insert(item)
  51. err = global.DEFAULT_DB.Create(&item).Error
  52. if err != nil {
  53. return
  54. }
  55. lastId = int64(item.BaseFromHisugarDataId)
  56. return
  57. }
  58. func AddEdbDataFromHisugar(edbCode string) (err error) {
  59. //o := orm.NewOrm()
  60. hisugarBaseDataAll, err := GetHisugarDataByCode(edbCode)
  61. if err != nil && !utils.IsErrNoRow(err) {
  62. return
  63. }
  64. var isAdd bool
  65. addSql := ` INSERT INTO edb_data_hisugar(edb_info_id,edb_code,data_time,value,create_time,modify_time,data_timestamp) values `
  66. existMap := make(map[string]string)
  67. for _, sv := range hisugarBaseDataAll {
  68. eDate := sv.DataTime
  69. var timeStr string
  70. var dataTime time.Time
  71. var sDataTime string
  72. var timestamp int64
  73. sDataTime = eDate
  74. dataTime, err = time.ParseInLocation(utils.FormatDate, eDate, time.Local)
  75. if err != nil {
  76. fmt.Println("time.Parse Err:" + eDate)
  77. return err
  78. }
  79. timestamp = dataTime.UnixNano() / 1e6
  80. timeStr = fmt.Sprintf("%d", timestamp)
  81. value := strings.Replace(sv.Value, "%", "", -1)
  82. if _, ok := existMap[sDataTime]; !ok {
  83. addSql += GetAddSql("0", edbCode, sDataTime, timeStr, value)
  84. fmt.Println("edbCode:", edbCode)
  85. fmt.Println("sDataTime:", sDataTime)
  86. fmt.Println("timeStr:", timeStr)
  87. fmt.Println("value:", value)
  88. isAdd = true
  89. }
  90. existMap[eDate] = value
  91. }
  92. if isAdd {
  93. addSql = strings.TrimRight(addSql, ",")
  94. utils.FileLog.Info("addSql:" + addSql)
  95. //_, err = o.Raw(addSql).Exec()
  96. err = global.DEFAULT_DB.Exec(addSql).Error
  97. if err != nil {
  98. return err
  99. }
  100. }
  101. return
  102. }
  103. // GetHisugarDataByCode
  104. func GetHisugarDataByCode(indexCode string) (items []*BaseFromHisugarData, err error) {
  105. //o := orm.NewOrm()
  106. sql := "SELECT * FROM base_from_hisugar_data WHERE index_code=? "
  107. //_, err = o.Raw(sql, indexCode).QueryRows(&items)
  108. err = global.DEFAULT_DB.Raw(sql, indexCode).Find(&items).Error
  109. return
  110. }
  111. // RefreshEdbDataFromHisugar 刷新隆众资讯
  112. func RefreshEdbDataFromHisugar(edbInfoId int, edbCode, startDate string) (err error) {
  113. source := utils.DATA_SOURCE_OILCHEM
  114. subSource := utils.DATA_SUB_SOURCE_EDB
  115. //o := orm.NewOrm()
  116. if err != nil {
  117. return
  118. }
  119. edbInfoIdStr := strconv.Itoa(edbInfoId)
  120. //计算数据
  121. var condition string
  122. var pars []interface{}
  123. if edbCode != "" {
  124. condition += " AND index_code=? "
  125. pars = append(pars, edbCode)
  126. }
  127. if startDate != "" {
  128. condition += " AND data_time>=? "
  129. pars = append(pars, startDate)
  130. }
  131. hisugarDataList, err := GetBaseFromHisugarDataByCondition(condition, pars)
  132. if err != nil {
  133. return
  134. }
  135. // 真实数据的最大日期 , 插入规则配置的日期
  136. var realDataMaxDate, edbDataInsertConfigDate time.Time
  137. var edbDataInsertConfig *EdbDataInsertConfig
  138. var isFindConfigDateRealData bool //是否找到配置日期的实际数据的值
  139. {
  140. edbDataInsertConfig, err = GetEdbDataInsertConfigByEdbId(edbInfoId)
  141. if err != nil && !utils.IsErrNoRow(err) {
  142. return
  143. }
  144. if edbDataInsertConfig != nil {
  145. edbDataInsertConfigDate = edbDataInsertConfig.Date
  146. }
  147. }
  148. var existCondition string
  149. var existPars []interface{}
  150. existCondition += " AND edb_info_id=? "
  151. existPars = append(existPars, edbInfoId)
  152. if startDate != "" {
  153. existCondition += " AND data_time>=? "
  154. existPars = append(existPars, startDate)
  155. }
  156. existList, err := GetEdbDataByCondition(source, subSource, existCondition, existPars)
  157. if err != nil {
  158. return err
  159. }
  160. existMap := make(map[string]*EdbInfoSearchData)
  161. for _, v := range existList {
  162. existMap[v.DataTime] = v
  163. }
  164. addSql := ` INSERT INTO edb_data_hisugar(edb_info_id,edb_code,data_time,value,create_time,modify_time,data_timestamp) values `
  165. var isAdd bool
  166. for _, v := range hisugarDataList {
  167. item := v
  168. eDate := item.DataTime
  169. dataTime, err := time.ParseInLocation(utils.FormatDate, eDate, time.Local)
  170. if err != nil {
  171. return err
  172. }
  173. if findItem, ok := existMap[v.DataTime]; !ok {
  174. sValue := item.Value
  175. timestamp := dataTime.UnixNano() / 1e6
  176. timeStr := fmt.Sprintf("%d", timestamp)
  177. addSql += GetAddSql(edbInfoIdStr, edbCode, eDate, timeStr, sValue)
  178. isAdd = true
  179. } else {
  180. if findItem != nil && utils.SubFloatToString(findItem.Value, 30) != item.Value {
  181. err = ModifyEdbDataById(source, subSource, findItem.EdbDataId, item.Value)
  182. if err != nil {
  183. return err
  184. }
  185. }
  186. }
  187. // 下面代码主要目的是处理掉手动插入的数据判断
  188. {
  189. if realDataMaxDate.IsZero() || dataTime.After(realDataMaxDate) {
  190. realDataMaxDate = dataTime
  191. }
  192. if edbDataInsertConfigDate.IsZero() || dataTime.Equal(edbDataInsertConfigDate) {
  193. isFindConfigDateRealData = true
  194. }
  195. }
  196. }
  197. // 处理手工数据补充的配置
  198. HandleConfigInsertEdbData(realDataMaxDate, edbDataInsertConfig, edbInfoId, source, subSource, existMap, isFindConfigDateRealData)
  199. if isAdd {
  200. addSql = strings.TrimRight(addSql, ",")
  201. //_, err = o.Raw(addSql).Exec()
  202. err = global.DEFAULT_DB.Exec(addSql).Error
  203. if err != nil {
  204. fmt.Println("RefreshEdbDataFromSci add Err", err.Error())
  205. return
  206. }
  207. }
  208. return
  209. }
  210. func GetBaseFromHisugarDataByCondition(condition string, pars []interface{}) (list []*BaseFromHisugarData, err error) {
  211. //o := orm.NewOrm()
  212. sql := `SELECT * FROM base_from_hisugar_data WHERE 1=1 `
  213. if condition != "" {
  214. sql += condition
  215. }
  216. //_, err = o.Raw(sql, pars).QueryRows(&list)
  217. err = global.DEFAULT_DB.Raw(sql, pars...).Find(&list).Error
  218. return
  219. }
  220. type HandleHisugarEdbDataReq struct {
  221. List []*BaseFromHisugarIndexReq
  222. }
  223. func GetBaseFromHisugarIndex() (list []*BaseFromHisugarIndex, err error) {
  224. //o := orm.NewOrm()
  225. sql := `SELECT * FROM base_from_hisugar_index group by index_name `
  226. //_, err = o.Raw(sql).QueryRows(&list)
  227. err = global.DEFAULT_DB.Raw(sql).Find(&list).Error
  228. return
  229. }
  230. func GetBaseFromHisugarData(indexCode, dataTime string) (item *BaseFromHisugarData, err error) {
  231. //o := orm.NewOrm()
  232. sql := `SELECT * FROM base_from_hisugar_data where index_code=? And data_time = ? `
  233. //err = o.Raw(sql, indexCode, dataTime).QueryRow(&item)
  234. err = global.DEFAULT_DB.Raw(sql, indexCode, dataTime).First(&item).Error
  235. return
  236. }
  237. // UpdateBaseFromSci99Data
  238. func UpdateBaseFromHisugarData(value, indexCode, dataTime string) (err error) {
  239. //o := orm.NewOrm()
  240. sql := `UPDATE base_from_sci99_data SET value=?,modify_time=NOW() WHERE index_code = ? AND data_time = ? `
  241. //_, err = o.Raw(sql, value, indexCode, dataTime).Exec()
  242. err = global.DEFAULT_DB.Exec(sql, value, indexCode, dataTime).Error
  243. return
  244. }
  245. type BaseFromHisugarIndexReq struct {
  246. BaseFromHisugarIndexId int // 主键ID
  247. IndexCode string // 指标编码
  248. IndexName string // 指标名称
  249. ClassifyId uint // 分类ID
  250. Unit string // 单位
  251. Frequency string // 频度
  252. Describe string // 指标描述
  253. DataTime string // 数据日期
  254. Sort int // 排序
  255. CreateTime time.Time // 创建时间
  256. ModifyTime time.Time // 修改时间
  257. IndexNameStr string // 指标名称字符串
  258. MarketName string // 市场名称
  259. Value string // 值
  260. }