base_from_sci99.go 11 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338
  1. package models
  2. import (
  3. "eta/eta_index_lib/global"
  4. "eta/eta_index_lib/utils"
  5. "fmt"
  6. "strconv"
  7. "strings"
  8. "time"
  9. )
  10. // BaseFromSci99Index 代表卓创资讯-原始指标表的结构
  11. type BaseFromSci99Index struct {
  12. BaseFromSciIndexId int `gorm:"column:base_from_sci_index_id;primaryKey"` // 主键,自动递增
  13. //BaseFromSciIndexId int `orm:"column(base_from_sci_index_id);pk"` // 主键,自动递增
  14. IndexCode string // 指标编码
  15. IndexName string // 指标名称
  16. ClassifyId int // 分类Id
  17. Unit string // 单位
  18. Frequency string // 频度
  19. Describe string // 指标描述
  20. CreateTime time.Time // 创建时间
  21. ModifyTime time.Time // 修改时间
  22. StartDate time.Time `gorm:"column:start_date"`
  23. EndDate time.Time `gorm:"column:end_date"`
  24. LatestValue float64 `gorm:"column:latest_value"` // 数据最新值
  25. }
  26. // BaseFromSci99Data 代表卓创资讯-原始指标数据表的结构
  27. type BaseFromSci99DataItem struct {
  28. BaseFromSciDataId int `gorm:"column:base_from_sci_data_id;primaryKey"` // 主键,自动递增
  29. //BaseFromSciDataId int `orm:"column(base_from_sci_data_id);pk"` // 主键,自动递增
  30. BaseFromSciIndexId int // 指标id
  31. IndexCode string // 指标编码
  32. DataTime string // 数据日期
  33. Value string // 数据值
  34. CreateTime time.Time // 创建时间
  35. ModifyTime time.Time // 修改时间
  36. }
  37. // BaseFromSci99Data 代表卓创资讯-原始指标数据表的结构
  38. type BaseFromSci99Data struct {
  39. BaseFromSciDataId int `gorm:"column:base_from_sci_data_id;primaryKey"` // 主键,自动递增
  40. //BaseFromSciDataId int `orm:"column(base_from_sci_data_id);pk"` // 主键,自动递增
  41. BaseFromSciIndexId int // 指标id
  42. IndexCode string // 指标编码
  43. DataTime string // 数据日期
  44. Value float64 // 数据值
  45. CreateTime time.Time // 创建时间
  46. ModifyTime time.Time // 修改时间
  47. }
  48. // BaseFromSci99Classify 代表卓创资讯-原始指标分类表的结构
  49. type BaseFromSci99Classify struct {
  50. BaseFromSciClassifyId int `gorm:"column:base_from_sci_classify_id;primaryKey"` // 主键,自动递增
  51. //BaseFromSciClassifyId int `orm:"column(base_from_sci_classify_id);pk"` // 主键,自动递增
  52. ClassifyName string // 分类名称
  53. Sort int // 排序
  54. CreateTime time.Time // 创建时间
  55. ModifyTime time.Time // 修改时间
  56. }
  57. func GetBaseFromSci99DataDataByCondition(condition string, pars []interface{}) (item []*BaseFromIcpiData, err error) {
  58. sql := ` SELECT * FROM base_from_sci99_data WHERE 1=1 `
  59. //o := orm.NewOrm()
  60. if condition != "" {
  61. sql += condition
  62. }
  63. sql += ` ORDER BY data_time DESC `
  64. //_, err = o.Raw(sql, pars).QueryRows(&item)
  65. err = global.DEFAULT_DB.Raw(sql, pars...).Find(&item).Error
  66. return
  67. }
  68. // 添加数据
  69. func AddBaseFromSci99Index(item *BaseFromSci99Index) (lastId int64, err error) {
  70. //o := orm.NewOrm()
  71. //lastId, err = o.Insert(item)
  72. err = global.DEFAULT_DB.Create(&item).Error
  73. if err != nil {
  74. return
  75. }
  76. lastId = int64(item.BaseFromSciIndexId)
  77. return
  78. }
  79. func AddBaseFromSci99Classify(item *BaseFromSci99Classify) (lastId int64, err error) {
  80. //o := orm.NewOrm()
  81. //lastId, err = o.Insert(item)
  82. err = global.DEFAULT_DB.Create(&item).Error
  83. if err != nil {
  84. return
  85. }
  86. lastId = int64(item.BaseFromSciClassifyId)
  87. return
  88. }
  89. func AddBaseFromSci99DataMulti(item []*BaseFromSci99Data) (err error) {
  90. //o := orm.NewOrm()
  91. //_, err = o.InsertMulti(1000, item)
  92. err = global.DEFAULT_DB.CreateInBatches(&item, 1000).Error
  93. return
  94. }
  95. func GetBaseFromSci99Index() (list []*BaseFromSci99Index, err error) {
  96. //o := orm.NewOrm()
  97. sql := `SELECT * FROM base_from_sci99_index `
  98. //_, err = o.Raw(sql).QueryRows(&list)
  99. err = global.DEFAULT_DB.Raw(sql).Find(&list).Error
  100. return
  101. }
  102. func GetBaseFromSci99DataByIndexCode(indexCode string) (items []*BaseFromSci99DataItem, err error) {
  103. //o := orm.NewOrm()
  104. sql := `SELECT * FROM base_from_sci99_data WHERE index_code=? `
  105. //_, err = o.Raw(sql, indexCode).QueryRows(&items)
  106. err = global.DEFAULT_DB.Raw(sql, indexCode).Find(&items).Error
  107. return
  108. }
  109. // 新增卓创资讯指标数据
  110. func AddEdbDataFromSci99(edbCode string) (err error) {
  111. //o := orm.NewOrm()
  112. dataAll, err := GetBaseFromSci99DataByIndexCode(edbCode)
  113. if err != nil && !utils.IsErrNoRow(err) {
  114. return
  115. }
  116. var isAdd bool
  117. addSql := ` INSERT INTO edb_data_sci99(edb_info_id,edb_code,data_time,value,create_time,modify_time,data_timestamp) values `
  118. existMap := make(map[string]string)
  119. for _, sv := range dataAll {
  120. eDate := sv.DataTime
  121. dataTime, err := time.ParseInLocation(utils.FormatDate, eDate, time.Local)
  122. if err != nil {
  123. fmt.Println("time.Parse Err:" + eDate)
  124. return err
  125. }
  126. timestamp := dataTime.UnixNano() / 1e6
  127. timeStr := fmt.Sprintf("%d", timestamp)
  128. if _, ok := existMap[eDate]; !ok {
  129. addSql += GetAddSql("0", edbCode, eDate, timeStr, sv.Value)
  130. isAdd = true
  131. }
  132. }
  133. if isAdd {
  134. addSql = strings.TrimRight(addSql, ",")
  135. utils.FileLog.Info("addSql:" + addSql)
  136. //_, err = o.Raw(addSql).Exec()
  137. err = global.DEFAULT_DB.Exec(addSql).Error
  138. if err != nil {
  139. return err
  140. }
  141. }
  142. return
  143. }
  144. // 刷新卓创资讯指标数据
  145. func RefreshEdbDataFromSci99(edbInfoId int, edbCode, startDate string) (err error) {
  146. source := utils.DATA_SOURCE_SCI99
  147. subSource := utils.DATA_SUB_SOURCE_EDB
  148. //o := orm.NewOrm()
  149. if err != nil {
  150. return
  151. }
  152. edbInfoIdStr := strconv.Itoa(edbInfoId)
  153. //计算数据
  154. var condition string
  155. var pars []interface{}
  156. if edbCode != "" {
  157. condition += " AND index_code=? "
  158. pars = append(pars, edbCode)
  159. }
  160. if startDate != "" {
  161. condition += " AND data_time>=? "
  162. pars = append(pars, startDate)
  163. }
  164. dataList, err := GetBaseFromSci99DataDataByCondition(condition, pars)
  165. if err != nil {
  166. return
  167. }
  168. // 真实数据的最大日期 , 插入规则配置的日期
  169. var realDataMaxDate, edbDataInsertConfigDate time.Time
  170. var edbDataInsertConfig *EdbDataInsertConfig
  171. var isFindConfigDateRealData bool //是否找到配置日期的实际数据的值
  172. {
  173. edbDataInsertConfig, err = GetEdbDataInsertConfigByEdbId(edbInfoId)
  174. if err != nil && !utils.IsErrNoRow(err) {
  175. return
  176. }
  177. if edbDataInsertConfig != nil {
  178. edbDataInsertConfigDate = edbDataInsertConfig.Date
  179. }
  180. }
  181. //获取指标所有数据
  182. var existCondition string
  183. var existPars []interface{}
  184. existCondition += " AND edb_info_id=? "
  185. existPars = append(existPars, edbInfoId)
  186. if startDate != "" {
  187. existCondition += " AND data_time>=? "
  188. existPars = append(existPars, startDate)
  189. }
  190. existList, err := GetEdbDataByCondition(source, subSource, existCondition, existPars)
  191. if err != nil {
  192. return err
  193. }
  194. existMap := make(map[string]*EdbInfoSearchData)
  195. for _, v := range existList {
  196. existMap[v.DataTime] = v
  197. }
  198. addSql := ` INSERT INTO edb_data_sci99(edb_info_id,edb_code,data_time,value,create_time,modify_time,data_timestamp) values `
  199. var isAdd bool
  200. for _, v := range dataList {
  201. item := v
  202. itemValue := v.Value
  203. eDate := item.DataTime
  204. dataTime, err := time.ParseInLocation(utils.FormatDate, eDate, time.Local)
  205. if err != nil {
  206. return err
  207. }
  208. if _, ok := existMap[v.DataTime]; !ok {
  209. sValue := itemValue
  210. if sValue != "" {
  211. timestamp := dataTime.UnixNano() / 1e6
  212. timeStr := fmt.Sprintf("%d", timestamp)
  213. saveValue := sValue
  214. if findItem, ok := existMap[eDate]; !ok {
  215. addSql += GetAddSql(edbInfoIdStr, edbCode, eDate, timeStr, saveValue)
  216. isAdd = true
  217. } else {
  218. if findItem != nil && utils.SubFloatToString(findItem.Value, 30) != sValue {
  219. err = ModifyEdbDataById(source, subSource, findItem.EdbDataId, sValue)
  220. if err != nil {
  221. return err
  222. }
  223. }
  224. }
  225. }
  226. }
  227. // 下面代码主要目的是处理掉手动插入的数据判断
  228. {
  229. if realDataMaxDate.IsZero() || dataTime.After(realDataMaxDate) {
  230. realDataMaxDate = dataTime
  231. }
  232. if edbDataInsertConfigDate.IsZero() || dataTime.Equal(edbDataInsertConfigDate) {
  233. isFindConfigDateRealData = true
  234. }
  235. }
  236. }
  237. // 处理手工数据补充的配置
  238. HandleConfigInsertEdbData(realDataMaxDate, edbDataInsertConfig, edbInfoId, source, subSource, existMap, isFindConfigDateRealData)
  239. if isAdd {
  240. addSql = strings.TrimRight(addSql, ",")
  241. //_, err = o.Raw(addSql).Exec()
  242. err = global.DEFAULT_DB.Exec(addSql).Error
  243. if err != nil {
  244. return err
  245. }
  246. }
  247. return
  248. }
  249. func GetBaseFromSci99Classify() (list []*BaseFromSci99Classify, err error) {
  250. //o := orm.NewOrm()
  251. sql := `SELECT * FROM base_from_sci99_classify `
  252. //_, err = o.Raw(sql).QueryRows(&list)
  253. err = global.DEFAULT_DB.Raw(sql).Find(&list).Error
  254. return
  255. }
  256. func GetBaseFromTradeSci99DataAll(indexCode string) (list []*BaseFromSci99Data, err error) {
  257. //o := orm.NewOrm()
  258. sql := `SELECT * FROM base_from_sci99_data where index_code=?`
  259. ///_, err = o.Raw(sql, indexCode).QueryRows(&list)
  260. err = global.DEFAULT_DB.Raw(sql, indexCode).Find(&list).Error
  261. return
  262. }
  263. func GetBaseFromTradeSci99LatestData(indexCode string) (date time.Time, err error) {
  264. //o := orm.NewOrm()
  265. sql := `SELECT data_time FROM base_from_sci99_data where index_code=? ORDER BY data_time DESC limit 1 `
  266. //err = o.Raw(sql, indexCode).QueryRow(&date)
  267. err = global.DEFAULT_DB.Raw(sql, indexCode).Scan(&date).Error
  268. return
  269. }
  270. // UpdateBaseFromSci99Data
  271. func UpdateBaseFromSci99Data(value float64, indexCode, dataTime string) (err error) {
  272. //o := orm.NewOrm()
  273. sql := `UPDATE base_from_sci99_data SET value=?,modify_time=NOW() WHERE index_code = ? AND data_time = ? `
  274. //_, err = o.Raw(sql, value, indexCode, dataTime).Exec()
  275. err = global.DEFAULT_DB.Exec(sql, value, indexCode, dataTime).Error
  276. return
  277. }
  278. func (m *BaseFromSci99Index) GetMaxAndMinDateByIndexCode(indexCode string) (item *EdbInfoMaxAndMinInfo, err error) {
  279. //o := orm.NewOrm()
  280. sql := ` SELECT MIN(data_time) AS min_date,MAX(data_time) AS max_date,MIN(value) AS min_value,MAX(value) AS max_value FROM base_from_sci99_data WHERE index_code=? `
  281. //err = o.Raw(sql, indexCode).QueryRow(&item)
  282. err = global.DEFAULT_DB.Raw(sql, indexCode).First(&item).Error
  283. if err != nil {
  284. return
  285. }
  286. // 获取最新值
  287. var latest_value float64
  288. sql = ` SELECT value AS latest_value FROM base_from_sci99_data WHERE index_code=? ORDER BY data_time DESC LIMIT 1 `
  289. //err = o.Raw(sql, indexCode).QueryRow(&latest_value)
  290. err = global.DEFAULT_DB.Raw(sql, indexCode).Scan(&latest_value).Error
  291. if err != nil {
  292. return
  293. }
  294. item.LatestValue = latest_value
  295. return
  296. }
  297. func (m *BaseFromSci99Index) ModifyIndexMaxAndMinDate(indexCode string, item *EdbInfoMaxAndMinInfo) (err error) {
  298. //o := orm.NewOrm()
  299. sql := ` UPDATE base_from_sci99_index SET start_date=?,end_date=?,latest_value=?,modify_time=NOW() WHERE index_code=? `
  300. //_, err = o.Raw(sql, item.MinDate, item.MaxDate, item.LatestValue, indexCode).Exec()
  301. err = global.DEFAULT_DB.Exec(sql, item.MinDate, item.MaxDate, item.LatestValue, indexCode).Error
  302. return
  303. }