base_from_sci99.go 6.8 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242
  1. package models
  2. import (
  3. "eta/eta_index_lib/utils"
  4. "fmt"
  5. "github.com/beego/beego/v2/client/orm"
  6. "strconv"
  7. "strings"
  8. "time"
  9. )
  10. // BaseFromSci99Index 代表卓创资讯-原始指标表的结构
  11. type BaseFromSci99Index struct {
  12. BaseFromSciIndexId int `orm:"column(base_from_sci_index_id);pk"` // 主键,自动递增
  13. IndexCode string // 指标编码
  14. IndexName string // 指标名称
  15. ClassifyId int // 分类Id
  16. Unit string // 单位
  17. Frequency string // 频度
  18. Describe string // 指标描述
  19. CreateTime time.Time // 创建时间
  20. ModifyTime time.Time // 修改时间
  21. }
  22. // BaseFromSci99Data 代表卓创资讯-原始指标数据表的结构
  23. type BaseFromSci99Data struct {
  24. BaseFromSciDataId int `orm:"column(base_from_sci_data_id);pk"` // 主键,自动递增
  25. BaseFromSciIndexId int // 指标id
  26. IndexCode string // 指标编码
  27. DataTime string // 数据日期
  28. Value string // 数据值
  29. CreateTime time.Time // 创建时间
  30. ModifyTime time.Time // 修改时间
  31. }
  32. // BaseFromSci99Classify 代表卓创资讯-原始指标分类表的结构
  33. type BaseFromSci99Classify struct {
  34. BaseFromSciClassifyId int `orm:"column(base_from_sci_classify_id);pk"` // 主键,自动递增
  35. ClassifyName string // 分类名称
  36. Sort int // 排序
  37. CreateTime time.Time // 创建时间
  38. ModifyTime time.Time // 修改时间
  39. }
  40. func GetBaseFromSci99DataDataByCondition(condition string, pars []interface{}) (item []*BaseFromIcpiData, err error) {
  41. sql := ` SELECT * FROM base_from_sci99_data WHERE 1=1 `
  42. o := orm.NewOrm()
  43. if condition != "" {
  44. sql += condition
  45. }
  46. sql += ` ORDER BY data_time DESC `
  47. _, err = o.Raw(sql, pars).QueryRows(&item)
  48. return
  49. }
  50. // 添加数据
  51. func AddBaseFromSci99Index(item *BaseFromSci99Index) (lastId int64, err error) {
  52. o := orm.NewOrmUsingDB("data")
  53. lastId, err = o.Insert(item)
  54. return
  55. }
  56. func AddBaseFromSci99Classify(item *BaseFromSci99Classify) (lastId int64, err error) {
  57. o := orm.NewOrmUsingDB("data")
  58. lastId, err = o.Insert(item)
  59. return
  60. }
  61. func AddBaseFromSci99DataMulti(item []*BaseFromSci99Data) (err error) {
  62. o := orm.NewOrmUsingDB("data")
  63. _, err = o.InsertMulti(1000, item)
  64. return
  65. }
  66. func GetBaseFromSci99Index() (list []*BaseFromSci99Index, err error) {
  67. o := orm.NewOrmUsingDB("data")
  68. sql := `SELECT * FROM base_from_sci99_index `
  69. _, err = o.Raw(sql).QueryRows(&list)
  70. return
  71. }
  72. func GetBaseFromSci99DataByIndexCode(indexCode string) (items []*BaseFromSci99Data, err error) {
  73. o := orm.NewOrm()
  74. sql := `SELECT * FROM base_from_sci99_data WHERE index_code=? `
  75. _, err = o.Raw(sql, indexCode).QueryRows(&items)
  76. return
  77. }
  78. // 新增卓创资讯指标数据
  79. func AddEdbDataFromSci99(edbCode string) (err error) {
  80. o := orm.NewOrm()
  81. dataAll, err := GetBaseFromSci99DataByIndexCode(edbCode)
  82. if err != nil && err.Error() != utils.ErrNoRow() {
  83. return
  84. }
  85. var isAdd bool
  86. addSql := ` INSERT INTO edb_data_sci99(edb_info_id,edb_code,data_time,value,create_time,modify_time,data_timestamp) values `
  87. existMap := make(map[string]string)
  88. for _, sv := range dataAll {
  89. eDate := sv.DataTime
  90. dataTime, err := time.ParseInLocation(utils.FormatDate, eDate, time.Local)
  91. if err != nil {
  92. fmt.Println("time.Parse Err:" + eDate)
  93. return err
  94. }
  95. timestamp := dataTime.UnixNano() / 1e6
  96. timeStr := fmt.Sprintf("%d", timestamp)
  97. if _, ok := existMap[eDate]; !ok {
  98. addSql += GetAddSql("0", edbCode, eDate, timeStr, sv.Value)
  99. isAdd = true
  100. }
  101. }
  102. if isAdd {
  103. addSql = strings.TrimRight(addSql, ",")
  104. utils.FileLog.Info("addSql:" + addSql)
  105. _, err = o.Raw(addSql).Exec()
  106. if err != nil {
  107. return err
  108. }
  109. }
  110. return
  111. }
  112. // 刷新卓创资讯指标数据
  113. func RefreshEdbDataFromSci99(edbInfoId int, edbCode, startDate string) (err error) {
  114. source := utils.DATA_SOURCE_SCI99
  115. subSource := utils.DATA_SUB_SOURCE_EDB
  116. o := orm.NewOrm()
  117. if err != nil {
  118. return
  119. }
  120. edbInfoIdStr := strconv.Itoa(edbInfoId)
  121. //计算数据
  122. var condition string
  123. var pars []interface{}
  124. if edbCode != "" {
  125. condition += " AND index_code=? "
  126. pars = append(pars, edbCode)
  127. }
  128. if startDate != "" {
  129. condition += " AND data_time>=? "
  130. pars = append(pars, startDate)
  131. }
  132. dataList, err := GetBaseFromSci99DataDataByCondition(condition, pars)
  133. if err != nil {
  134. return
  135. }
  136. // 真实数据的最大日期 , 插入规则配置的日期
  137. var realDataMaxDate, edbDataInsertConfigDate time.Time
  138. var edbDataInsertConfig *EdbDataInsertConfig
  139. var isFindConfigDateRealData bool //是否找到配置日期的实际数据的值
  140. {
  141. edbDataInsertConfig, err = GetEdbDataInsertConfigByEdbId(edbInfoId)
  142. if err != nil && err.Error() != utils.ErrNoRow() {
  143. return
  144. }
  145. if edbDataInsertConfig != nil {
  146. edbDataInsertConfigDate = edbDataInsertConfig.Date
  147. }
  148. }
  149. //获取指标所有数据
  150. var existCondition string
  151. var existPars []interface{}
  152. existCondition += " AND edb_info_id=? "
  153. existPars = append(existPars, edbInfoId)
  154. if startDate != "" {
  155. existCondition += " AND data_time>=? "
  156. existPars = append(existPars, startDate)
  157. }
  158. existList, err := GetEdbDataByCondition(source, subSource, existCondition, existPars)
  159. if err != nil {
  160. return err
  161. }
  162. existMap := make(map[string]*EdbInfoSearchData)
  163. for _, v := range existList {
  164. existMap[v.DataTime] = v
  165. }
  166. addSql := ` INSERT INTO edb_data_sci99(edb_info_id,edb_code,data_time,value,create_time,modify_time,data_timestamp) values `
  167. var isAdd bool
  168. for _, v := range dataList {
  169. item := v
  170. itemValue := v.Value
  171. eDate := item.DataTime
  172. dataTime, err := time.ParseInLocation(utils.FormatDate, eDate, time.Local)
  173. if err != nil {
  174. return err
  175. }
  176. if _, ok := existMap[v.DataTime]; !ok {
  177. sValue := itemValue
  178. if sValue != "" {
  179. timestamp := dataTime.UnixNano() / 1e6
  180. timeStr := fmt.Sprintf("%d", timestamp)
  181. saveValue := sValue
  182. if findItem, ok := existMap[eDate]; !ok {
  183. addSql += GetAddSql(edbInfoIdStr, edbCode, eDate, timeStr, saveValue)
  184. isAdd = true
  185. } else {
  186. if findItem != nil && utils.SubFloatToString(findItem.Value, 30) != sValue {
  187. err = ModifyEdbDataById(source, subSource, findItem.EdbDataId, sValue)
  188. if err != nil {
  189. return err
  190. }
  191. }
  192. }
  193. }
  194. }
  195. // 下面代码主要目的是处理掉手动插入的数据判断
  196. {
  197. if realDataMaxDate.IsZero() || dataTime.After(realDataMaxDate) {
  198. realDataMaxDate = dataTime
  199. }
  200. if edbDataInsertConfigDate.IsZero() || dataTime.Equal(edbDataInsertConfigDate) {
  201. isFindConfigDateRealData = true
  202. }
  203. }
  204. }
  205. // 处理手工数据补充的配置
  206. HandleConfigInsertEdbData(realDataMaxDate, edbDataInsertConfig, edbInfoId, source, subSource, existMap, isFindConfigDateRealData)
  207. if isAdd {
  208. addSql = strings.TrimRight(addSql, ",")
  209. _, err = o.Raw(addSql).Exec()
  210. if err != nil {
  211. return err
  212. }
  213. }
  214. return
  215. }