base_from_coal.go 9.3 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306
  1. package models
  2. import (
  3. "eta/eta_index_lib/utils"
  4. "fmt"
  5. "github.com/beego/beego/v2/client/orm"
  6. "strconv"
  7. "strings"
  8. "time"
  9. )
  10. type BaseFromCoalmineIndex struct {
  11. BaseFromCoalmineCompanyIndexId int
  12. IndexName string `description:"持买单量指标名称"`
  13. IndexCode string `description:"持买单量指标编码"`
  14. DealValue string `description:"成交量"`
  15. DataTime string `description:"数据日期"`
  16. Source string `description:"来源"`
  17. Province string `description:"省份"`
  18. City string `description:"城市"`
  19. GroupName string `description:"集团名称"`
  20. Unit string `description:"单位"`
  21. Frequency string `description:"频率"`
  22. CreateTime string `description:"插入时间"`
  23. ModifyTime string `description:"修改时间"`
  24. }
  25. func GetBaseFromCoalIndexByCode(suffix, indexCode string) (items []*BaseFromCoalmineIndex, err error) {
  26. o := orm.NewOrm()
  27. sql := `SELECT * FROM base_from_coalmine_%s WHERE index_code=? `
  28. sql = fmt.Sprintf(sql, suffix)
  29. _, err = o.Raw(sql, indexCode).QueryRows(&items)
  30. return
  31. }
  32. func GetCoalDataByTradeCode(suffix, condition string, pars []interface{}) (item []*BaseFromCoalmineIndex, err error) {
  33. sql := ` SELECT * FROM base_from_coalmine_%s WHERE 1=1 `
  34. sql = fmt.Sprintf(sql, suffix)
  35. o := orm.NewOrm()
  36. if condition != "" {
  37. sql += condition
  38. }
  39. sql += ` ORDER BY data_time DESC `
  40. _, err = o.Raw(sql, pars).QueryRows(&item)
  41. return
  42. }
  43. func AddEdbDataFromCoal(edbCode string) (err error) {
  44. var suffix string
  45. if strings.Contains(edbCode, "jsm") {
  46. suffix = "jsm_index"
  47. } else if strings.Contains(edbCode, "company") {
  48. suffix = "company_index"
  49. } else if strings.Contains(edbCode, "firm") {
  50. suffix = "firm_index"
  51. } else if strings.Contains(edbCode, "coastal") {
  52. suffix = "coastal_index"
  53. } else if strings.Contains(edbCode, "inland") {
  54. suffix = "inland_index"
  55. }
  56. o := orm.NewOrm()
  57. coalBaseDataAll, err := GetBaseFromCoalIndexByCode(suffix, edbCode)
  58. if err != nil && err.Error() != utils.ErrNoRow() {
  59. return
  60. }
  61. var isAdd bool
  62. addSql := ` INSERT INTO edb_data_coal(edb_info_id,edb_code,data_time,value,create_time,modify_time,data_timestamp) values `
  63. existMap := make(map[string]string)
  64. for _, sv := range coalBaseDataAll {
  65. eDate := sv.DataTime
  66. var timeStr string
  67. var dataTime time.Time
  68. var sDataTime string
  69. var timestamp int64
  70. if suffix == "firm_index" {
  71. syear := eDate[:4]
  72. year, _ := strconv.Atoi(syear)
  73. smonth := eDate[7:]
  74. smonth = strings.Split(smonth, "月份")[0]
  75. month, _ := strconv.Atoi(smonth)
  76. var day int
  77. if strings.Contains(eDate, "上旬") {
  78. day = 15
  79. currentLocation := time.Now().Location()
  80. dataTime = time.Date(year, time.Month(month), day, 0, 0, 0, 0, currentLocation)
  81. sDataTime = dataTime.Format(utils.FormatDate)
  82. timestamp = dataTime.UnixNano() / 1e6
  83. timeStr = fmt.Sprintf("%d", timestamp)
  84. } else if strings.Contains(eDate, "中旬") {
  85. day = 25
  86. currentLocation := time.Now().Location()
  87. dataTime = time.Date(year, time.Month(month), day, 0, 0, 0, 0, currentLocation)
  88. sDataTime = dataTime.Format(utils.FormatDate)
  89. timestamp = dataTime.UnixNano() / 1e6
  90. timeStr = fmt.Sprintf("%d", timestamp)
  91. } else {
  92. currentLocation := time.Now().Location()
  93. firstOfMonth := time.Date(year, time.Month(month), 1, 0, 0, 0, 0, currentLocation)
  94. dataTime = firstOfMonth.AddDate(0, 1, -1)
  95. sDataTime = dataTime.Format(utils.FormatDate)
  96. timestamp = dataTime.UnixNano() / 1e6
  97. timeStr = fmt.Sprintf("%d", timestamp)
  98. }
  99. } else {
  100. sDataTime = eDate
  101. dataTime, err = time.ParseInLocation(utils.FormatDate, eDate, time.Local)
  102. if err != nil {
  103. fmt.Println("time.Parse Err:" + eDate)
  104. return err
  105. }
  106. timestamp = dataTime.UnixNano() / 1e6
  107. timeStr = fmt.Sprintf("%d", timestamp)
  108. }
  109. value := strings.Replace(sv.DealValue, "%", "", -1)
  110. if _, ok := existMap[sDataTime]; !ok {
  111. addSql += GetAddSql("0", edbCode, sDataTime, timeStr, value)
  112. fmt.Println("edbCode:", edbCode)
  113. fmt.Println("sDataTime:", sDataTime)
  114. fmt.Println("timeStr:", timeStr)
  115. fmt.Println("value:", value)
  116. isAdd = true
  117. }
  118. existMap[eDate] = value
  119. }
  120. if isAdd {
  121. addSql = strings.TrimRight(addSql, ",")
  122. utils.FileLog.Info("addSql:" + addSql)
  123. _, err = o.Raw(addSql).Exec()
  124. if err != nil {
  125. return err
  126. }
  127. }
  128. return
  129. }
  130. func RefreshEdbDataFromCoal(edbInfoId int, edbCode, startDate string) (err error) {
  131. source := utils.DATA_SOURCE_COAL
  132. o := orm.NewOrm()
  133. if err != nil {
  134. return
  135. }
  136. var suffix string
  137. if strings.Contains(edbCode, "jsm") {
  138. suffix = "jsm_index"
  139. } else if strings.Contains(edbCode, "company") {
  140. suffix = "company_index"
  141. } else if strings.Contains(edbCode, "firm") {
  142. suffix = "firm_index"
  143. } else if strings.Contains(edbCode, "coastal") {
  144. suffix = "coastal_index"
  145. } else if strings.Contains(edbCode, "inland") {
  146. suffix = "inland_index"
  147. }
  148. edbInfoIdStr := strconv.Itoa(edbInfoId)
  149. //计算数据
  150. var condition string
  151. var pars []interface{}
  152. if edbCode != "" {
  153. condition += " AND index_code=? "
  154. pars = append(pars, edbCode)
  155. }
  156. if startDate != "" {
  157. condition += " AND data_time>=? "
  158. pars = append(pars, startDate)
  159. }
  160. glDataList, err := GetCoalDataByTradeCode(suffix, condition, pars)
  161. if err != nil {
  162. return
  163. }
  164. // 真实数据的最大日期 , 插入规则配置的日期
  165. var realDataMaxDate, edbDataInsertConfigDate time.Time
  166. var edbDataInsertConfig *EdbDataInsertConfig
  167. var isFindConfigDateRealData bool //是否找到配置日期的实际数据的值
  168. {
  169. edbDataInsertConfig, err = GetEdbDataInsertConfigByEdbId(edbInfoId)
  170. if err != nil && err.Error() != utils.ErrNoRow() {
  171. return
  172. }
  173. if edbDataInsertConfig != nil {
  174. edbDataInsertConfigDate = edbDataInsertConfig.Date
  175. }
  176. }
  177. //获取指标所有数据
  178. var existCondition string
  179. var existPars []interface{}
  180. existCondition += " AND edb_info_id=? "
  181. existPars = append(existPars, edbInfoId)
  182. if startDate != "" {
  183. existCondition += " AND data_time>=? "
  184. existPars = append(existPars, startDate)
  185. }
  186. existList, err := GetEdbDataByCondition(source, existCondition, existPars)
  187. if err != nil {
  188. return err
  189. }
  190. existMap := make(map[string]*EdbInfoSearchData)
  191. for _, v := range existList {
  192. existMap[v.DataTime] = v
  193. }
  194. addSql := ` INSERT INTO edb_data_coal(edb_info_id,edb_code,data_time,value,create_time,modify_time,data_timestamp) values `
  195. var isAdd bool
  196. for _, v := range glDataList {
  197. var value string
  198. value = strings.Replace(v.DealValue, "%", "", -1)
  199. item := v
  200. itemValue := value
  201. if _, ok := existMap[v.DataTime]; !ok {
  202. eDate := item.DataTime
  203. var timeStr string
  204. var dataTime time.Time
  205. var sDataTime string
  206. var timestamp int64
  207. if suffix == "firm_index" {
  208. syear := eDate[:4]
  209. year, _ := strconv.Atoi(syear)
  210. smonth := eDate[7:]
  211. smonth = strings.Split(smonth, "月份")[0]
  212. month, _ := strconv.Atoi(smonth)
  213. var day int
  214. if strings.Contains(eDate, "上旬") {
  215. day = 15
  216. currentLocation := time.Now().Location()
  217. dataTime = time.Date(year, time.Month(month), day, 0, 0, 0, 0, currentLocation)
  218. sDataTime = dataTime.Format(utils.FormatDate)
  219. timestamp = dataTime.UnixNano() / 1e6
  220. timeStr = fmt.Sprintf("%d", timestamp)
  221. } else if strings.Contains(eDate, "中旬") {
  222. day = 25
  223. currentLocation := time.Now().Location()
  224. dataTime = time.Date(year, time.Month(month), day, 0, 0, 0, 0, currentLocation)
  225. sDataTime = dataTime.Format(utils.FormatDate)
  226. timestamp = dataTime.UnixNano() / 1e6
  227. timeStr = fmt.Sprintf("%d", timestamp)
  228. } else {
  229. currentLocation := time.Now().Location()
  230. firstOfMonth := time.Date(year, time.Month(month), 1, 0, 0, 0, 0, currentLocation)
  231. dataTime = firstOfMonth.AddDate(0, 1, -1)
  232. sDataTime = dataTime.Format(utils.FormatDate)
  233. timestamp = dataTime.UnixNano() / 1e6
  234. timeStr = fmt.Sprintf("%d", timestamp)
  235. }
  236. } else {
  237. sDataTime = eDate
  238. dataTime, err = time.ParseInLocation(utils.FormatDate, eDate, time.Local)
  239. if err != nil {
  240. fmt.Println("time.Parse Err:" + eDate)
  241. return err
  242. }
  243. timestamp = dataTime.UnixNano() / 1e6
  244. timeStr = fmt.Sprintf("%d", timestamp)
  245. }
  246. sValue := itemValue
  247. if sValue != "" {
  248. saveValue := sValue
  249. if findItem, ok := existMap[eDate]; !ok {
  250. addSql += GetAddSql(edbInfoIdStr, edbCode, sDataTime, timeStr, saveValue)
  251. isAdd = true
  252. } else {
  253. if findItem != nil && utils.SubFloatToString(findItem.Value, 30) != sValue {
  254. err = ModifyEdbDataById(source, findItem.EdbDataId, sValue)
  255. if err != nil {
  256. return err
  257. }
  258. }
  259. }
  260. }
  261. // 下面代码主要目的是处理掉手动插入的数据判断
  262. {
  263. if realDataMaxDate.IsZero() || dataTime.After(realDataMaxDate) {
  264. realDataMaxDate = dataTime
  265. }
  266. if edbDataInsertConfigDate.IsZero() || dataTime.Equal(edbDataInsertConfigDate) {
  267. isFindConfigDateRealData = true
  268. }
  269. }
  270. }
  271. }
  272. // 处理手工数据补充的配置
  273. HandleConfigInsertEdbData(realDataMaxDate, edbDataInsertConfig, edbInfoId, source, existMap, isFindConfigDateRealData)
  274. if isAdd {
  275. addSql = strings.TrimRight(addSql, ",")
  276. _, err = o.Raw(addSql).Exec()
  277. if err != nil {
  278. return err
  279. }
  280. }
  281. return
  282. }