base_from_coal.go 9.5 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311
  1. package models
  2. import (
  3. "eta/eta_index_lib/global"
  4. "eta/eta_index_lib/utils"
  5. "fmt"
  6. "strconv"
  7. "strings"
  8. "time"
  9. )
  10. type BaseFromCoalmineIndex struct {
  11. BaseFromCoalmineCompanyIndexId int
  12. IndexName string `description:"持买单量指标名称"`
  13. IndexCode string `description:"持买单量指标编码"`
  14. DealValue string `description:"成交量"`
  15. DataTime string `description:"数据日期"`
  16. Source string `description:"来源"`
  17. Province string `description:"省份"`
  18. City string `description:"城市"`
  19. GroupName string `description:"集团名称"`
  20. Unit string `description:"单位"`
  21. Frequency string `description:"频率"`
  22. CreateTime string `description:"插入时间"`
  23. ModifyTime string `description:"修改时间"`
  24. }
  25. func GetBaseFromCoalIndexByCode(suffix, indexCode string) (items []*BaseFromCoalmineIndex, err error) {
  26. //o := orm.NewOrm()
  27. sql := `SELECT * FROM base_from_coalmine_%s WHERE index_code=? `
  28. sql = fmt.Sprintf(sql, suffix)
  29. //_, err = o.Raw(sql, indexCode).QueryRows(&items)
  30. err = global.DEFAULT_DB.Raw(sql, indexCode).Find(&items).Error
  31. return
  32. }
  33. func GetCoalDataByTradeCode(suffix, condition string, pars []interface{}) (item []*BaseFromCoalmineIndex, err error) {
  34. sql := ` SELECT * FROM base_from_coalmine_%s WHERE 1=1 `
  35. sql = fmt.Sprintf(sql, suffix)
  36. //o := orm.NewOrm()
  37. if condition != "" {
  38. sql += condition
  39. }
  40. sql += ` ORDER BY data_time DESC `
  41. err = global.DEFAULT_DB.Raw(sql, pars...).Find(&item).Error
  42. //_, err = o.Raw(sql, pars).QueryRows(&item)
  43. return
  44. }
  45. func AddEdbDataFromCoal(edbCode string) (err error) {
  46. var suffix string
  47. if strings.Contains(edbCode, "jsm") {
  48. suffix = "jsm_index"
  49. } else if strings.Contains(edbCode, "company") {
  50. suffix = "company_index"
  51. } else if strings.Contains(edbCode, "firm") {
  52. suffix = "firm_index"
  53. } else if strings.Contains(edbCode, "coastal") {
  54. suffix = "coastal_index"
  55. } else if strings.Contains(edbCode, "inland") {
  56. suffix = "inland_index"
  57. }
  58. //o := orm.NewOrm()
  59. coalBaseDataAll, err := GetBaseFromCoalIndexByCode(suffix, edbCode)
  60. if err != nil && !utils.IsErrNoRow(err) {
  61. return
  62. }
  63. var isAdd bool
  64. addSql := ` INSERT INTO edb_data_coal(edb_info_id,edb_code,data_time,value,create_time,modify_time,data_timestamp) values `
  65. existMap := make(map[string]string)
  66. for _, sv := range coalBaseDataAll {
  67. eDate := sv.DataTime
  68. var timeStr string
  69. var dataTime time.Time
  70. var sDataTime string
  71. var timestamp int64
  72. if suffix == "firm_index" {
  73. syear := eDate[:4]
  74. year, _ := strconv.Atoi(syear)
  75. smonth := eDate[7:]
  76. smonth = strings.Split(smonth, "月份")[0]
  77. month, _ := strconv.Atoi(smonth)
  78. var day int
  79. if strings.Contains(eDate, "上旬") {
  80. day = 15
  81. currentLocation := time.Now().Location()
  82. dataTime = time.Date(year, time.Month(month), day, 0, 0, 0, 0, currentLocation)
  83. sDataTime = dataTime.Format(utils.FormatDate)
  84. timestamp = dataTime.UnixNano() / 1e6
  85. timeStr = fmt.Sprintf("%d", timestamp)
  86. } else if strings.Contains(eDate, "中旬") {
  87. day = 25
  88. currentLocation := time.Now().Location()
  89. dataTime = time.Date(year, time.Month(month), day, 0, 0, 0, 0, currentLocation)
  90. sDataTime = dataTime.Format(utils.FormatDate)
  91. timestamp = dataTime.UnixNano() / 1e6
  92. timeStr = fmt.Sprintf("%d", timestamp)
  93. } else {
  94. currentLocation := time.Now().Location()
  95. firstOfMonth := time.Date(year, time.Month(month), 1, 0, 0, 0, 0, currentLocation)
  96. dataTime = firstOfMonth.AddDate(0, 1, -1)
  97. sDataTime = dataTime.Format(utils.FormatDate)
  98. timestamp = dataTime.UnixNano() / 1e6
  99. timeStr = fmt.Sprintf("%d", timestamp)
  100. }
  101. } else {
  102. sDataTime = eDate
  103. dataTime, err = time.ParseInLocation(utils.FormatDate, eDate, time.Local)
  104. if err != nil {
  105. fmt.Println("time.Parse Err:" + eDate)
  106. return err
  107. }
  108. timestamp = dataTime.UnixNano() / 1e6
  109. timeStr = fmt.Sprintf("%d", timestamp)
  110. }
  111. value := strings.Replace(sv.DealValue, "%", "", -1)
  112. if _, ok := existMap[sDataTime]; !ok {
  113. addSql += GetAddSql("0", edbCode, sDataTime, timeStr, value)
  114. fmt.Println("edbCode:", edbCode)
  115. fmt.Println("sDataTime:", sDataTime)
  116. fmt.Println("timeStr:", timeStr)
  117. fmt.Println("value:", value)
  118. isAdd = true
  119. }
  120. existMap[eDate] = value
  121. }
  122. if isAdd {
  123. addSql = strings.TrimRight(addSql, ",")
  124. utils.FileLog.Info("addSql:" + addSql)
  125. //_, err = o.Raw(addSql).Exec()
  126. err = global.DEFAULT_DB.Exec(addSql).Error
  127. if err != nil {
  128. return err
  129. }
  130. }
  131. return
  132. }
  133. func RefreshEdbDataFromCoal(edbInfoId int, edbCode, startDate string) (err error) {
  134. source := utils.DATA_SOURCE_COAL
  135. subSource := utils.DATA_SUB_SOURCE_EDB
  136. //o := orm.NewOrm()
  137. if err != nil {
  138. return
  139. }
  140. var suffix string
  141. if strings.Contains(edbCode, "jsm") {
  142. suffix = "jsm_index"
  143. } else if strings.Contains(edbCode, "company") {
  144. suffix = "company_index"
  145. } else if strings.Contains(edbCode, "firm") {
  146. suffix = "firm_index"
  147. } else if strings.Contains(edbCode, "coastal") {
  148. suffix = "coastal_index"
  149. } else if strings.Contains(edbCode, "inland") {
  150. suffix = "inland_index"
  151. }
  152. edbInfoIdStr := strconv.Itoa(edbInfoId)
  153. //计算数据
  154. var condition string
  155. var pars []interface{}
  156. if edbCode != "" {
  157. condition += " AND index_code=? "
  158. pars = append(pars, edbCode)
  159. }
  160. if startDate != "" {
  161. condition += " AND data_time>=? "
  162. pars = append(pars, startDate)
  163. }
  164. glDataList, err := GetCoalDataByTradeCode(suffix, condition, pars)
  165. if err != nil {
  166. return
  167. }
  168. // 真实数据的最大日期 , 插入规则配置的日期
  169. var realDataMaxDate, edbDataInsertConfigDate time.Time
  170. var edbDataInsertConfig *EdbDataInsertConfig
  171. var isFindConfigDateRealData bool //是否找到配置日期的实际数据的值
  172. {
  173. edbDataInsertConfig, err = GetEdbDataInsertConfigByEdbId(edbInfoId)
  174. if err != nil && !utils.IsErrNoRow(err) {
  175. return
  176. }
  177. if edbDataInsertConfig != nil {
  178. edbDataInsertConfigDate = edbDataInsertConfig.Date
  179. }
  180. }
  181. //获取指标所有数据
  182. var existCondition string
  183. var existPars []interface{}
  184. existCondition += " AND edb_info_id=? "
  185. existPars = append(existPars, edbInfoId)
  186. if startDate != "" {
  187. existCondition += " AND data_time>=? "
  188. existPars = append(existPars, startDate)
  189. }
  190. existList, err := GetEdbDataByCondition(source, subSource, existCondition, existPars)
  191. if err != nil {
  192. return err
  193. }
  194. existMap := make(map[string]*EdbInfoSearchData)
  195. for _, v := range existList {
  196. existMap[v.DataTime] = v
  197. }
  198. addSql := ` INSERT INTO edb_data_coal(edb_info_id,edb_code,data_time,value,create_time,modify_time,data_timestamp) values `
  199. var isAdd bool
  200. for _, v := range glDataList {
  201. var value string
  202. value = strings.Replace(v.DealValue, "%", "", -1)
  203. item := v
  204. itemValue := value
  205. if _, ok := existMap[v.DataTime]; !ok {
  206. eDate := item.DataTime
  207. var timeStr string
  208. var dataTime time.Time
  209. var sDataTime string
  210. var timestamp int64
  211. if suffix == "firm_index" {
  212. syear := eDate[:4]
  213. year, _ := strconv.Atoi(syear)
  214. smonth := eDate[7:]
  215. smonth = strings.Split(smonth, "月份")[0]
  216. month, _ := strconv.Atoi(smonth)
  217. var day int
  218. if strings.Contains(eDate, "上旬") {
  219. day = 15
  220. currentLocation := time.Now().Location()
  221. dataTime = time.Date(year, time.Month(month), day, 0, 0, 0, 0, currentLocation)
  222. sDataTime = dataTime.Format(utils.FormatDate)
  223. timestamp = dataTime.UnixNano() / 1e6
  224. timeStr = fmt.Sprintf("%d", timestamp)
  225. } else if strings.Contains(eDate, "中旬") {
  226. day = 25
  227. currentLocation := time.Now().Location()
  228. dataTime = time.Date(year, time.Month(month), day, 0, 0, 0, 0, currentLocation)
  229. sDataTime = dataTime.Format(utils.FormatDate)
  230. timestamp = dataTime.UnixNano() / 1e6
  231. timeStr = fmt.Sprintf("%d", timestamp)
  232. } else {
  233. currentLocation := time.Now().Location()
  234. firstOfMonth := time.Date(year, time.Month(month), 1, 0, 0, 0, 0, currentLocation)
  235. dataTime = firstOfMonth.AddDate(0, 1, -1)
  236. sDataTime = dataTime.Format(utils.FormatDate)
  237. timestamp = dataTime.UnixNano() / 1e6
  238. timeStr = fmt.Sprintf("%d", timestamp)
  239. }
  240. } else {
  241. sDataTime = eDate
  242. dataTime, err = time.ParseInLocation(utils.FormatDate, eDate, time.Local)
  243. if err != nil {
  244. fmt.Println("time.Parse Err:" + eDate)
  245. return err
  246. }
  247. timestamp = dataTime.UnixNano() / 1e6
  248. timeStr = fmt.Sprintf("%d", timestamp)
  249. }
  250. sValue := itemValue
  251. if sValue != "" {
  252. saveValue := sValue
  253. if findItem, ok := existMap[eDate]; !ok {
  254. addSql += GetAddSql(edbInfoIdStr, edbCode, sDataTime, timeStr, saveValue)
  255. isAdd = true
  256. } else {
  257. if findItem != nil && utils.SubFloatToString(findItem.Value, 30) != sValue {
  258. err = ModifyEdbDataById(source, subSource, findItem.EdbDataId, sValue)
  259. if err != nil {
  260. return err
  261. }
  262. }
  263. }
  264. }
  265. // 下面代码主要目的是处理掉手动插入的数据判断
  266. {
  267. if realDataMaxDate.IsZero() || dataTime.After(realDataMaxDate) {
  268. realDataMaxDate = dataTime
  269. }
  270. if edbDataInsertConfigDate.IsZero() || dataTime.Equal(edbDataInsertConfigDate) {
  271. isFindConfigDateRealData = true
  272. }
  273. }
  274. }
  275. }
  276. // 处理手工数据补充的配置
  277. HandleConfigInsertEdbData(realDataMaxDate, edbDataInsertConfig, edbInfoId, source, subSource, existMap, isFindConfigDateRealData)
  278. if isAdd {
  279. addSql = strings.TrimRight(addSql, ",")
  280. //_, err = o.Raw(addSql).Exec()
  281. err = global.DEFAULT_DB.Exec(addSql).Error
  282. if err != nil {
  283. return err
  284. }
  285. }
  286. return
  287. }