base_from_smm.go 16 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574
  1. package models
  2. import (
  3. "eta/eta_index_lib/utils"
  4. "fmt"
  5. "strconv"
  6. "strings"
  7. "time"
  8. "github.com/beego/beego/v2/client/orm"
  9. )
  10. type BaseFromSmmData struct {
  11. SmmDataId int `orm:"column(smm_data_id);pk"`
  12. BaseFromSmmIndexId int
  13. IndexCode string
  14. DataTime string
  15. Value string
  16. CreateTime time.Time
  17. ModifyTime time.Time
  18. DataTimestamp int64
  19. }
  20. func AddBaseFromSmmData(item []*BaseFromSmmData) (err error) {
  21. o := orm.NewOrm()
  22. _, err = o.InsertMulti(len(item), item)
  23. return
  24. }
  25. func GetBaseFromSmmDataByCondition(condition string, pars []interface{}) (list []*BaseFromSmmData, err error) {
  26. o := orm.NewOrm()
  27. sql := `SELECT * FROM base_from_smm_data WHERE 1=1 `
  28. if condition != "" {
  29. sql += condition
  30. }
  31. _, err = o.Raw(sql, pars).QueryRows(&list)
  32. return
  33. }
  34. // 新增有色指标数据
  35. func AddEdbDataFromSmm(edbCode string, smmBaseDataAll []*BaseFromSmmData) (err error) {
  36. o := orm.NewOrm()
  37. var isAdd bool
  38. addSql := ` INSERT INTO edb_data_ys(edb_info_id,edb_code,data_time,value,create_time,modify_time,data_timestamp) values `
  39. existMap := make(map[string]string)
  40. for _, sv := range smmBaseDataAll {
  41. eDate := sv.DataTime
  42. dataTime, err := time.ParseInLocation(utils.FormatDate, eDate, time.Local)
  43. if err != nil {
  44. return err
  45. }
  46. timestamp := dataTime.UnixNano() / 1e6
  47. timeStr := fmt.Sprintf("%d", timestamp)
  48. if _, ok := existMap[eDate]; !ok {
  49. addSql += GetAddSql("0", edbCode, eDate, timeStr, sv.Value)
  50. isAdd = true
  51. }
  52. existMap[eDate] = sv.Value
  53. }
  54. if isAdd {
  55. addSql = strings.TrimRight(addSql, ",")
  56. utils.FileLog.Info("addSql:" + addSql)
  57. _, err = o.Raw(addSql).Exec()
  58. if err != nil {
  59. return err
  60. }
  61. }
  62. return
  63. }
  64. // RefreshEdbDataFromSmm 刷新有色指标数据
  65. func RefreshEdbDataFromSmm(edbInfoId int, edbCode, startDate string) (err error) {
  66. source := utils.DATA_SOURCE_YS
  67. subSource := utils.DATA_SUB_SOURCE_EDB
  68. o := orm.NewOrm()
  69. if err != nil {
  70. return
  71. }
  72. edbInfoIdStr := strconv.Itoa(edbInfoId)
  73. //计算数据
  74. var condition string
  75. var pars []interface{}
  76. if edbCode != "" {
  77. condition += " AND index_code=? "
  78. pars = append(pars, edbCode)
  79. }
  80. if startDate != "" {
  81. condition += " AND data_time>=? "
  82. pars = append(pars, startDate)
  83. }
  84. smmDataList, err := GetBaseFromSmmDataByCondition(condition, pars)
  85. if err != nil {
  86. return
  87. }
  88. // 真实数据的最大日期 , 插入规则配置的日期
  89. var realDataMaxDate, edbDataInsertConfigDate time.Time
  90. var edbDataInsertConfig *EdbDataInsertConfig
  91. var isFindConfigDateRealData bool //是否找到配置日期的实际数据的值
  92. {
  93. edbDataInsertConfig, err = GetEdbDataInsertConfigByEdbId(edbInfoId)
  94. if err != nil && err.Error() != utils.ErrNoRow() {
  95. return
  96. }
  97. if edbDataInsertConfig != nil {
  98. edbDataInsertConfigDate = edbDataInsertConfig.Date
  99. }
  100. }
  101. var existCondition string
  102. var existPars []interface{}
  103. existCondition += " AND edb_info_id=? "
  104. existPars = append(existPars, edbInfoId)
  105. if startDate != "" {
  106. existCondition += " AND data_time>=? "
  107. existPars = append(existPars, startDate)
  108. }
  109. existList, err := GetEdbDataByCondition(source, subSource, existCondition, existPars)
  110. if err != nil {
  111. return err
  112. }
  113. existMap := make(map[string]*EdbInfoSearchData)
  114. for _, v := range existList {
  115. existMap[v.DataTime] = v
  116. }
  117. addSql := ` INSERT INTO edb_data_ys(edb_info_id,edb_code,data_time,value,create_time,modify_time,data_timestamp) values `
  118. var isAdd bool
  119. for _, v := range smmDataList {
  120. item := v
  121. eDate := item.DataTime
  122. dataTime, err := time.ParseInLocation(utils.FormatDate, eDate, time.Local)
  123. if err != nil {
  124. return err
  125. }
  126. if findItem, ok := existMap[v.DataTime]; !ok {
  127. sValue := item.Value
  128. timestamp := dataTime.UnixNano() / 1e6
  129. timeStr := fmt.Sprintf("%d", timestamp)
  130. addSql += GetAddSql(edbInfoIdStr, edbCode, eDate, timeStr, sValue)
  131. isAdd = true
  132. } else {
  133. if findItem != nil && utils.SubFloatToString(findItem.Value, 30) != item.Value {
  134. err = ModifyEdbDataById(source, subSource, findItem.EdbDataId, item.Value)
  135. if err != nil {
  136. return err
  137. }
  138. }
  139. }
  140. // 下面代码主要目的是处理掉手动插入的数据判断
  141. {
  142. if realDataMaxDate.IsZero() || dataTime.After(realDataMaxDate) {
  143. realDataMaxDate = dataTime
  144. }
  145. if edbDataInsertConfigDate.IsZero() || dataTime.Equal(edbDataInsertConfigDate) {
  146. isFindConfigDateRealData = true
  147. }
  148. }
  149. }
  150. // 处理手工数据补充的配置
  151. HandleConfigInsertEdbData(realDataMaxDate, edbDataInsertConfig, edbInfoId, source, subSource, existMap, isFindConfigDateRealData)
  152. if isAdd {
  153. addSql = strings.TrimRight(addSql, ",")
  154. _, err = o.Raw(addSql).Exec()
  155. if err != nil {
  156. fmt.Println("RefreshEdbDataFromSmm add Err", err.Error())
  157. return
  158. }
  159. }
  160. return
  161. }
  162. type BaseFromSmmIndex struct {
  163. BaseFromSmmIndexId int64 `orm:"column(base_from_smm_index_id);pk"`
  164. ClassifyId int
  165. Interface string
  166. Name string
  167. IndexCode string
  168. IndexName string
  169. Type1 string `orm:"column(type_1)"`
  170. Type2 string `orm:"column(type_2)"`
  171. Type3 string `orm:"column(type_3)"`
  172. Frequency string
  173. Unit string
  174. ApiStartTime string
  175. ApiUpdateTime string
  176. StartTime string
  177. FinishTime string
  178. BaseFileName string
  179. RenameFileName string
  180. StartDate string
  181. EndDate string
  182. TerminalCode string `description:"编码"`
  183. CreateTime time.Time
  184. ModifyTime time.Time
  185. DataState string
  186. ReleaseTime string
  187. }
  188. func (d *BaseFromSmmIndex) AddBaseFromSmmIndex() (lastId int64, err error) {
  189. o := orm.NewOrm()
  190. lastId, err = o.Insert(d)
  191. return
  192. }
  193. func (d *BaseFromSmmIndex) GetSmmIndexItem(indexCode string) (item *BaseFromSmmIndex, err error) {
  194. o := orm.NewOrm()
  195. sql := ` SELECT * FROM base_from_smm_index WHERE index_code=? `
  196. fmt.Println(sql, indexCode)
  197. err = o.Raw(sql, indexCode).QueryRow(&item)
  198. return
  199. }
  200. // Add 新增
  201. func (m *BaseFromSmmIndex) Add() (err error) {
  202. o := orm.NewOrm()
  203. _, err = o.Insert(m)
  204. return
  205. }
  206. func GetBaseFromSmmDataBySmmCode(smmCode string) (list []*BaseFromSmmData, err error) {
  207. o := orm.NewOrm()
  208. sql := ` SELECT * FROM base_from_smm_data WHERE index_code=? `
  209. _, err = o.Raw(sql, smmCode).QueryRows(&list)
  210. return
  211. }
  212. func ModifySmmIndexSort(baseFromSmmIndexId int64) (err error) {
  213. o := orm.NewOrm()
  214. sql := ` UPDATE base_from_smm_index SET sort=? WHERE base_from_smm_index_id=? `
  215. _, err = o.Raw(sql, baseFromSmmIndexId, baseFromSmmIndexId).Exec()
  216. return
  217. }
  218. func GetSmmIndexInfoMaxAndMinInfo(indexCode string) (item *EdbInfoMaxAndMinInfo, err error) {
  219. o := orm.NewOrm()
  220. sql := ` SELECT MIN(data_time) AS min_date,MAX(data_time) AS max_date,MIN(value) AS min_value,MAX(value) AS max_value FROM base_from_smm_data WHERE index_code=? `
  221. err = o.Raw(sql, indexCode).QueryRow(&item)
  222. if err != nil {
  223. return
  224. }
  225. // 获取最新值
  226. var latest_value float64
  227. sql = ` SELECT value AS latest_value FROM base_from_smm_data WHERE index_code=? ORDER BY data_time DESC LIMIT 1 `
  228. err = o.Raw(sql, indexCode).QueryRow(&latest_value)
  229. if err != nil {
  230. return
  231. }
  232. item.LatestValue = latest_value
  233. return
  234. }
  235. func ModifySmmIndexMaxAndMinInfo(indexCode string, item *EdbInfoMaxAndMinInfo) (err error) {
  236. o := orm.NewOrm()
  237. sql := ` UPDATE base_from_smm_index SET start_date=?,end_date=?,end_value=?,modify_time=NOW() WHERE index_code=? `
  238. _, err = o.Raw(sql, item.MinDate, item.MaxDate, item.LatestValue, indexCode).Exec()
  239. return
  240. }
  241. // HandleSmmExcelDataReq SMM的excel数据
  242. type HandleSmmExcelDataReq struct {
  243. BaseFilePath string
  244. RenameFilePath string
  245. IndexName string
  246. Source string
  247. IndexCode string
  248. Unit string
  249. Frequency string
  250. ExcelDataMap map[string]string
  251. TerminalCode string `description:"编码"`
  252. }
  253. // MultiUpdateBaseFromSmmDataValue 批量更新SMM指标数据
  254. func MultiUpdateBaseFromSmmDataValue(items []*BaseFromSmmData) (err error) {
  255. if len(items) == 0 {
  256. return
  257. }
  258. o := orm.NewOrm()
  259. sql := `UPDATE base_from_smm_data SET value = ?, modify_time = NOW() WHERE index_code = ? AND data_time = ? LIMIT 1`
  260. p, err := o.Raw(sql).Prepare()
  261. if err != nil {
  262. return
  263. }
  264. defer func() {
  265. _ = p.Close()
  266. }()
  267. for _, v := range items {
  268. if v.IndexCode == "" || v.DataTime == "" {
  269. continue
  270. }
  271. _, err = p.Exec(v.Value, v.IndexCode, v.DataTime)
  272. if err != nil {
  273. return
  274. }
  275. }
  276. return
  277. }
  278. type AddSmmIndexToDataSourceResp struct {
  279. EdbCode string `description:"指标编码"`
  280. ClassifyId int `description:"指标分类"`
  281. SysUserId int `description:"操作人id"`
  282. SysUserRealName string `description:"操作人真实名称"`
  283. }
  284. func (m *BaseFromSmmIndex) GetIndexItem(indexCode string) (item *BaseFromSmmIndex, err error) {
  285. o := orm.NewOrm()
  286. sql := `SELECT * FROM base_from_smm_index WHERE index_code = ? `
  287. err = o.Raw(sql, indexCode).QueryRow(&item)
  288. return
  289. }
  290. // 修改
  291. func (m *BaseFromSmmIndex) Update(updateParams, whereParam map[string]interface{}) (err error) {
  292. to := orm.NewOrm()
  293. ptrStructOrTableName := "base_from_smm_index"
  294. qs := to.QueryTable(ptrStructOrTableName)
  295. for expr, exprV := range whereParam {
  296. qs = qs.Filter(expr, exprV)
  297. }
  298. _, err = qs.Update(updateParams)
  299. return
  300. }
  301. // BridgeZhongJiIndexDataParams 桥接服务-获取中基指标数据入参
  302. type BridgeZhongJiIndexDataParams struct {
  303. EdbCode string `json:"edb_code" form:"edb_code" description:"指标编码"`
  304. }
  305. type BridgeZhongJiIndexListParams struct {
  306. Page int `json:"page" form:"page" description:"页码"`
  307. PageSize int `json:"page_size" form:"page_size" description:"每页指标个数,每页不能超过500条,超过会固定为500"`
  308. }
  309. type BaseFromSmmDataList struct {
  310. Data []*BaseFromSmmData
  311. IndexCode string
  312. }
  313. type UpdateRule struct {
  314. RuleDesc string `json:"rule_desc"`
  315. Notice string `json:"notice"`
  316. ReleaseTime string `json:"release_time"`
  317. RuleInfo UpdateRuleInfo `json:"rule_info"`
  318. }
  319. type UpdateRuleInfo struct {
  320. UpdCal string `json:"upd_cal"`
  321. UpdCalUnit string `json:"upd_cal_unit"`
  322. PeriodOffset int `json:"period_offset"`
  323. UpdCount int `json:"upd_count"`
  324. }
  325. type DataItem struct {
  326. QuotaID string `json:"quota_id"`
  327. QuotaName string `json:"quota_name"`
  328. Mark string `json:"mark"`
  329. Unit string `json:"unit"`
  330. ValueType string `json:"value_type"`
  331. Source string `json:"source"`
  332. ListOrder int `json:"list_order"`
  333. DataStart string `json:"data_start"`
  334. DataEnd string `json:"data_end"`
  335. Frequency string `json:"frequency"`
  336. Precision int `json:"precision"`
  337. TypeCode string `json:"type_code"`
  338. TypeName string `json:"type_name"`
  339. TypeCodeAll string `json:"type_code_all"`
  340. TypeAll string `json:"type_all"`
  341. Tags string `json:"tags"`
  342. AuthStart string `json:"auth_start"`
  343. AuthFinish string `json:"auth_finish"`
  344. AuthLang string `json:"auth_lang"`
  345. AuthModule string `json:"auth_module"`
  346. DataDur string `json:"data_dur"`
  347. Spec string `json:"spec"`
  348. Area string `json:"area"`
  349. DataState string `json:"data_state"`
  350. UpdateRule UpdateRule `json:"update_rule"`
  351. }
  352. type ListData struct {
  353. Total int `json:"total"`
  354. CurrentPage int `json:"current_page"`
  355. PageSize int `json:"page_size"`
  356. PageCount int `json:"page_count"`
  357. Data []DataItem `json:"data"`
  358. }
  359. type SmmIndexListResponse struct {
  360. Code int `json:"code"`
  361. Msg string `json:"msg"`
  362. Data ListData `json:"data"`
  363. }
  364. type SmmListResp struct {
  365. Code int `json:"code"`
  366. Msg string `json:"msg"`
  367. Data SmmIndexListResponse `json:"data"`
  368. }
  369. func AddBaseFromSmmIndex(list []*BaseFromSmmIndex) (lastId int64, err error) {
  370. o := orm.NewOrm()
  371. _, err = o.InsertMulti(len(list), list)
  372. return
  373. }
  374. // BridgeZhongJiIndexLatestDataParams 桥接服务-获取中基更新指标数据入参
  375. type BridgeZhongJiIndexLatestDataParams struct {
  376. EdbCode string `json:"edb_code" form:"edb_code" description:"指标编码"`
  377. LastTime int64 `json:"last_time" form:"last_time" description:"上次更新时间戳(秒)"`
  378. }
  379. func GetBaseFromSmmIndex() (list []*BaseFromSmmIndex, err error) {
  380. o := orm.NewOrm()
  381. sql := ` SELECT * FROM base_from_smm_index`
  382. _, err = o.Raw(sql).QueryRows(&list)
  383. return
  384. }
  385. func ModifyBaseFromSmmIndex(item *BaseFromSmmIndex) (err error) {
  386. o := orm.NewOrm()
  387. sql := ` UPDATE base_from_smm_index SET end_date = ?, modify_time=NOW(), data_state= ? `
  388. _, err = o.Raw(sql, item.EndDate, item.DataState).Exec()
  389. return
  390. }
  391. type SmmLatestDataResponse struct {
  392. Code int `json:"code"`
  393. Msg string `json:"msg"`
  394. Data SmmLatestResp `json:"data"`
  395. }
  396. type SmmLatestResp struct {
  397. Code int `json:"code"`
  398. Msg string `json:"msg"`
  399. Data SmmLatestData `json:"data"`
  400. }
  401. type SmmLatestData struct {
  402. DataLen int `json:"data_len"`
  403. DataList []Datum `json:"data_list"`
  404. }
  405. type Datum struct {
  406. QuotaID string `json:"quota_id"`
  407. Date string `json:"date"`
  408. Value string `json:"value"`
  409. Mark string `json:"mark"`
  410. Create string `json:"create"`
  411. Update string `json:"update"`
  412. }
  413. // RefreshEdbDataFromSmmToEdb 刷新有色指标数据
  414. func RefreshEdbDataFromSmmToEdb(edbInfoId int, edbCode, startDate string, smmDataList []*BaseFromSmmData) (err error) {
  415. source := utils.DATA_SOURCE_YS
  416. subSource := utils.DATA_SUB_SOURCE_EDB
  417. o := orm.NewOrm()
  418. if err != nil {
  419. return
  420. }
  421. edbInfoIdStr := strconv.Itoa(edbInfoId)
  422. //计算数据
  423. var condition string
  424. var pars []interface{}
  425. if edbCode != "" {
  426. condition += " AND index_code=? "
  427. pars = append(pars, edbCode)
  428. }
  429. if startDate != "" {
  430. condition += " AND data_time>=? "
  431. pars = append(pars, startDate)
  432. }
  433. // 真实数据的最大日期 , 插入规则配置的日期
  434. var realDataMaxDate, edbDataInsertConfigDate time.Time
  435. var edbDataInsertConfig *EdbDataInsertConfig
  436. var isFindConfigDateRealData bool //是否找到配置日期的实际数据的值
  437. {
  438. edbDataInsertConfig, err = GetEdbDataInsertConfigByEdbId(edbInfoId)
  439. if err != nil && err.Error() != utils.ErrNoRow() {
  440. return
  441. }
  442. if edbDataInsertConfig != nil {
  443. edbDataInsertConfigDate = edbDataInsertConfig.Date
  444. }
  445. }
  446. var existCondition string
  447. var existPars []interface{}
  448. existCondition += " AND edb_info_id=? "
  449. existPars = append(existPars, edbInfoId)
  450. if startDate != "" {
  451. existCondition += " AND data_time>=? "
  452. existPars = append(existPars, startDate)
  453. }
  454. existList, err := GetEdbDataByCondition(source, subSource, existCondition, existPars)
  455. if err != nil {
  456. return err
  457. }
  458. existMap := make(map[string]*EdbInfoSearchData)
  459. for _, v := range existList {
  460. existMap[v.DataTime] = v
  461. }
  462. addSql := ` INSERT INTO edb_data_ys(edb_info_id,edb_code,data_time,value,create_time,modify_time,data_timestamp) values `
  463. var isAdd bool
  464. for _, v := range smmDataList {
  465. item := v
  466. eDate := item.DataTime
  467. dataTime, err := time.ParseInLocation(utils.FormatDate, eDate, time.Local)
  468. if err != nil {
  469. return err
  470. }
  471. if findItem, ok := existMap[v.DataTime]; !ok {
  472. sValue := item.Value
  473. timestamp := dataTime.UnixNano() / 1e6
  474. timeStr := fmt.Sprintf("%d", timestamp)
  475. addSql += GetAddSql(edbInfoIdStr, edbCode, eDate, timeStr, sValue)
  476. isAdd = true
  477. } else {
  478. if findItem != nil && utils.SubFloatToString(findItem.Value, 30) != item.Value {
  479. err = ModifyEdbDataById(source, subSource, findItem.EdbDataId, item.Value)
  480. if err != nil {
  481. return err
  482. }
  483. }
  484. }
  485. // 下面代码主要目的是处理掉手动插入的数据判断
  486. {
  487. if realDataMaxDate.IsZero() || dataTime.After(realDataMaxDate) {
  488. realDataMaxDate = dataTime
  489. }
  490. if edbDataInsertConfigDate.IsZero() || dataTime.Equal(edbDataInsertConfigDate) {
  491. isFindConfigDateRealData = true
  492. }
  493. }
  494. }
  495. // 处理手工数据补充的配置
  496. HandleConfigInsertEdbData(realDataMaxDate, edbDataInsertConfig, edbInfoId, source, subSource, existMap, isFindConfigDateRealData)
  497. if isAdd {
  498. addSql = strings.TrimRight(addSql, ",")
  499. _, err = o.Raw(addSql).Exec()
  500. if err != nil {
  501. fmt.Println("RefreshEdbDataFromSmm add Err", err.Error())
  502. return
  503. }
  504. }
  505. return
  506. }