temp_fix.go 21 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533
  1. package services
  2. import (
  3. aiPredictModel "eta/eta_api/models/ai_predict_model"
  4. "eta/eta_api/models/data_manage/excel"
  5. dataSourceModel "eta/eta_api/models/data_source"
  6. "eta/eta_api/services/elastic"
  7. excel2 "eta/eta_api/services/excel"
  8. "eta/eta_api/utils"
  9. "fmt"
  10. )
  11. // FixNewEs 修复新的es数据
  12. func FixNewEs() {
  13. fmt.Println("开始修复")
  14. var err error
  15. defer func() {
  16. if err != nil {
  17. tips := fmt.Sprintf("修复es历史数据失败, %v", err)
  18. fmt.Println(tips)
  19. utils.FileLog.Info(tips)
  20. }
  21. }()
  22. // 表格
  23. if utils.EsExcelIndexName == "" {
  24. err = fmt.Errorf("表格索引名为空")
  25. return
  26. }
  27. fmt.Println("开始创建表格索引:", utils.EsExcelIndexName)
  28. indexMapping := `{"mappings":{"properties":{"ExcelInfoId":{"type":"long"},"Source":{"type":"long"},"ExcelType":{"type":"long"},"ExcelName":{"type":"text","fields":{"keyword":{"type":"keyword","ignore_above":256}}},"UniqueCode":{"type":"text","fields":{"keyword":{"type":"keyword","ignore_above":256}}},"ExcelClassifyId":{"type":"long"},"SysUserId":{"type":"long"},"SysUserRealName":{"type":"text","fields":{"keyword":{"type":"keyword","ignore_above":256}}},"Content":{"type":"text","fields":{"keyword":{"type":"keyword","ignore_above":256}}},"ExcelImage":{"type":"text","fields":{"keyword":{"type":"keyword","ignore_above":256}}},"FileUrl":{"type":"text","fields":{"keyword":{"type":"keyword","ignore_above":256}}},"Sort":{"type":"long"},"IsDelete":{"type":"long"},"ModifyTime":{"type":"text","fields":{"keyword":{"type":"keyword","ignore_above":256}}},"CreateTime":{"type":"text","fields":{"keyword":{"type":"keyword","ignore_above":256}}},"IsJoinPermission":{"type":"long"},"ParentId":{"type":"long"},"BalanceType":{"type":"long"},"UpdateUserId":{"type":"long"},"UpdateUserRealName":{"type":"text","fields":{"keyword":{"type":"keyword","ignore_above":256}}},"RelExcelInfoId":{"type":"long"},"VersionName":{"type":"text","fields":{"keyword":{"type":"keyword","ignore_above":256}}},"SourcesFrom":{"type":"text","fields":{"keyword":{"type":"keyword","ignore_above":256}}}}}}`
  29. if e := elastic.CreateEsIndex(utils.EsExcelIndexName, indexMapping); e != nil {
  30. err = fmt.Errorf("创建excel索引失败, %v", e)
  31. return
  32. }
  33. fmt.Println("表格索引创建成功")
  34. fmt.Println("开始修复表格数据")
  35. excels, e := excel.GetExcelInfoAll()
  36. if e != nil {
  37. err = fmt.Errorf("获取所有excel表格失败, %v", e)
  38. return
  39. }
  40. for _, v := range excels {
  41. fmt.Printf("ExcelId: %d 修复中\n", v.ExcelInfoId)
  42. excel2.EsAddOrEditExcel(v.ExcelInfoId)
  43. }
  44. fmt.Println("表格数据修复成功")
  45. // 数据源
  46. fmt.Println("数据源索引名:", utils.EsDataSourceIndexName)
  47. if utils.EsDataSourceIndexName == "" {
  48. err = fmt.Errorf("数据源索引名为空")
  49. return
  50. }
  51. fmt.Println("开始创建数据源索引:", utils.EsDataSourceIndexName)
  52. indexMappingB := `{"mappings":{"properties":{"ClassifyId":{"type":"long"},"CreateTime":{"type":"text","fields":{"keyword":{"type":"keyword","ignore_above":256}}},"EndDate":{"type":"text","fields":{"keyword":{"type":"keyword","ignore_above":256}}},"Frequency":{"type":"text","fields":{"keyword":{"type":"keyword","ignore_above":256}}},"IndexCode":{"type":"text","fields":{"keyword":{"type":"keyword","ignore_above":256}}},"IndexName":{"type":"text","fields":{"keyword":{"type":"keyword","ignore_above":256}}},"IsDeleted":{"type":"long"},"LatestValue":{"type":"text"},"ModifyTime":{"type":"text","fields":{"keyword":{"type":"keyword","ignore_above":256}}},"PrimaryId":{"type":"long"},"Source":{"type":"long"},"SourceName":{"type":"text","fields":{"keyword":{"type":"keyword","ignore_above":256}}},"SubSource":{"type":"long"},"StartDate":{"type":"text","fields":{"keyword":{"type":"keyword","ignore_above":256}}},"Unit":{"type":"text","fields":{"keyword":{"type":"keyword","ignore_above":256}}}}}}`
  53. //indexMappingB := `{"settings":{"analysis":{"analyzer":{"ik_analyzer":{"type":"custom","tokenizer":"ik_max_word"},"ngram_analyzer":{"type":"custom","tokenizer":"standard","filter":["lowercase","ngram_filter"]}},"filter":{"ngram_filter":{"type":"ngram","min_gram":1,"max_gram":2,"token_chars":["letter","digit"]}}}},"mappings":{"properties":{"ClassifyId":{"type":"long"},"CreateTime":{"type":"text","fields":{"keyword":{"type":"keyword","ignore_above":256}}},"EndDate":{"type":"text","fields":{"keyword":{"type":"keyword","ignore_above":256}}},"Frequency":{"type":"text","analyzer":"ik_smart"},"IndexCode":{"type":"text","fields":{"ngram":{"type":"text","analyzer":"ngram_analyzer","search_analyzer":"standard"},"keyword":{"type":"keyword","ignore_above":256}}},"IndexName":{"type":"text","fields":{"ik":{"type":"text","analyzer":"ik_analyzer","search_analyzer":"ik_analyzer"},"ngram":{"type":"text","analyzer":"ngram_analyzer","search_analyzer":"standard"},"keyword":{"type":"keyword","ignore_above":256}}},"IsDeleted":{"type":"long"},"LatestValue":{"type":"double"},"ModifyTime":{"type":"text","fields":{"keyword":{"type":"keyword","ignore_above":256}}},"PrimaryId":{"type":"long"},"Source":{"type":"long"},"SourceName":{"type":"text","analyzer":"ik_smart"},"SubSource":{"type":"long"},"StartDate":{"type":"text","fields":{"keyword":{"type":"keyword","ignore_above":256}}},"Unit":{"type":"text","analyzer":"ik_smart"}}}}`
  54. if e := elastic.CreateEsIndex(utils.EsDataSourceIndexName, indexMappingB); e != nil {
  55. err = fmt.Errorf("创建数据源索引失败, %v", e)
  56. return
  57. }
  58. fmt.Println("数据源索引创建成功")
  59. fmt.Println("开始修复数据源")
  60. var cond string
  61. var pars []interface{}
  62. // 睿咨得
  63. rzdOb := new(dataSourceModel.BaseFromRzdIndex)
  64. {
  65. list, e := rzdOb.GetItemsByCondition(cond, pars, []string{}, "")
  66. if e != nil {
  67. err = fmt.Errorf("获取睿咨得失败, %v", e)
  68. return
  69. }
  70. for _, v := range list {
  71. indexItem := v.Format2SearchDataSource(v)
  72. fmt.Printf("写入中:睿咨得-%d\n", indexItem.PrimaryId)
  73. docId := fmt.Sprintf("%d-%d", indexItem.Source, indexItem.PrimaryId)
  74. if e := elastic.EsAddOrEditDataSourceIndex(utils.EsDataSourceIndexName, docId, indexItem); e != nil {
  75. err = fmt.Errorf("睿咨得-写入es失败, %v", e)
  76. return
  77. }
  78. }
  79. }
  80. // 泛糖科技
  81. hisugarOb := new(dataSourceModel.BaseFromHisugarIndex)
  82. {
  83. list, e := hisugarOb.GetItemsByCondition(cond, pars, []string{}, "")
  84. if e != nil {
  85. err = fmt.Errorf("获取泛糖科技失败, %v", e)
  86. return
  87. }
  88. for _, v := range list {
  89. indexItem := v.Format2SearchDataSource(v)
  90. fmt.Printf("写入中:泛糖科技-%d\n", indexItem.PrimaryId)
  91. docId := fmt.Sprintf("%d-%d", indexItem.Source, indexItem.PrimaryId)
  92. if e := elastic.EsAddOrEditDataSourceIndex(utils.EsDataSourceIndexName, docId, indexItem); e != nil {
  93. err = fmt.Errorf("泛糖科技-写入es失败, %v", e)
  94. return
  95. }
  96. }
  97. }
  98. // 粮油商务网
  99. lyOb := new(dataSourceModel.BaseFromLyIndex)
  100. {
  101. list, e := lyOb.GetItemsByCondition(cond, pars, []string{}, "")
  102. if e != nil {
  103. err = fmt.Errorf("获取粮油商务网失败, %v", e)
  104. return
  105. }
  106. for _, v := range list {
  107. indexItem := v.Format2SearchDataSource(v)
  108. fmt.Printf("写入中:粮油商务网-%d\n", indexItem.PrimaryId)
  109. docId := fmt.Sprintf("%d-%d", indexItem.Source, indexItem.PrimaryId)
  110. if e := elastic.EsAddOrEditDataSourceIndex(utils.EsDataSourceIndexName, docId, indexItem); e != nil {
  111. err = fmt.Errorf("粮油商务网-写入es失败, %v", e)
  112. return
  113. }
  114. }
  115. }
  116. // 卓创红期
  117. sciHqOb := new(dataSourceModel.BaseFromSciHqIndex)
  118. {
  119. list, e := sciHqOb.GetItemsByCondition(cond, pars, []string{}, "")
  120. if e != nil {
  121. err = fmt.Errorf("获取卓创红期失败, %v", e)
  122. return
  123. }
  124. for _, v := range list {
  125. indexItem := v.Format2SearchDataSource(v)
  126. fmt.Printf("写入中:卓创红期-%d\n", indexItem.PrimaryId)
  127. docId := fmt.Sprintf("%d-%d", indexItem.Source, indexItem.PrimaryId)
  128. if e := elastic.EsAddOrEditDataSourceIndex(utils.EsDataSourceIndexName, docId, indexItem); e != nil {
  129. err = fmt.Errorf("卓创红期-写入es失败, %v", e)
  130. return
  131. }
  132. }
  133. }
  134. // 同花顺高频
  135. thsHfOb := new(dataSourceModel.BaseFromThsHfIndex)
  136. {
  137. list, e := thsHfOb.GetItemsByCondition(cond, pars, []string{}, "")
  138. if e != nil {
  139. err = fmt.Errorf("获取同花顺高频失败, %v", e)
  140. return
  141. }
  142. for _, v := range list {
  143. indexItem := v.Format2SearchDataSource(v)
  144. fmt.Printf("写入中:同花顺高频-%d\n", indexItem.PrimaryId)
  145. docId := fmt.Sprintf("%d-%d", indexItem.Source, indexItem.PrimaryId)
  146. if e := elastic.EsAddOrEditDataSourceIndex(utils.EsDataSourceIndexName, docId, indexItem); e != nil {
  147. err = fmt.Errorf("同花顺高频-写入es失败, %v", e)
  148. return
  149. }
  150. }
  151. }
  152. // 隆众资讯
  153. oilchemOb := new(dataSourceModel.BaseFromOilchemIndex)
  154. {
  155. list, e := oilchemOb.GetItemsByCondition(cond, pars, []string{}, "")
  156. if e != nil {
  157. err = fmt.Errorf("获取隆众资讯失败, %v", e)
  158. return
  159. }
  160. for _, v := range list {
  161. indexItem := v.Format2SearchDataSource(v)
  162. fmt.Printf("写入中:隆众资讯-%d\n", indexItem.PrimaryId)
  163. docId := fmt.Sprintf("%d-%d", indexItem.Source, indexItem.PrimaryId)
  164. if e := elastic.EsAddOrEditDataSourceIndex(utils.EsDataSourceIndexName, docId, indexItem); e != nil {
  165. err = fmt.Errorf("隆众资讯-写入es失败, %v", e)
  166. return
  167. }
  168. }
  169. }
  170. // CCF化纤信息
  171. ccfOb := new(dataSourceModel.BaseFromCcfIndex)
  172. {
  173. list, e := ccfOb.GetItemsByCondition(cond, pars, []string{}, "")
  174. if e != nil {
  175. err = fmt.Errorf("获取CCF化纤信息失败, %v", e)
  176. return
  177. }
  178. for _, v := range list {
  179. indexItem := v.Format2SearchDataSource(v)
  180. fmt.Printf("写入中:CCF化纤信息-%d\n", indexItem.PrimaryId)
  181. docId := fmt.Sprintf("%d-%d", indexItem.Source, indexItem.PrimaryId)
  182. if e := elastic.EsAddOrEditDataSourceIndex(utils.EsDataSourceIndexName, docId, indexItem); e != nil {
  183. err = fmt.Errorf("CCF化纤信息-写入es失败, %v", e)
  184. return
  185. }
  186. }
  187. }
  188. // 上海钢联
  189. mysteelOb := new(dataSourceModel.BaseFromMysteelChemicalIndex)
  190. {
  191. list, e := mysteelOb.GetItemsByCondition(cond, pars, []string{}, "")
  192. if e != nil {
  193. err = fmt.Errorf("获取上海钢联失败, %v", e)
  194. return
  195. }
  196. for _, v := range list {
  197. indexItem := v.Format2SearchDataSource(v)
  198. fmt.Printf("写入中:上海钢联-%d\n", indexItem.PrimaryId)
  199. docId := fmt.Sprintf("%d-%d", indexItem.Source, indexItem.PrimaryId)
  200. if e := elastic.EsAddOrEditDataSourceIndex(utils.EsDataSourceIndexName, docId, indexItem); e != nil {
  201. err = fmt.Errorf("上海钢联-写入es失败, %v", e)
  202. return
  203. }
  204. }
  205. }
  206. // SMM、有色原始数据库
  207. smmOb := new(dataSourceModel.BaseFromSmmIndex)
  208. {
  209. list, e := smmOb.GetItemsByCondition(cond, pars, []string{}, "")
  210. if e != nil {
  211. err = fmt.Errorf("获取有色原始数据库失败, %v", e)
  212. return
  213. }
  214. for _, v := range list {
  215. indexItem := v.Format2SearchDataSource(v)
  216. fmt.Printf("写入中:有色原始数据库-%d\n", indexItem.PrimaryId)
  217. docId := fmt.Sprintf("%d-%d", indexItem.Source, indexItem.PrimaryId)
  218. if e := elastic.EsAddOrEditDataSourceIndex(utils.EsDataSourceIndexName, docId, indexItem); e != nil {
  219. err = fmt.Errorf("有色原始数据库-写入es失败, %v", e)
  220. return
  221. }
  222. }
  223. }
  224. // 百川盈孚
  225. baiinfoOb := new(dataSourceModel.BaseFromBaiinfoIndex)
  226. {
  227. list, e := baiinfoOb.GetItemsByCondition(cond, pars, []string{}, "")
  228. if e != nil {
  229. err = fmt.Errorf("获取百川盈孚失败, %v", e)
  230. return
  231. }
  232. for _, v := range list {
  233. indexItem := v.Format2SearchDataSource(v)
  234. fmt.Printf("写入中:百川盈孚-%d\n", indexItem.PrimaryId)
  235. docId := fmt.Sprintf("%d-%d", indexItem.Source, indexItem.PrimaryId)
  236. if e := elastic.EsAddOrEditDataSourceIndex(utils.EsDataSourceIndexName, docId, indexItem); e != nil {
  237. err = fmt.Errorf("百川盈孚-写入es失败, %v", e)
  238. return
  239. }
  240. }
  241. }
  242. // 红桃3
  243. sciOb := new(dataSourceModel.BaseFromSciIndex)
  244. {
  245. list, e := sciOb.GetItemsByCondition(cond, pars, []string{}, "")
  246. if e != nil {
  247. err = fmt.Errorf("获取红桃3失败, %v", e)
  248. return
  249. }
  250. for _, v := range list {
  251. indexItem := v.Format2SearchDataSource(v)
  252. fmt.Printf("写入中:红桃3-%d\n", indexItem.PrimaryId)
  253. docId := fmt.Sprintf("%d-%d", indexItem.Source, indexItem.PrimaryId)
  254. if e := elastic.EsAddOrEditDataSourceIndex(utils.EsDataSourceIndexName, docId, indexItem); e != nil {
  255. err = fmt.Errorf("红桃3-写入es失败, %v", e)
  256. return
  257. }
  258. }
  259. }
  260. // 中国煤炭市场网
  261. coalmineOb := new(dataSourceModel.BaseFromCoalmineMapping)
  262. {
  263. list, e := coalmineOb.GetItemsByCondition(cond, pars, []string{}, "")
  264. if e != nil {
  265. err = fmt.Errorf("获取中国煤炭市场网失败, %v", e)
  266. return
  267. }
  268. for _, v := range list {
  269. indexItem := v.Format2SearchDataSource(v)
  270. fmt.Printf("写入中:中国煤炭市场网-%d\n", indexItem.PrimaryId)
  271. docId := fmt.Sprintf("%d-%d", indexItem.Source, indexItem.PrimaryId)
  272. if e := elastic.EsAddOrEditDataSourceIndex(utils.EsDataSourceIndexName, docId, indexItem); e != nil {
  273. err = fmt.Errorf("中国煤炭市场网-写入es失败, %v", e)
  274. return
  275. }
  276. }
  277. }
  278. // EIA STEO报告
  279. eiaOb := new(dataSourceModel.BaseFromEiaSteoIndex)
  280. {
  281. list, e := eiaOb.GetItemsByCondition(cond, pars, []string{}, "")
  282. if e != nil {
  283. err = fmt.Errorf("获取EIA STEO报告失败, %v", e)
  284. return
  285. }
  286. for _, v := range list {
  287. indexItem := v.Format2SearchDataSource(v)
  288. fmt.Printf("写入中:EIA STEO报告-%d\n", indexItem.PrimaryId)
  289. docId := fmt.Sprintf("%d-%d", indexItem.Source, indexItem.PrimaryId)
  290. if e := elastic.EsAddOrEditDataSourceIndex(utils.EsDataSourceIndexName, docId, indexItem); e != nil {
  291. err = fmt.Errorf("EIA STEO报告-写入es失败, %v", e)
  292. return
  293. }
  294. }
  295. }
  296. // ICPI消费价格指数
  297. icpiOb := new(dataSourceModel.BaseFromIcpiIndex)
  298. {
  299. list, e := icpiOb.GetItemsByCondition(cond, pars, []string{}, "")
  300. if e != nil {
  301. err = fmt.Errorf("获取ICPI消费价格指数失败, %v", e)
  302. return
  303. }
  304. for _, v := range list {
  305. indexItem := v.Format2SearchDataSource(v)
  306. fmt.Printf("写入中:ICPI消费价格指数-%d\n", indexItem.PrimaryId)
  307. docId := fmt.Sprintf("%d-%d", indexItem.Source, indexItem.PrimaryId)
  308. if e := elastic.EsAddOrEditDataSourceIndex(utils.EsDataSourceIndexName, docId, indexItem); e != nil {
  309. err = fmt.Errorf("ICPI消费价格指数-写入es失败, %v", e)
  310. return
  311. }
  312. }
  313. }
  314. // 涌益咨询
  315. yongyiOb := new(dataSourceModel.BaseFromYongyiIndex)
  316. {
  317. list, e := yongyiOb.GetItemsByCondition(cond, pars, []string{}, "")
  318. if e != nil {
  319. err = fmt.Errorf("获取涌益咨询失败, %v", e)
  320. return
  321. }
  322. for _, v := range list {
  323. indexItem := v.Format2SearchDataSource(v)
  324. fmt.Printf("写入中:涌益咨询-%d\n", indexItem.PrimaryId)
  325. docId := fmt.Sprintf("%d-%d", indexItem.Source, indexItem.PrimaryId)
  326. if e := elastic.EsAddOrEditDataSourceIndex(utils.EsDataSourceIndexName, docId, indexItem); e != nil {
  327. err = fmt.Errorf("涌益咨询-写入es失败, %v", e)
  328. return
  329. }
  330. }
  331. }
  332. // 汾渭数据
  333. fenweiOb := new(dataSourceModel.BaseFromFenweiIndex)
  334. {
  335. list, e := fenweiOb.GetItemsByCondition(cond, pars, []string{}, "")
  336. if e != nil {
  337. err = fmt.Errorf("获取汾渭数据失败, %v", e)
  338. return
  339. }
  340. for _, v := range list {
  341. indexItem := v.Format2SearchDataSource(v)
  342. fmt.Printf("写入中:汾渭数据-%d\n", indexItem.PrimaryId)
  343. docId := fmt.Sprintf("%d-%d", indexItem.Source, indexItem.PrimaryId)
  344. if e := elastic.EsAddOrEditDataSourceIndex(utils.EsDataSourceIndexName, docId, indexItem); e != nil {
  345. err = fmt.Errorf("汾渭数据-写入es失败, %v", e)
  346. return
  347. }
  348. }
  349. }
  350. // 卓创数据
  351. sci99Ob := new(dataSourceModel.BaseFromSci99Index)
  352. {
  353. list, e := sci99Ob.GetItemsByCondition(cond, pars, []string{}, "")
  354. if e != nil {
  355. err = fmt.Errorf("获取卓创数据失败, %v", e)
  356. return
  357. }
  358. for _, v := range list {
  359. indexItem := v.Format2SearchDataSource(v)
  360. fmt.Printf("写入中:卓创数据-%d\n", indexItem.PrimaryId)
  361. docId := fmt.Sprintf("%d-%d", indexItem.Source, indexItem.PrimaryId)
  362. if e := elastic.EsAddOrEditDataSourceIndex(utils.EsDataSourceIndexName, docId, indexItem); e != nil {
  363. err = fmt.Errorf("卓创数据-写入es失败, %v", e)
  364. return
  365. }
  366. }
  367. }
  368. // 钢联原始指标库
  369. glOb := new(dataSourceModel.BaseFromGlIndex)
  370. {
  371. list, e := glOb.GetItemsByCondition(cond, pars, []string{}, "")
  372. if e != nil {
  373. err = fmt.Errorf("获取钢联原始指标库失败, %v", e)
  374. return
  375. }
  376. for _, v := range list {
  377. indexItem := v.Format2SearchDataSource(v)
  378. fmt.Printf("写入中:钢联原始指标库-%d\n", indexItem.PrimaryId)
  379. docId := fmt.Sprintf("%d-%d", indexItem.Source, indexItem.PrimaryId)
  380. if e := elastic.EsAddOrEditDataSourceIndex(utils.EsDataSourceIndexName, docId, indexItem); e != nil {
  381. err = fmt.Errorf("钢联原始指标库-写入es失败, %v", e)
  382. return
  383. }
  384. }
  385. }
  386. // 手工指标录入
  387. manualOb := new(dataSourceModel.BaseFromManualEdb)
  388. {
  389. list, e := manualOb.GetItemsByCondition(cond, pars, []string{}, "")
  390. if e != nil {
  391. err = fmt.Errorf("获取手工指标失败, %v", e)
  392. return
  393. }
  394. for _, v := range list {
  395. indexItem := v.Format2SearchDataSource(v)
  396. fmt.Printf("写入中:手工指标-%s\n", indexItem.IndexCode)
  397. docId := fmt.Sprintf("%d-%s", indexItem.Source, indexItem.IndexCode)
  398. if e := elastic.EsAddOrEditDataSourceIndex(utils.EsDataSourceIndexName, docId, indexItem); e != nil {
  399. err = fmt.Errorf("手工指标-写入es失败, %v", e)
  400. return
  401. }
  402. }
  403. }
  404. // Bloomberg
  405. bloombergOb := new(dataSourceModel.BaseFromBloombergIndex)
  406. {
  407. list, e := bloombergOb.GetItemsByCondition(cond, pars, []string{}, "")
  408. if e != nil {
  409. err = fmt.Errorf("获取Bloomberg失败, %v", e)
  410. return
  411. }
  412. for _, v := range list {
  413. indexItem := v.Format2SearchDataSource(v)
  414. fmt.Printf("写入中:Bloomberg-%d\n", indexItem.PrimaryId)
  415. docId := fmt.Sprintf("%d-%d", indexItem.Source, indexItem.PrimaryId)
  416. if e := elastic.EsAddOrEditDataSourceIndex(utils.EsDataSourceIndexName, docId, indexItem); e != nil {
  417. err = fmt.Errorf("Bloomberg-写入es失败, %v", e)
  418. return
  419. }
  420. }
  421. }
  422. // 煤炭江湖
  423. mtjhOb := new(dataSourceModel.BaseFromMtjhMapping)
  424. {
  425. list, e := mtjhOb.GetItemsByCondition(cond, pars, []string{}, "")
  426. if e != nil {
  427. err = fmt.Errorf("获取煤炭江湖失败, %v", e)
  428. return
  429. }
  430. for _, v := range list {
  431. indexItem := v.Format2SearchDataSource(v)
  432. fmt.Printf("写入中:煤炭江湖-%d\n", indexItem.PrimaryId)
  433. docId := fmt.Sprintf("%d-%d", indexItem.Source, indexItem.PrimaryId)
  434. if e := elastic.EsAddOrEditDataSourceIndex(utils.EsDataSourceIndexName, docId, indexItem); e != nil {
  435. err = fmt.Errorf("煤炭江湖-写入es失败, %v", e)
  436. return
  437. }
  438. }
  439. }
  440. // AI预测模型
  441. aiPredictOb := new(aiPredictModel.AiPredictModelIndex)
  442. {
  443. list, e := aiPredictOb.GetItemsByCondition(cond, pars, []string{}, "")
  444. if e != nil {
  445. err = fmt.Errorf("获取AI预测模型失败, %v", e)
  446. return
  447. }
  448. for _, v := range list {
  449. indexItem := new(dataSourceModel.SearchDataSource)
  450. indexItem.PrimaryId = v.AiPredictModelIndexId
  451. indexItem.IndexName = v.IndexName
  452. indexItem.IndexCode = v.IndexCode
  453. indexItem.ClassifyId = v.ClassifyId
  454. indexItem.Source = utils.DATA_SOURCE_AI_PREDICT_MODEL
  455. indexItem.SourceName = "AI预测模型"
  456. indexItem.CreateTime = utils.TimeTransferString(utils.FormatDateTime, v.CreateTime)
  457. indexItem.ModifyTime = utils.TimeTransferString(utils.FormatDateTime, v.ModifyTime)
  458. fmt.Printf("写入中:AI预测模型-%d\n", indexItem.PrimaryId)
  459. docId := fmt.Sprintf("%d-%d", indexItem.Source, indexItem.PrimaryId)
  460. if e := elastic.EsAddOrEditDataSourceIndex(utils.EsDataSourceIndexName, docId, indexItem); e != nil {
  461. err = fmt.Errorf("AI预测模型-写入es失败, %v", e)
  462. return
  463. }
  464. }
  465. }
  466. // 美国农业部
  467. usdaOb := new(dataSourceModel.BaseFromUsdaFasIndex)
  468. {
  469. list, e := usdaOb.GetItemsByCondition(cond, pars, []string{}, "")
  470. if e != nil {
  471. err = fmt.Errorf("获取美国农业部失败, %v", e)
  472. return
  473. }
  474. for _, v := range list {
  475. indexItem := v.Format2SearchDataSource(v)
  476. fmt.Printf("写入中:美国农业部-%d\n", indexItem.PrimaryId)
  477. docId := fmt.Sprintf("%d-%d", indexItem.Source, indexItem.PrimaryId)
  478. if e := elastic.EsAddOrEditDataSourceIndex(utils.EsDataSourceIndexName, docId, indexItem); e != nil {
  479. err = fmt.Errorf("美国农业部-写入es失败, %v", e)
  480. return
  481. }
  482. }
  483. }
  484. // 自有数据
  485. businessOb := new(dataSourceModel.BaseFromBusinessIndex)
  486. {
  487. list, e := businessOb.GetItemsByCondition(cond, pars, []string{}, "")
  488. if e != nil {
  489. err = fmt.Errorf("获取自有数据失败, %v", e)
  490. return
  491. }
  492. for _, v := range list {
  493. indexItem := v.Format2SearchDataSource(v)
  494. fmt.Printf("写入中:自有数据-%d\n", indexItem.PrimaryId)
  495. docId := fmt.Sprintf("%d-%d", indexItem.Source, indexItem.PrimaryId)
  496. if e := elastic.EsAddOrEditDataSourceIndex(utils.EsDataSourceIndexName, docId, indexItem); e != nil {
  497. err = fmt.Errorf("自有数据-写入es失败, %v", e)
  498. return
  499. }
  500. }
  501. }
  502. fmt.Println("修复完成")
  503. }