elastic.go 56 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551155215531554155515561557155815591560156115621563156415651566156715681569157015711572157315741575157615771578157915801581158215831584158515861587158815891590159115921593159415951596159715981599160016011602160316041605160616071608160916101611161216131614161516161617161816191620162116221623162416251626162716281629163016311632163316341635163616371638163916401641164216431644164516461647164816491650165116521653165416551656165716581659166016611662166316641665166616671668166916701671167216731674167516761677167816791680168116821683168416851686168716881689169016911692169316941695169616971698169917001701170217031704170517061707170817091710171117121713171417151716171717181719172017211722172317241725172617271728172917301731173217331734173517361737173817391740174117421743174417451746174717481749175017511752175317541755175617571758175917601761176217631764176517661767176817691770177117721773177417751776177717781779178017811782178317841785178617871788178917901791179217931794179517961797179817991800180118021803180418051806180718081809181018111812181318141815181618171818181918201821182218231824182518261827182818291830183118321833183418351836183718381839184018411842184318441845184618471848184918501851185218531854185518561857185818591860186118621863186418651866186718681869187018711872187318741875187618771878187918801881188218831884188518861887188818891890189118921893189418951896189718981899190019011902190319041905190619071908190919101911191219131914191519161917191819191920192119221923192419251926192719281929193019311932193319341935193619371938193919401941194219431944194519461947194819491950195119521953195419551956195719581959196019611962196319641965196619671968196919701971197219731974197519761977197819791980198119821983198419851986198719881989199019911992199319941995199619971998199920002001200220032004200520062007200820092010201120122013201420152016201720182019202020212022202320242025202620272028202920302031203220332034203520362037203820392040204120422043204420452046204720482049205020512052205320542055205620572058205920602061206220632064206520662067206820692070207120722073207420752076207720782079208020812082208320842085208620872088208920902091209220932094209520962097209820992100210121022103210421052106210721082109211021112112211321142115211621172118211921202121212221232124212521262127212821292130213121322133213421352136213721382139214021412142214321442145214621472148214921502151215221532154215521562157215821592160
  1. package elastic
  2. import (
  3. "context"
  4. "encoding/json"
  5. "errors"
  6. "eta/eta_api/models"
  7. "eta/eta_api/models/data_manage"
  8. "eta/eta_api/models/data_manage/excel"
  9. dataSourceModel "eta/eta_api/models/data_source"
  10. "eta/eta_api/utils"
  11. "fmt"
  12. "github.com/olivere/elastic/v7"
  13. "strconv"
  14. "strings"
  15. )
  16. // indexName:索引名称
  17. // mappingJson:表结构
  18. func EsCreateIndex(indexName, mappingJson string) (err error) {
  19. client := utils.EsClient
  20. //定义表结构
  21. exists, err := client.IndexExists(indexName).Do(context.Background()) //<5>
  22. if err != nil {
  23. return
  24. }
  25. if !exists {
  26. resp, err := client.CreateIndex(indexName).BodyJson(mappingJson).Do(context.Background())
  27. //BodyJson(bodyJson).Do(context.Background())
  28. if err != nil {
  29. fmt.Println("CreateIndex Err:" + err.Error())
  30. return err
  31. }
  32. fmt.Println(resp.Index, resp.ShardsAcknowledged, resp.Acknowledged)
  33. } else {
  34. fmt.Println(indexName + " 已存在")
  35. }
  36. return
  37. }
  38. // 删除数据
  39. func EsDeleteData(indexName, docId string) (err error) {
  40. client := utils.EsClient
  41. resp, err := client.Delete().Index(indexName).Id(docId).Do(context.Background())
  42. fmt.Println(resp)
  43. if err != nil {
  44. return
  45. }
  46. if resp.Status == 0 {
  47. fmt.Println("删除成功")
  48. } else {
  49. fmt.Println("AddData", resp.Status, resp.Result)
  50. }
  51. return
  52. }
  53. func MappingModify(indexName, mappingJson string) {
  54. client := utils.EsClient
  55. result, err := client.PutMapping().Index(indexName).BodyString(mappingJson).Do(context.Background())
  56. fmt.Println(err)
  57. fmt.Println(result)
  58. return
  59. }
  60. // EsAddOrEditEdbInfoData 新增/修改es中的指标数据
  61. func EsAddOrEditEdbInfoData(indexName, docId string, item *data_manage.EdbInfoList) (err error) {
  62. defer func() {
  63. if err != nil {
  64. fmt.Println("EsAddOrEditData Err:", err.Error())
  65. }
  66. }()
  67. client := utils.EsClient
  68. resp, err := client.Index().Index(indexName).Id(docId).BodyJson(item).Do(context.Background())
  69. if err != nil {
  70. fmt.Println("新增失败:", err.Error())
  71. return err
  72. }
  73. fmt.Println(resp)
  74. if resp.Status == 0 {
  75. fmt.Println("新增成功", resp.Result)
  76. err = nil
  77. } else {
  78. fmt.Println("AddData", resp.Status, resp.Result)
  79. }
  80. return
  81. }
  82. // SearchEdbInfoData 查询es中的指标数据
  83. func SearchEdbInfoData(indexName, keywordStr string, from, size, filterSource, source int, edbInfoType int8, frequency string, noPermissionEdbInfoIdList []int) (total int64, list []*data_manage.EdbInfoList, err error) {
  84. list = make([]*data_manage.EdbInfoList, 0)
  85. defer func() {
  86. if err != nil {
  87. fmt.Println("EsAddOrEditData Err:", err.Error())
  88. }
  89. }()
  90. highlight := elastic.NewHighlight()
  91. highlight = highlight.Fields(elastic.NewHighlighterField("EdbCode"), elastic.NewHighlighterField("EdbName"))
  92. highlight = highlight.PreTags("<font color='red'>").PostTags("</font>")
  93. //var source map[string]interface{}
  94. //source := map[string]interface{}{
  95. // "query": map[string]interface{}{
  96. // "match_all": map[string]interface{}{},
  97. // },
  98. //}
  99. mustMap := make([]interface{}, 0)
  100. mustNotMap := make([]interface{}, 0)
  101. //source := map[string]interface{}{
  102. // "query": map[string]interface{}{
  103. // "bool": map[string]interface{}{
  104. // "must": map[string]interface{}{
  105. // "query_string": map[string]interface{}{
  106. // "query": keywordStr,
  107. // "fields": []string{"EdbCode", "EdbName"},
  108. // },
  109. // },
  110. // },
  111. // },
  112. //}
  113. switch filterSource {
  114. case 2:
  115. //source = map[string]interface{}{
  116. // "query": map[string]interface{}{
  117. // "bool": map[string]interface{}{
  118. // "must": map[string]interface{}{
  119. // "query_string": map[string]interface{}{
  120. // "query": keywordStr,
  121. // },
  122. // },
  123. // "filter": []interface{}{
  124. // map[string]interface{}{
  125. // "term": map[string]interface{}{
  126. // "Frequency.keyword": "月度",
  127. // },
  128. // }},
  129. // },
  130. // },
  131. //}
  132. mustMap = []interface{}{
  133. map[string]interface{}{
  134. "term": map[string]interface{}{
  135. "Frequency.keyword": "月度",
  136. //"Frequency.keyword": "月度",
  137. },
  138. },
  139. }
  140. case 3:
  141. //source = map[string]interface{}{
  142. // "query": map[string]interface{}{
  143. // "bool": map[string]interface{}{
  144. // "must": map[string]interface{}{
  145. // "query_string": map[string]interface{}{
  146. // "query": keywordStr,
  147. // },
  148. // },
  149. // "must_not": []interface{}{
  150. // map[string]interface{}{
  151. // "match": map[string]interface{}{
  152. // "Frequency.keyword": "日度",
  153. // },
  154. // }},
  155. // },
  156. // },
  157. //}
  158. ////注释掉,所有频度都可以变频 2022-08-31 14:31:28
  159. //mustNotMap = []interface{}{
  160. // map[string]interface{}{
  161. // "match": map[string]interface{}{
  162. // "Frequency.keyword": "日度",
  163. // //"Frequency.keyword": "月度",
  164. // },
  165. // },
  166. //}
  167. case 4:
  168. //source = map[string]interface{}{
  169. // "query": map[string]interface{}{
  170. // "bool": map[string]interface{}{
  171. // "must": map[string]interface{}{
  172. // "query_string": map[string]interface{}{
  173. // "query": keywordStr,
  174. // },
  175. // },
  176. // "filter": []interface{}{
  177. // map[string]interface{}{
  178. // "term": map[string]interface{}{
  179. // "EdbType": 1,
  180. // },
  181. // }},
  182. // },
  183. // },
  184. //}
  185. mustMap = []interface{}{
  186. map[string]interface{}{
  187. "term": map[string]interface{}{
  188. "EdbType": 1,
  189. },
  190. },
  191. }
  192. case 5:
  193. mustMap = []interface{}{
  194. map[string]interface{}{
  195. "term": map[string]interface{}{
  196. "Source": 6,
  197. },
  198. },
  199. }
  200. case 6:
  201. mustNotMap = []interface{}{
  202. map[string]interface{}{
  203. "match": map[string]interface{}{
  204. "Frequency.keyword": "年度",
  205. },
  206. },
  207. }
  208. }
  209. //指标来源
  210. if source > 0 {
  211. mustMap = append(mustMap, map[string]interface{}{
  212. "term": map[string]interface{}{
  213. "Source": source,
  214. //"Frequency.keyword": "月度",
  215. },
  216. })
  217. }
  218. if frequency != "" {
  219. mustMap = append(mustMap, map[string]interface{}{
  220. "term": map[string]interface{}{
  221. "Frequency.keyword": frequency,
  222. //"Frequency.keyword": "月度",
  223. },
  224. })
  225. }
  226. // noPermissionEdbInfoIdList 无权限指标id
  227. if len(noPermissionEdbInfoIdList) > 0 {
  228. mustNotMap = append(mustNotMap, map[string]interface{}{
  229. "terms": map[string]interface{}{
  230. "EdbInfoId": noPermissionEdbInfoIdList,
  231. //"Frequency.keyword": "月度",
  232. },
  233. })
  234. }
  235. // 指标类型:普通指标、预测指标(小于0 代表不区分指标是普通还是预测)
  236. if edbInfoType >= 0 {
  237. mustMap = append(mustMap, map[string]interface{}{
  238. "term": map[string]interface{}{
  239. "EdbInfoType": edbInfoType,
  240. },
  241. })
  242. }
  243. //普通指标
  244. //mustMap = append(mustMap, map[string]interface{}{
  245. // "term": map[string]interface{}{
  246. // "EdbInfoType": 0,
  247. // //"Frequency.keyword": "月度",
  248. // },
  249. //})
  250. //关键字匹配
  251. //shouldMap := map[string]interface{}{
  252. // "should": []interface{}{
  253. // map[string]interface{}{
  254. // "match": map[string]interface{}{
  255. // "EdbCode": keywordStr,
  256. // //"Frequency.keyword": "月度",
  257. // },
  258. // },
  259. // map[string]interface{}{
  260. // "match": map[string]interface{}{
  261. // "EdbName": keywordStr,
  262. // //"Frequency.keyword": "月度",
  263. // },
  264. // },
  265. // map[string]interface{}{
  266. // "match": map[string]interface{}{
  267. // "EdbNameEn": keywordStr,
  268. // //"Frequency.keyword": "月度",
  269. // },
  270. // },
  271. // },
  272. //}
  273. // 默认使用中文名字字段去匹配
  274. keywordNameKey := `EdbName`
  275. // 如果没有中文,则使用英文名称字段去匹配
  276. if !utils.ContainsChinese(keywordStr) {
  277. keywordNameKey = `EdbNameEn`
  278. }
  279. shouldMap := map[string]interface{}{
  280. "should": []interface{}{
  281. map[string]interface{}{
  282. "match": map[string]interface{}{
  283. "EdbCode": keywordStr,
  284. //"Frequency.keyword": "月度",
  285. },
  286. },
  287. map[string]interface{}{
  288. "match": map[string]interface{}{
  289. keywordNameKey: keywordStr,
  290. //"Frequency.keyword": "月度",
  291. },
  292. },
  293. },
  294. }
  295. //mustMap = append(mustMap, map[string]interface{}{
  296. // "bool": shouldMap,
  297. //})
  298. return searchEdbInfoData(indexName, mustMap, mustNotMap, shouldMap, from, size)
  299. }
  300. // SearchEdbInfoData 查询es中的指标数据
  301. func SearchEdbInfoDataByfrequency(indexName, keywordStr string, from, size int, edbInfoType int8, frequency []string, noPermissionEdbInfoIdList []int) (total int64, list []*data_manage.EdbInfoList, err error) {
  302. list = make([]*data_manage.EdbInfoList, 0)
  303. defer func() {
  304. if err != nil {
  305. fmt.Println("EsAddOrEditData Err:", err.Error())
  306. }
  307. }()
  308. mustMap := make([]interface{}, 0)
  309. mustNotMap := make([]interface{}, 0)
  310. if len(frequency) != 0 {
  311. mustMap = append(mustMap, map[string]interface{}{
  312. "terms": map[string]interface{}{
  313. "Frequency.keyword": frequency,
  314. //"Frequency.keyword": "月度",
  315. },
  316. })
  317. }
  318. // noPermissionEdbInfoIdList 无权限指标id
  319. if len(noPermissionEdbInfoIdList) > 0 {
  320. mustNotMap = append(mustNotMap, map[string]interface{}{
  321. "terms": map[string]interface{}{
  322. "EdbInfoId": noPermissionEdbInfoIdList,
  323. //"Frequency.keyword": "月度",
  324. },
  325. })
  326. }
  327. // 指标类型:普通指标、预测指标(小于0 代表不区分指标是普通还是预测)
  328. if edbInfoType >= 0 {
  329. mustMap = append(mustMap, map[string]interface{}{
  330. "term": map[string]interface{}{
  331. "EdbInfoType": edbInfoType,
  332. },
  333. })
  334. }
  335. // 默认使用中文名字字段去匹配
  336. keywordNameKey := `EdbName`
  337. // 如果没有中文,则使用英文名称字段去匹配
  338. if !utils.ContainsChinese(keywordStr) {
  339. keywordNameKey = `EdbNameEn`
  340. }
  341. shouldMap := map[string]interface{}{
  342. "should": []interface{}{
  343. map[string]interface{}{
  344. "match": map[string]interface{}{
  345. "EdbCode": keywordStr,
  346. //"Frequency.keyword": "月度",
  347. },
  348. },
  349. map[string]interface{}{
  350. "match": map[string]interface{}{
  351. keywordNameKey: keywordStr,
  352. //"Frequency.keyword": "月度",
  353. },
  354. },
  355. },
  356. }
  357. return searchEdbInfoData(indexName, mustMap, mustNotMap, shouldMap, from, size)
  358. }
  359. func SearchEdbInfoDataBak(indexName, keywordStr string, from, size, filterSource, source int, frequency string) (total int64, list []*data_manage.EdbInfoList, err error) {
  360. list = make([]*data_manage.EdbInfoList, 0)
  361. defer func() {
  362. if err != nil {
  363. fmt.Println("EsAddOrEditData Err:", err.Error())
  364. }
  365. }()
  366. client := utils.EsClient
  367. //queryString := elastic.NewQueryStringQuery(keywordStr)
  368. //boolQueryJson, err := json.Marshal(queryString)
  369. //if err != nil {
  370. // fmt.Println("boolQueryJson err:", err)
  371. //} else {
  372. // fmt.Println("boolQueryJson ", string(boolQueryJson))
  373. //}
  374. highlight := elastic.NewHighlight()
  375. highlight = highlight.Fields(elastic.NewHighlighterField("EdbCode"), elastic.NewHighlighterField("EdbName"))
  376. highlight = highlight.PreTags("<font color='red'>").PostTags("</font>")
  377. //query := elastic.RawStringQuery(`{"match_all":{}}`)
  378. //var source map[string]interface{}
  379. //source := map[string]interface{}{
  380. // "query": map[string]interface{}{
  381. // "match_all": map[string]interface{}{},
  382. // },
  383. //}
  384. mustMap := make([]interface{}, 0)
  385. mustNotMap := make([]interface{}, 0)
  386. //source := map[string]interface{}{
  387. // "query": map[string]interface{}{
  388. // "bool": map[string]interface{}{
  389. // "must": map[string]interface{}{
  390. // "query_string": map[string]interface{}{
  391. // "query": keywordStr,
  392. // "fields": []string{"EdbCode", "EdbName"},
  393. // },
  394. // },
  395. // },
  396. // },
  397. //}
  398. switch filterSource {
  399. case 2:
  400. //source = map[string]interface{}{
  401. // "query": map[string]interface{}{
  402. // "bool": map[string]interface{}{
  403. // "must": map[string]interface{}{
  404. // "query_string": map[string]interface{}{
  405. // "query": keywordStr,
  406. // },
  407. // },
  408. // "filter": []interface{}{
  409. // map[string]interface{}{
  410. // "term": map[string]interface{}{
  411. // "Frequency.keyword": "月度",
  412. // },
  413. // }},
  414. // },
  415. // },
  416. //}
  417. mustMap = []interface{}{
  418. map[string]interface{}{
  419. "term": map[string]interface{}{
  420. "Frequency.keyword": "月度",
  421. //"Frequency.keyword": "月度",
  422. },
  423. },
  424. }
  425. case 3:
  426. //source = map[string]interface{}{
  427. // "query": map[string]interface{}{
  428. // "bool": map[string]interface{}{
  429. // "must": map[string]interface{}{
  430. // "query_string": map[string]interface{}{
  431. // "query": keywordStr,
  432. // },
  433. // },
  434. // "must_not": []interface{}{
  435. // map[string]interface{}{
  436. // "match": map[string]interface{}{
  437. // "Frequency.keyword": "日度",
  438. // },
  439. // }},
  440. // },
  441. // },
  442. //}
  443. ////注释掉,所有频度都可以变频 2022-08-31 14:31:28
  444. //mustNotMap = []interface{}{
  445. // map[string]interface{}{
  446. // "match": map[string]interface{}{
  447. // "Frequency.keyword": "日度",
  448. // //"Frequency.keyword": "月度",
  449. // },
  450. // },
  451. //}
  452. case 4:
  453. //source = map[string]interface{}{
  454. // "query": map[string]interface{}{
  455. // "bool": map[string]interface{}{
  456. // "must": map[string]interface{}{
  457. // "query_string": map[string]interface{}{
  458. // "query": keywordStr,
  459. // },
  460. // },
  461. // "filter": []interface{}{
  462. // map[string]interface{}{
  463. // "term": map[string]interface{}{
  464. // "EdbType": 1,
  465. // },
  466. // }},
  467. // },
  468. // },
  469. //}
  470. mustMap = []interface{}{
  471. map[string]interface{}{
  472. "term": map[string]interface{}{
  473. "EdbType": 1,
  474. },
  475. },
  476. }
  477. case 5:
  478. mustMap = []interface{}{
  479. map[string]interface{}{
  480. "term": map[string]interface{}{
  481. "Source": 6,
  482. },
  483. },
  484. }
  485. case 6:
  486. mustNotMap = []interface{}{
  487. map[string]interface{}{
  488. "match": map[string]interface{}{
  489. "Frequency.keyword": "年度",
  490. },
  491. },
  492. }
  493. }
  494. //指标来源
  495. if source > 0 {
  496. mustMap = append(mustMap, map[string]interface{}{
  497. "term": map[string]interface{}{
  498. "Source": source,
  499. //"Frequency.keyword": "月度",
  500. },
  501. })
  502. }
  503. if frequency != "" {
  504. mustMap = append(mustMap, map[string]interface{}{
  505. "term": map[string]interface{}{
  506. "Frequency.keyword": frequency,
  507. //"Frequency.keyword": "月度",
  508. },
  509. })
  510. }
  511. //普通指标
  512. //mustMap = append(mustMap, map[string]interface{}{
  513. // "term": map[string]interface{}{
  514. // "EdbInfoType": 0,
  515. // //"Frequency.keyword": "月度",
  516. // },
  517. //})
  518. //关键字匹配
  519. shouldMap := map[string]interface{}{
  520. "should": []interface{}{
  521. map[string]interface{}{
  522. "match": map[string]interface{}{
  523. "EdbCode": keywordStr,
  524. //"Frequency.keyword": "月度",
  525. },
  526. },
  527. map[string]interface{}{
  528. "match": map[string]interface{}{
  529. "EdbName": keywordStr,
  530. //"Frequency.keyword": "月度",
  531. },
  532. },
  533. // 因为关键词被分了,所以需要用下面的语句来让他 整个词 查询,从而加重整词的权重
  534. map[string]interface{}{
  535. "match": map[string]interface{}{
  536. "EdbCode": map[string]interface{}{
  537. "query": keywordStr,
  538. "operator": "and",
  539. },
  540. //"Frequency.keyword": "月度",
  541. },
  542. },
  543. // 因为关键词被分了,所以需要用下面的语句来让他 整个词 查询,从而加重整词的权重
  544. map[string]interface{}{
  545. "match": map[string]interface{}{
  546. "EdbName": map[string]interface{}{
  547. "query": keywordStr,
  548. "operator": "and",
  549. },
  550. //"Frequency.keyword": "月度",
  551. },
  552. },
  553. },
  554. }
  555. mustMap = append(mustMap, map[string]interface{}{
  556. "bool": shouldMap,
  557. })
  558. queryMap := map[string]interface{}{
  559. "query": map[string]interface{}{
  560. "bool": map[string]interface{}{
  561. "must": mustMap,
  562. "must_not": mustNotMap,
  563. //"should": shouldMap,
  564. },
  565. },
  566. }
  567. //根据条件数量统计
  568. requestTotalHits := client.Count(indexName).BodyJson(queryMap)
  569. total, err = requestTotalHits.Do(context.Background())
  570. if err != nil {
  571. return
  572. }
  573. queryMap["from"] = from
  574. queryMap["size"] = size
  575. jsonBytes, _ := json.Marshal(queryMap)
  576. fmt.Println(string(jsonBytes))
  577. //queryStr := fmt.Sprintf(`{"query":{"bool":{"must":{"query_string":{"query":"%s","fields":["EdbCode","EdbName"]}}}}}`, keywordStr)
  578. //switch filterSource {
  579. //case 2:
  580. // queryStr = fmt.Sprintf(`{"query":{"bool":{"must":{"query_string":{"query":"%s","fields":["EdbCode","EdbName"]}},"filter":{"term":{"Frequency.keyword":"%s"}}}}}`, keywordStr, "月度")
  581. //case 3:
  582. // queryStr = fmt.Sprintf(`{"query":{"bool":{"must":{"query_string":{"query":"%s","fields":["EdbCode","EdbName"]}},"must_not":[{"match":{"Frequency.keyword":"%s"}}]}}}`, keywordStr, "日度")
  583. //case 4:
  584. // queryStr = fmt.Sprintf(`{"query":{"bool":{"must":{"query_string":{"query":"%s","fields":["EdbCode","EdbName"]}},"must_not":[{"match":{"EdbType":1}}]}}}`, keywordStr)
  585. //}
  586. //queryString := elastic.RawStringQuery(queryStr)
  587. //fmt.Println("queryString:", queryString)
  588. //queryString := elastic.NewMatchQuery("EdbCode", keywordStr)
  589. //request := client.Search(indexName).Highlight(highlight).From(from).Size(size).Query(queryString)
  590. request := client.Search(indexName).Highlight(highlight).Source(queryMap) // sets the JSON request
  591. //requestJson, err := json.Marshal(request)
  592. //if err != nil {
  593. // fmt.Println("requestJson err:", err)
  594. //}
  595. //fmt.Println("requestJson ", string(requestJson))
  596. searchMap := make(map[string]string)
  597. searchResp, err := request.Do(context.Background())
  598. if err != nil {
  599. return
  600. }
  601. fmt.Println(searchResp)
  602. fmt.Println(searchResp.Status)
  603. if searchResp.Status != 0 {
  604. return
  605. }
  606. if searchResp.Hits != nil {
  607. for _, v := range searchResp.Hits.Hits {
  608. if _, ok := searchMap[v.Id]; !ok {
  609. itemJson, tmpErr := v.Source.MarshalJSON()
  610. if tmpErr != nil {
  611. err = tmpErr
  612. fmt.Println("movieJson err:", err)
  613. return
  614. }
  615. edbInfoItem := new(data_manage.EdbInfoList)
  616. tmpErr = json.Unmarshal(itemJson, &edbInfoItem)
  617. if tmpErr != nil {
  618. fmt.Println("json.Unmarshal movieJson err:", tmpErr)
  619. err = tmpErr
  620. return
  621. }
  622. if len(v.Highlight["EdbCode"]) > 0 {
  623. edbInfoItem.EdbCode = v.Highlight["EdbCode"][0]
  624. }
  625. if len(v.Highlight["EdbName"]) > 0 {
  626. edbInfoItem.EdbCode = v.Highlight["EdbName"][0]
  627. }
  628. list = append(list, edbInfoItem)
  629. searchMap[v.Id] = v.Id
  630. }
  631. }
  632. }
  633. //for _, v := range result {
  634. // fmt.Println(v)
  635. //}
  636. return
  637. }
  638. // SearchAddPredictEdbInfoData 查询允许添加预测指标的数据
  639. func SearchAddPredictEdbInfoData(indexName, keywordStr string, noPermissionEdbInfoIdList []int, from, size int) (total int64, list []*data_manage.EdbInfoList, err error) {
  640. list = make([]*data_manage.EdbInfoList, 0)
  641. defer func() {
  642. if err != nil {
  643. fmt.Println("EsAddOrEditData Err:", err.Error())
  644. }
  645. }()
  646. highlight := elastic.NewHighlight()
  647. highlight = highlight.Fields(elastic.NewHighlighterField("EdbCode"), elastic.NewHighlighterField("EdbName"))
  648. highlight = highlight.PreTags("<font color='red'>").PostTags("</font>")
  649. mustMap := make([]interface{}, 0)
  650. mustNotMap := make([]interface{}, 0)
  651. mustNotMap = []interface{}{
  652. //map[string]interface{}{
  653. // "terms": map[string]interface{}{
  654. // "Frequency.keyword": []string{"日度", "周度", "月度"},
  655. // },
  656. // //"match": map[string]interface{}{
  657. // // "Frequency": []string{"日度", "周度", "月度"},
  658. // // //"Frequency.keyword": []string{"日度", "周度", "月度"},
  659. // //},
  660. //},
  661. }
  662. // 指标类型:普通指标、预算指标
  663. mustMap = append(mustMap, map[string]interface{}{
  664. "term": map[string]interface{}{
  665. "EdbInfoType": 0,
  666. },
  667. })
  668. mustMap = append(mustMap, map[string]interface{}{
  669. "terms": map[string]interface{}{
  670. "Frequency.keyword": []string{"日度", "周度", "月度"},
  671. },
  672. })
  673. //关键字匹配
  674. //shouldMap := map[string]interface{}{
  675. // "should": []interface{}{
  676. // map[string]interface{}{
  677. // "match": map[string]interface{}{
  678. // "EdbCode": keywordStr,
  679. // //"Frequency.keyword": "月度",
  680. // },
  681. // },
  682. // map[string]interface{}{
  683. // "match": map[string]interface{}{
  684. // "EdbName": keywordStr,
  685. // //"Frequency.keyword": "月度",
  686. // },
  687. // },
  688. // map[string]interface{}{
  689. // "match": map[string]interface{}{
  690. // "EdbNameEn": keywordStr,
  691. // //"Frequency.keyword": "月度",
  692. // },
  693. // },
  694. // },
  695. //}
  696. // 默认使用中文名字字段去匹配
  697. keywordNameKey := `EdbName`
  698. // 如果没有中文,则使用英文名称字段去匹配
  699. if !utils.ContainsChinese(keywordStr) {
  700. keywordNameKey = `EdbNameEn`
  701. }
  702. shouldMap := map[string]interface{}{
  703. "should": []interface{}{
  704. map[string]interface{}{
  705. "match": map[string]interface{}{
  706. "EdbCode": keywordStr,
  707. //"Frequency.keyword": "月度",
  708. },
  709. },
  710. map[string]interface{}{
  711. "match": map[string]interface{}{
  712. keywordNameKey: keywordStr,
  713. //"Frequency.keyword": "月度",
  714. },
  715. },
  716. },
  717. }
  718. // noPermissionEdbInfoIdList 无权限指标id
  719. if len(noPermissionEdbInfoIdList) > 0 {
  720. mustNotMap = append(mustNotMap, map[string]interface{}{
  721. "terms": map[string]interface{}{
  722. "EdbInfoId": noPermissionEdbInfoIdList,
  723. //"Frequency.keyword": "月度",
  724. },
  725. })
  726. }
  727. return searchEdbInfoData(indexName, mustMap, mustNotMap, shouldMap, from, size)
  728. }
  729. // searchEdbInfoData 查询es中的指标数据
  730. func searchEdbInfoData(indexName string, mustMap, mustNotMap []interface{}, shouldMap map[string]interface{}, from, size int) (total int64, list []*data_manage.EdbInfoList, err error) {
  731. list = make([]*data_manage.EdbInfoList, 0)
  732. defer func() {
  733. if err != nil {
  734. fmt.Println("EsAddOrEditData Err:", err.Error())
  735. }
  736. }()
  737. client := utils.EsClient
  738. //queryString := elastic.NewQueryStringQuery(keywordStr)
  739. //boolQueryJson, err := json.Marshal(queryString)
  740. //if err != nil {
  741. // fmt.Println("boolQueryJson err:", err)
  742. //} else {
  743. // fmt.Println("boolQueryJson ", string(boolQueryJson))
  744. //}
  745. highlight := elastic.NewHighlight()
  746. highlight = highlight.Fields(elastic.NewHighlighterField("EdbCode"), elastic.NewHighlighterField("EdbName"))
  747. highlight = highlight.PreTags("<font color='red'>").PostTags("</font>")
  748. //query := elastic.RawStringQuery(`{"match_all":{}}`)
  749. //关键字匹配
  750. mustMap = append(mustMap, map[string]interface{}{
  751. "bool": shouldMap,
  752. })
  753. queryMap := map[string]interface{}{
  754. "query": map[string]interface{}{
  755. "bool": map[string]interface{}{
  756. "must": mustMap,
  757. "must_not": mustNotMap,
  758. //"should": shouldMap,
  759. },
  760. },
  761. }
  762. //根据条件数量统计
  763. requestTotalHits := client.Count(indexName).BodyJson(queryMap)
  764. total, err = requestTotalHits.Do(context.Background())
  765. if err != nil {
  766. return
  767. }
  768. queryMap["from"] = from
  769. queryMap["size"] = size
  770. queryMap["highlight"] = map[string]interface{}{
  771. "fields": map[string]interface{}{
  772. "EdbName": map[string]interface{}{},
  773. },
  774. "pre_tags": "<span style=\"color:#0052D9\">",
  775. "post_tags": "</span>",
  776. }
  777. jsonBytes, _ := json.Marshal(queryMap)
  778. fmt.Println(string(jsonBytes))
  779. //queryStr := fmt.Sprintf(`{"query":{"bool":{"must":{"query_string":{"query":"%s","fields":["EdbCode","EdbName"]}}}}}`, keywordStr)
  780. //switch filterSource {
  781. //case 2:
  782. // queryStr = fmt.Sprintf(`{"query":{"bool":{"must":{"query_string":{"query":"%s","fields":["EdbCode","EdbName"]}},"filter":{"term":{"Frequency.keyword":"%s"}}}}}`, keywordStr, "月度")
  783. //case 3:
  784. // queryStr = fmt.Sprintf(`{"query":{"bool":{"must":{"query_string":{"query":"%s","fields":["EdbCode","EdbName"]}},"must_not":[{"match":{"Frequency.keyword":"%s"}}]}}}`, keywordStr, "日度")
  785. //case 4:
  786. // queryStr = fmt.Sprintf(`{"query":{"bool":{"must":{"query_string":{"query":"%s","fields":["EdbCode","EdbName"]}},"must_not":[{"match":{"EdbType":1}}]}}}`, keywordStr)
  787. //}
  788. //queryString := elastic.RawStringQuery(queryStr)
  789. //fmt.Println("queryString:", queryString)
  790. //queryString := elastic.NewMatchQuery("EdbCode", keywordStr)
  791. //request := client.Search(indexName).Highlight(highlight).From(from).Size(size).Query(queryString)
  792. request := client.Search(indexName).Highlight(highlight).Source(queryMap) // sets the JSON request
  793. //requestJson, err := json.Marshal(request)
  794. //if err != nil {
  795. // fmt.Println("requestJson err:", err)
  796. //}
  797. //fmt.Println("requestJson ", string(requestJson))
  798. searchMap := make(map[string]string)
  799. searchResp, err := request.Do(context.Background())
  800. if err != nil {
  801. return
  802. }
  803. fmt.Println(searchResp)
  804. fmt.Println(searchResp.Status)
  805. if searchResp.Status != 0 {
  806. return
  807. }
  808. //total = searchResp.TotalHits()
  809. if searchResp.Hits != nil {
  810. for _, v := range searchResp.Hits.Hits {
  811. if _, ok := searchMap[v.Id]; !ok {
  812. itemJson, tmpErr := v.Source.MarshalJSON()
  813. if tmpErr != nil {
  814. err = tmpErr
  815. fmt.Println("movieJson err:", err)
  816. return
  817. }
  818. edbInfoItem := new(data_manage.EdbInfoList)
  819. tmpErr = json.Unmarshal(itemJson, &edbInfoItem)
  820. if tmpErr != nil {
  821. fmt.Println("json.Unmarshal movieJson err:", tmpErr)
  822. err = tmpErr
  823. return
  824. }
  825. if len(v.Highlight["EdbCode"]) > 0 {
  826. edbInfoItem.EdbCode = v.Highlight["EdbCode"][0]
  827. }
  828. if len(v.Highlight["EdbName"]) > 0 {
  829. // 搜索结果高亮用新字段,原EdbName直接高亮展示上会有点影响
  830. edbInfoItem.SearchText = v.Highlight["EdbName"][0]
  831. //edbInfoItem.EdbName = v.Highlight["EdbName"][0]
  832. } else {
  833. edbInfoItem.SearchText = edbInfoItem.EdbName
  834. }
  835. list = append(list, edbInfoItem)
  836. searchMap[v.Id] = v.Id
  837. }
  838. }
  839. }
  840. return
  841. }
  842. // EsDeleteEdbInfoData 删除es中的指标数据
  843. func EsDeleteEdbInfoData(indexName, docId string) (err error) {
  844. defer func() {
  845. if err != nil {
  846. fmt.Println("EsDeleteEdbInfoData Err:", err.Error())
  847. }
  848. }()
  849. client := utils.EsClient
  850. resp, err := client.Delete().Index(indexName).Id(docId).Do(context.Background())
  851. fmt.Println(resp)
  852. if err != nil {
  853. return
  854. }
  855. if resp.Status == 0 {
  856. fmt.Println("删除成功")
  857. } else {
  858. fmt.Println("AddData", resp.Status, resp.Result)
  859. }
  860. return
  861. }
  862. // EsAddOrEditReport 新增编辑es报告
  863. func EsAddOrEditReport(indexName, docId string, item *models.ElasticReportDetail) (err error) {
  864. defer func() {
  865. if err != nil {
  866. fmt.Println("EsAddOrEditReport Err:", err.Error())
  867. }
  868. }()
  869. client := utils.EsClient
  870. // docId为报告ID+章节ID
  871. searchById, err := client.Get().Index(indexName).Id(docId).Do(context.Background())
  872. if err != nil && !strings.Contains(err.Error(), "404") {
  873. fmt.Println("Get Err" + err.Error())
  874. return
  875. }
  876. if searchById != nil && searchById.Found {
  877. resp, err := client.Update().Index(indexName).Id(docId).Doc(map[string]interface{}{
  878. "ReportId": item.ReportId,
  879. "ReportChapterId": item.ReportChapterId,
  880. "Title": item.Title,
  881. "Abstract": item.Abstract,
  882. "BodyContent": item.BodyContent,
  883. "PublishTime": item.PublishTime,
  884. "PublishState": item.PublishState,
  885. "Author": item.Author,
  886. "ClassifyIdFirst": item.ClassifyIdFirst,
  887. "ClassifyNameFirst": item.ClassifyNameFirst,
  888. "ClassifyIdSecond": item.ClassifyIdSecond,
  889. "ClassifyNameSecond": item.ClassifyNameSecond,
  890. "Categories": item.Categories,
  891. "StageStr": item.StageStr,
  892. }).Do(context.Background())
  893. if err != nil {
  894. return err
  895. }
  896. //fmt.Println(resp.Status, resp.Result)
  897. if resp.Status == 0 {
  898. fmt.Println("修改成功" + docId)
  899. err = nil
  900. } else {
  901. fmt.Println("EditData", resp.Status, resp.Result)
  902. }
  903. } else {
  904. resp, err := client.Index().Index(indexName).Id(docId).BodyJson(item).Do(context.Background())
  905. if err != nil {
  906. fmt.Println("新增失败:", err.Error())
  907. return err
  908. }
  909. if resp.Status == 0 && resp.Result == "created" {
  910. fmt.Println("新增成功" + docId)
  911. return nil
  912. } else {
  913. fmt.Println("AddData", resp.Status, resp.Result)
  914. }
  915. }
  916. return
  917. }
  918. // AnalyzeResp 分词接口返回结构体
  919. type AnalyzeResp struct {
  920. Tokens []struct {
  921. EndOffset int64 `json:"end_offset"`
  922. Position int64 `json:"position"`
  923. StartOffset int64 `json:"start_offset"`
  924. Token string `json:"token"`
  925. Type string `json:"type"`
  926. } `json:"tokens"`
  927. }
  928. // Analyze 根据输入的文字获取分词后的文字
  929. func Analyze(content string) (contentList []string, err error) {
  930. defer func() {
  931. if err != nil {
  932. fmt.Println("Analyze Err:", err.Error())
  933. }
  934. }()
  935. client := utils.EsClient
  936. queryMap := map[string]string{
  937. "text": content,
  938. "analyzer": "ik_max_word",
  939. }
  940. res, err := client.PerformRequest(
  941. context.Background(),
  942. elastic.PerformRequestOptions{
  943. Method: "GET",
  944. Path: "/_analyze",
  945. Body: queryMap,
  946. Stream: false,
  947. },
  948. )
  949. if res.StatusCode == 200 {
  950. var analyzeResp AnalyzeResp
  951. tmpErr := json.Unmarshal(res.Body, &analyzeResp)
  952. if tmpErr != nil {
  953. err = errors.New("返回数据转结构体失败:" + tmpErr.Error())
  954. return
  955. }
  956. for _, v := range analyzeResp.Tokens {
  957. contentList = append(contentList, v.Token)
  958. }
  959. } else {
  960. err = errors.New("分词失败,返回code异常:" + strconv.Itoa(res.StatusCode))
  961. }
  962. return
  963. }
  964. // EsAddOrEditChartInfoData 新增/修改es中的图表数据
  965. func EsAddOrEditChartInfoData(indexName, docId string, item *data_manage.ChartInfo) (err error) {
  966. defer func() {
  967. if err != nil {
  968. fmt.Println("EsAddOrEditData Err:", err.Error())
  969. }
  970. }()
  971. client := utils.EsClient
  972. resp, err := client.Index().Index(indexName).Id(docId).BodyJson(item).Do(context.Background())
  973. if err != nil {
  974. fmt.Println("新增失败:", err.Error())
  975. return err
  976. }
  977. fmt.Println(resp)
  978. if resp.Status == 0 {
  979. fmt.Println("新增成功", resp.Result)
  980. err = nil
  981. } else {
  982. fmt.Println("AddData", resp.Status, resp.Result)
  983. }
  984. return
  985. }
  986. // EsDeleteDataV2 删除es中的数据
  987. func EsDeleteDataV2(indexName, docId string) (err error) {
  988. defer func() {
  989. if err != nil {
  990. fmt.Println("EsDeleteEdbInfoData Err:", err.Error())
  991. }
  992. }()
  993. client := utils.EsClient
  994. resp, err := client.Delete().Index(indexName).Id(docId).Do(context.Background())
  995. fmt.Println(resp)
  996. if err != nil {
  997. return
  998. }
  999. if resp.Status == 0 {
  1000. fmt.Println("删除成功")
  1001. } else {
  1002. fmt.Println("AddData", resp.Status, resp.Result)
  1003. }
  1004. return
  1005. }
  1006. // SearchChartInfoData 查询es中的图表数据
  1007. func SearchChartInfoData(indexName, keywordStr string, showSysId int, sourceList []int, noPermissionChartIdList []int, from, size int) (list []*data_manage.ChartInfoMore, total int64, err error) {
  1008. list = make([]*data_manage.ChartInfoMore, 0)
  1009. defer func() {
  1010. if err != nil {
  1011. fmt.Println("EsAddOrEditData Err:", err.Error())
  1012. }
  1013. }()
  1014. client := utils.EsClient
  1015. //queryString := elastic.NewQueryStringQuery(keywordStr)
  1016. //boolQueryJson, err := json.Marshal(queryString)
  1017. //if err != nil {
  1018. // fmt.Println("boolQueryJson err:", err)
  1019. //} else {
  1020. // fmt.Println("boolQueryJson ", string(boolQueryJson))
  1021. //}
  1022. highlight := elastic.NewHighlight()
  1023. highlight = highlight.Fields(elastic.NewHighlighterField("ChartName"))
  1024. highlight = highlight.PreTags("<font color='red'>").PostTags("</font>")
  1025. mustMap := make([]interface{}, 0)
  1026. mustNotMap := make([]interface{}, 0)
  1027. //指标来源
  1028. if showSysId > 0 {
  1029. mustMap = append(mustMap, map[string]interface{}{
  1030. "term": map[string]interface{}{
  1031. "SysUserId": showSysId,
  1032. //"Frequency.keyword": "月度",
  1033. },
  1034. })
  1035. }
  1036. mustMap = append(mustMap, map[string]interface{}{
  1037. "terms": map[string]interface{}{
  1038. "Source": sourceList,
  1039. },
  1040. })
  1041. //关键字匹配
  1042. //shouldMap := map[string]interface{}{
  1043. // "should": []interface{}{
  1044. // map[string]interface{}{
  1045. // "match": map[string]interface{}{
  1046. // "ChartName": keywordStr,
  1047. // //"Frequency.keyword": "月度",
  1048. // },
  1049. // },
  1050. // // 因为关键词被分了,所以需要用下面的语句来让他 整个词 查询,从而加重整词的权重
  1051. // map[string]interface{}{
  1052. // "match": map[string]interface{}{
  1053. // "ChartName": map[string]interface{}{
  1054. // "query": keywordStr,
  1055. // "operator": "and",
  1056. // },
  1057. // //"Frequency.keyword": "月度",
  1058. // },
  1059. // },
  1060. // map[string]interface{}{
  1061. // "match": map[string]interface{}{
  1062. // "ChartNameEn": keywordStr,
  1063. // //"Frequency.keyword": "月度",
  1064. // },
  1065. // },
  1066. // // 因为关键词被分了,所以需要用下面的语句来让他 整个词 查询,从而加重整词的权重
  1067. // map[string]interface{}{
  1068. // "match": map[string]interface{}{
  1069. // "ChartNameEn": map[string]interface{}{
  1070. // "query": keywordStr,
  1071. // "operator": "and",
  1072. // },
  1073. // //"Frequency.keyword": "月度",
  1074. // },
  1075. // },
  1076. // },
  1077. //}
  1078. // 默认使用中文名字字段去匹配
  1079. keywordNameKey := `ChartName`
  1080. // 如果没有中文,则使用英文名称字段去匹配
  1081. if !utils.ContainsChinese(keywordStr) {
  1082. keywordNameKey = `ChartNameEn`
  1083. }
  1084. shouldMap := map[string]interface{}{
  1085. "should": []interface{}{
  1086. map[string]interface{}{
  1087. "match": map[string]interface{}{
  1088. keywordNameKey: keywordStr,
  1089. //"Frequency.keyword": "月度",
  1090. },
  1091. },
  1092. // 因为关键词被分了,所以需要用下面的语句来让他 整个词 查询,从而加重整词的权重
  1093. map[string]interface{}{
  1094. "match": map[string]interface{}{
  1095. keywordNameKey: map[string]interface{}{
  1096. "query": keywordStr,
  1097. "operator": "and",
  1098. },
  1099. //"Frequency.keyword": "月度",
  1100. },
  1101. },
  1102. },
  1103. }
  1104. mustMap = append(mustMap, map[string]interface{}{
  1105. "bool": shouldMap,
  1106. })
  1107. // noPermissionEdbInfoIdList 无权限指标id
  1108. if len(noPermissionChartIdList) > 0 {
  1109. mustNotMap = append(mustNotMap, map[string]interface{}{
  1110. "terms": map[string]interface{}{
  1111. "ChartInfoId": noPermissionChartIdList,
  1112. //"Frequency.keyword": "月度",
  1113. },
  1114. })
  1115. }
  1116. queryMap := map[string]interface{}{
  1117. "query": map[string]interface{}{
  1118. "bool": map[string]interface{}{
  1119. "must": mustMap,
  1120. "must_not": mustNotMap,
  1121. //"should": shouldMap,
  1122. },
  1123. },
  1124. }
  1125. //根据条件数量统计
  1126. requestTotalHits := client.Count(indexName).BodyJson(queryMap)
  1127. total, err = requestTotalHits.Do(context.Background())
  1128. if err != nil {
  1129. return
  1130. }
  1131. // 分页查询
  1132. queryMap["from"] = from
  1133. queryMap["size"] = size
  1134. queryMap["highlight"] = map[string]interface{}{
  1135. "fields": map[string]interface{}{
  1136. keywordNameKey: map[string]interface{}{},
  1137. },
  1138. "pre_tags": "<span style=\"color:#0052D9\">",
  1139. "post_tags": "</span>",
  1140. }
  1141. jsonBytes, _ := json.Marshal(queryMap)
  1142. fmt.Println(string(jsonBytes))
  1143. request := client.Search(indexName).Highlight(highlight).Source(queryMap) // sets the JSON request
  1144. //requestJson, err := json.Marshal(request)
  1145. //if err != nil {
  1146. // fmt.Println("requestJson err:", err)
  1147. //}
  1148. //fmt.Println("requestJson ", string(requestJson))
  1149. searchMap := make(map[string]string)
  1150. searchResp, err := request.Do(context.Background())
  1151. if err != nil {
  1152. return
  1153. }
  1154. fmt.Println(searchResp)
  1155. fmt.Println(searchResp.Status)
  1156. if searchResp.Status != 0 {
  1157. return
  1158. }
  1159. if searchResp.Hits != nil {
  1160. for _, v := range searchResp.Hits.Hits {
  1161. if _, ok := searchMap[v.Id]; !ok {
  1162. itemJson, tmpErr := v.Source.MarshalJSON()
  1163. if tmpErr != nil {
  1164. err = tmpErr
  1165. fmt.Println("movieJson err:", err)
  1166. return
  1167. }
  1168. chartInfoItem := new(data_manage.ChartInfoMore)
  1169. tmpErr = json.Unmarshal(itemJson, &chartInfoItem)
  1170. if err != nil {
  1171. fmt.Println("json.Unmarshal chartInfoJson err:", err)
  1172. err = tmpErr
  1173. return
  1174. }
  1175. if len(v.Highlight[keywordNameKey]) > 0 {
  1176. //chartInfoItem.ChartName = v.Highlight["ChartName"][0]
  1177. chartInfoItem.SearchText = v.Highlight[keywordNameKey][0]
  1178. } else {
  1179. chartInfoItem.SearchText = chartInfoItem.ChartName
  1180. }
  1181. list = append(list, chartInfoItem)
  1182. searchMap[v.Id] = v.Id
  1183. }
  1184. }
  1185. }
  1186. //for _, v := range result {
  1187. // fmt.Println(v)
  1188. //}
  1189. return
  1190. }
  1191. // EsAddOrEditDataInterface 新增/修改es中的数据
  1192. func EsAddOrEditDataInterface(indexName, docId string, item interface{}) (err error) {
  1193. defer func() {
  1194. if err != nil {
  1195. fmt.Println("EsAddOrEditData Err:", err.Error())
  1196. }
  1197. }()
  1198. client := utils.EsClient
  1199. resp, err := client.Index().Index(indexName).Id(docId).BodyJson(item).Do(context.Background())
  1200. if err != nil {
  1201. fmt.Println("新增失败:", err.Error())
  1202. return err
  1203. }
  1204. fmt.Println(resp)
  1205. if resp.Status == 0 {
  1206. fmt.Println("新增成功", resp.Result)
  1207. err = nil
  1208. } else {
  1209. fmt.Println("AddData", resp.Status, resp.Result)
  1210. }
  1211. return
  1212. }
  1213. // SearchMyChartInfoData 查询es中的我的图表数据
  1214. func SearchMyChartInfoData(indexName, keywordStr string, adminId int, noPermissionChartIdList []int, from, size int) (list []*data_manage.MyChartList, total int64, err error) {
  1215. list = make([]*data_manage.MyChartList, 0)
  1216. defer func() {
  1217. if err != nil {
  1218. fmt.Println("EsAddOrEditData Err:", err.Error())
  1219. }
  1220. }()
  1221. client := utils.EsClient
  1222. //queryString := elastic.NewQueryStringQuery(keywordStr)
  1223. //boolQueryJson, err := json.Marshal(queryString)
  1224. //if err != nil {
  1225. // fmt.Println("boolQueryJson err:", err)
  1226. //} else {
  1227. // fmt.Println("boolQueryJson ", string(boolQueryJson))
  1228. //}
  1229. highlight := elastic.NewHighlight()
  1230. highlight = highlight.Fields(elastic.NewHighlighterField("ChartName"))
  1231. highlight = highlight.PreTags("<font color='red'>").PostTags("</font>")
  1232. mustMap := make([]interface{}, 0)
  1233. mustNotMap := make([]interface{}, 0)
  1234. //指标来源
  1235. if adminId > 0 {
  1236. mustMap = append(mustMap, map[string]interface{}{
  1237. "term": map[string]interface{}{
  1238. "AdminId": adminId,
  1239. //"Frequency.keyword": "月度",
  1240. },
  1241. })
  1242. }
  1243. //关键字匹配
  1244. //shouldMap := map[string]interface{}{
  1245. // "should": []interface{}{
  1246. // map[string]interface{}{
  1247. // "match": map[string]interface{}{
  1248. // "ChartName": keywordStr,
  1249. // //"Frequency.keyword": "月度",
  1250. // },
  1251. // },
  1252. // // 因为关键词被分了,所以需要用下面的语句来让他 整个词 查询,从而加重整词的权重
  1253. // map[string]interface{}{
  1254. // "match": map[string]interface{}{
  1255. // "ChartName": map[string]interface{}{
  1256. // "query": keywordStr,
  1257. // "operator": "and",
  1258. // },
  1259. // //"Frequency.keyword": "月度",
  1260. // },
  1261. // },
  1262. // map[string]interface{}{
  1263. // "match": map[string]interface{}{
  1264. // "ChartNameEn": keywordStr,
  1265. // //"Frequency.keyword": "月度",
  1266. // },
  1267. // },
  1268. // // 因为关键词被分了,所以需要用下面的语句来让他 整个词 查询,从而加重整词的权重
  1269. // map[string]interface{}{
  1270. // "match": map[string]interface{}{
  1271. // "ChartNameEn": map[string]interface{}{
  1272. // "query": keywordStr,
  1273. // "operator": "and",
  1274. // },
  1275. // //"Frequency.keyword": "月度",
  1276. // },
  1277. // },
  1278. // },
  1279. //}
  1280. // 默认使用中文名字字段去匹配
  1281. keywordNameKey := `ChartName`
  1282. // 如果没有中文,则使用英文名称字段去匹配
  1283. if !utils.ContainsChinese(keywordStr) {
  1284. keywordNameKey = `ChartNameEn`
  1285. }
  1286. shouldMap := map[string]interface{}{
  1287. "should": []interface{}{
  1288. map[string]interface{}{
  1289. "match": map[string]interface{}{
  1290. keywordNameKey: keywordStr,
  1291. //"Frequency.keyword": "月度",
  1292. },
  1293. },
  1294. // 因为关键词被分了,所以需要用下面的语句来让他 整个词 查询,从而加重整词的权重
  1295. map[string]interface{}{
  1296. "match": map[string]interface{}{
  1297. keywordNameKey: map[string]interface{}{
  1298. "query": keywordStr,
  1299. "operator": "and",
  1300. },
  1301. //"Frequency.keyword": "月度",
  1302. },
  1303. },
  1304. },
  1305. }
  1306. mustMap = append(mustMap, map[string]interface{}{
  1307. "bool": shouldMap,
  1308. })
  1309. // noPermissionEdbInfoIdList 无权限指标id
  1310. if len(noPermissionChartIdList) > 0 {
  1311. mustNotMap = append(mustNotMap, map[string]interface{}{
  1312. "terms": map[string]interface{}{
  1313. "ChartInfoId": noPermissionChartIdList,
  1314. //"Frequency.keyword": "月度",
  1315. },
  1316. })
  1317. }
  1318. queryMap := map[string]interface{}{
  1319. "query": map[string]interface{}{
  1320. "bool": map[string]interface{}{
  1321. "must": mustMap,
  1322. "must_not": mustNotMap,
  1323. //"should": shouldMap,
  1324. },
  1325. },
  1326. }
  1327. //根据条件数量统计
  1328. requestTotalHits := client.Count(indexName).BodyJson(queryMap)
  1329. total, err = requestTotalHits.Do(context.Background())
  1330. if err != nil {
  1331. return
  1332. }
  1333. // 分页查询
  1334. queryMap["from"] = from
  1335. queryMap["size"] = size
  1336. queryMap["highlight"] = map[string]interface{}{
  1337. "fields": map[string]interface{}{
  1338. keywordNameKey: map[string]interface{}{},
  1339. },
  1340. "pre_tags": "<span style=\"color:#0052D9\">",
  1341. "post_tags": "</span>",
  1342. }
  1343. jsonBytes, _ := json.Marshal(queryMap)
  1344. fmt.Println(string(jsonBytes))
  1345. request := client.Search(indexName).Highlight(highlight).Source(queryMap) // sets the JSON request
  1346. //requestJson, err := json.Marshal(request)
  1347. //if err != nil {
  1348. // fmt.Println("requestJson err:", err)
  1349. //}
  1350. //fmt.Println("requestJson ", string(requestJson))
  1351. searchMap := make(map[string]string)
  1352. searchResp, err := request.Do(context.Background())
  1353. if err != nil {
  1354. return
  1355. }
  1356. fmt.Println(searchResp)
  1357. fmt.Println(searchResp.Status)
  1358. if searchResp.Status != 0 {
  1359. return
  1360. }
  1361. if searchResp.Hits != nil {
  1362. for _, v := range searchResp.Hits.Hits {
  1363. if _, ok := searchMap[v.Id]; !ok {
  1364. itemJson, tmpErr := v.Source.MarshalJSON()
  1365. if tmpErr != nil {
  1366. err = tmpErr
  1367. fmt.Println("movieJson err:", err)
  1368. return
  1369. }
  1370. chartInfoItem := new(data_manage.MyChartList)
  1371. tmpErr = json.Unmarshal(itemJson, &chartInfoItem)
  1372. if err != nil {
  1373. fmt.Println("json.Unmarshal chartInfoJson err:", err)
  1374. err = tmpErr
  1375. return
  1376. }
  1377. if len(v.Highlight[keywordNameKey]) > 0 {
  1378. //chartInfoItem.ChartName = v.Highlight["ChartName"][0]
  1379. chartInfoItem.SearchText = v.Highlight[keywordNameKey][0]
  1380. } else {
  1381. chartInfoItem.SearchText = chartInfoItem.ChartName
  1382. }
  1383. list = append(list, chartInfoItem)
  1384. searchMap[v.Id] = v.Id
  1385. }
  1386. }
  1387. }
  1388. //for _, v := range result {
  1389. // fmt.Println(v)
  1390. //}
  1391. return
  1392. }
  1393. // SearchEdbInfoDataByAdminId 查询es中的指标数据
  1394. func SearchEdbInfoDataByAdminId(indexName, keywordStr string, from, size, filterSource, source int, edbInfoType uint8, frequency string, adminId int) (total int64, list []*data_manage.EdbInfoList, err error) {
  1395. list = make([]*data_manage.EdbInfoList, 0)
  1396. defer func() {
  1397. if err != nil {
  1398. fmt.Println("EsAddOrEditData Err:", err.Error())
  1399. }
  1400. }()
  1401. highlight := elastic.NewHighlight()
  1402. highlight = highlight.Fields(elastic.NewHighlighterField("EdbCode"), elastic.NewHighlighterField("EdbName"))
  1403. highlight = highlight.PreTags("<font color='red'>").PostTags("</font>")
  1404. //var source map[string]interface{}
  1405. //source := map[string]interface{}{
  1406. // "query": map[string]interface{}{
  1407. // "match_all": map[string]interface{}{},
  1408. // },
  1409. //}
  1410. mustMap := make([]interface{}, 0)
  1411. mustNotMap := make([]interface{}, 0)
  1412. //source := map[string]interface{}{
  1413. // "query": map[string]interface{}{
  1414. // "bool": map[string]interface{}{
  1415. // "must": map[string]interface{}{
  1416. // "query_string": map[string]interface{}{
  1417. // "query": keywordStr,
  1418. // "fields": []string{"EdbCode", "EdbName"},
  1419. // },
  1420. // },
  1421. // },
  1422. // },
  1423. //}
  1424. switch filterSource {
  1425. case 2:
  1426. //source = map[string]interface{}{
  1427. // "query": map[string]interface{}{
  1428. // "bool": map[string]interface{}{
  1429. // "must": map[string]interface{}{
  1430. // "query_string": map[string]interface{}{
  1431. // "query": keywordStr,
  1432. // },
  1433. // },
  1434. // "filter": []interface{}{
  1435. // map[string]interface{}{
  1436. // "term": map[string]interface{}{
  1437. // "Frequency.keyword": "月度",
  1438. // },
  1439. // }},
  1440. // },
  1441. // },
  1442. //}
  1443. mustMap = []interface{}{
  1444. map[string]interface{}{
  1445. "term": map[string]interface{}{
  1446. "Frequency.keyword": "月度",
  1447. //"Frequency.keyword": "月度",
  1448. },
  1449. },
  1450. }
  1451. case 3:
  1452. //source = map[string]interface{}{
  1453. // "query": map[string]interface{}{
  1454. // "bool": map[string]interface{}{
  1455. // "must": map[string]interface{}{
  1456. // "query_string": map[string]interface{}{
  1457. // "query": keywordStr,
  1458. // },
  1459. // },
  1460. // "must_not": []interface{}{
  1461. // map[string]interface{}{
  1462. // "match": map[string]interface{}{
  1463. // "Frequency.keyword": "日度",
  1464. // },
  1465. // }},
  1466. // },
  1467. // },
  1468. //}
  1469. ////注释掉,所有频度都可以变频 2022-08-31 14:31:28
  1470. //mustNotMap = []interface{}{
  1471. // map[string]interface{}{
  1472. // "match": map[string]interface{}{
  1473. // "Frequency.keyword": "日度",
  1474. // //"Frequency.keyword": "月度",
  1475. // },
  1476. // },
  1477. //}
  1478. case 4:
  1479. //source = map[string]interface{}{
  1480. // "query": map[string]interface{}{
  1481. // "bool": map[string]interface{}{
  1482. // "must": map[string]interface{}{
  1483. // "query_string": map[string]interface{}{
  1484. // "query": keywordStr,
  1485. // },
  1486. // },
  1487. // "filter": []interface{}{
  1488. // map[string]interface{}{
  1489. // "term": map[string]interface{}{
  1490. // "EdbType": 1,
  1491. // },
  1492. // }},
  1493. // },
  1494. // },
  1495. //}
  1496. mustMap = []interface{}{
  1497. map[string]interface{}{
  1498. "term": map[string]interface{}{
  1499. "EdbType": 1,
  1500. },
  1501. },
  1502. }
  1503. case 5:
  1504. mustMap = []interface{}{
  1505. map[string]interface{}{
  1506. "term": map[string]interface{}{
  1507. "Source": 6,
  1508. },
  1509. },
  1510. }
  1511. case 6:
  1512. mustNotMap = []interface{}{
  1513. map[string]interface{}{
  1514. "match": map[string]interface{}{
  1515. "Frequency.keyword": "年度",
  1516. },
  1517. },
  1518. }
  1519. }
  1520. //指标来源
  1521. if source > 0 {
  1522. mustMap = append(mustMap, map[string]interface{}{
  1523. "term": map[string]interface{}{
  1524. "Source": source,
  1525. //"Frequency.keyword": "月度",
  1526. },
  1527. })
  1528. }
  1529. if frequency != "" {
  1530. mustMap = append(mustMap, map[string]interface{}{
  1531. "term": map[string]interface{}{
  1532. "Frequency.keyword": frequency,
  1533. //"Frequency.keyword": "月度",
  1534. },
  1535. })
  1536. }
  1537. // 指标类型:普通指标、预算指标
  1538. mustMap = append(mustMap, map[string]interface{}{
  1539. "term": map[string]interface{}{
  1540. "EdbInfoType": edbInfoType,
  1541. },
  1542. })
  1543. //普通指标
  1544. //mustMap = append(mustMap, map[string]interface{}{
  1545. // "term": map[string]interface{}{
  1546. // "EdbInfoType": 0,
  1547. // //"Frequency.keyword": "月度",
  1548. // },
  1549. //})
  1550. //关键字匹配
  1551. //shouldMap := map[string]interface{}{
  1552. // "should": []interface{}{
  1553. // map[string]interface{}{
  1554. // "match": map[string]interface{}{
  1555. // "EdbCode": keywordStr,
  1556. // //"Frequency.keyword": "月度",
  1557. // },
  1558. // },
  1559. // map[string]interface{}{
  1560. // "match": map[string]interface{}{
  1561. // "EdbName": keywordStr,
  1562. // //"Frequency.keyword": "月度",
  1563. // },
  1564. // },
  1565. // },
  1566. //}
  1567. // 默认使用中文名字字段去匹配
  1568. keywordNameKey := `EdbName`
  1569. // 如果没有中文,则使用英文名称字段去匹配
  1570. if !utils.ContainsChinese(keywordStr) {
  1571. keywordNameKey = `EdbNameEn`
  1572. }
  1573. shouldMap := map[string]interface{}{
  1574. "should": []interface{}{
  1575. map[string]interface{}{
  1576. "match": map[string]interface{}{
  1577. "EdbCode": keywordStr,
  1578. //"Frequency.keyword": "月度",
  1579. },
  1580. },
  1581. map[string]interface{}{
  1582. "match": map[string]interface{}{
  1583. keywordNameKey: keywordStr,
  1584. //"Frequency.keyword": "月度",
  1585. },
  1586. },
  1587. },
  1588. }
  1589. mustMap = append(mustMap, map[string]interface{}{
  1590. "bool": shouldMap,
  1591. })
  1592. //创建人
  1593. if adminId > 0 {
  1594. mustMap = append(mustMap, map[string]interface{}{
  1595. "term": map[string]interface{}{
  1596. "SysUserId": adminId,
  1597. },
  1598. })
  1599. }
  1600. return searchEdbInfoData(indexName, mustMap, mustNotMap, shouldMap, from, size)
  1601. }
  1602. // EsAddOrEditExcelInfoData 新增/修改es中的表格数据
  1603. func EsAddOrEditExcelInfoData(indexName, docId string, item *excel.ExcelInfo) (err error) {
  1604. defer func() {
  1605. if err != nil {
  1606. tips := fmt.Sprintf("EsAddOrEditExcelInfoData err: %v", err)
  1607. utils.FileLog.Info(tips)
  1608. }
  1609. }()
  1610. client := utils.EsClient
  1611. resp, e := client.Index().Index(indexName).Id(docId).BodyJson(item).Do(context.Background())
  1612. if e != nil {
  1613. err = fmt.Errorf("resp err, %v", e)
  1614. return
  1615. }
  1616. if resp.Status != 0 {
  1617. err = fmt.Errorf("result err, status: %d, result: %s", resp.Status, resp.Result)
  1618. return
  1619. }
  1620. err = nil
  1621. fmt.Println("excel write to es success", resp.Result)
  1622. return
  1623. }
  1624. // SearchExcelInfoData 查询es中的表格
  1625. func SearchExcelInfoData(indexName, keyword string, source, adminId int, queryIds, exceptIds []int, from, size int) (total int64, list []*excel.SearchExcelInfo, err error) {
  1626. list = make([]*excel.SearchExcelInfo, 0)
  1627. defer func() {
  1628. if err != nil {
  1629. tips := fmt.Sprintf("SearchExcelInfoData err: %v", err)
  1630. utils.FileLog.Info(tips)
  1631. }
  1632. }()
  1633. client := utils.EsClient
  1634. mustMap := make([]interface{}, 0)
  1635. mustNotMap := make([]interface{}, 0)
  1636. mustMap = append(mustMap, map[string]interface{}{
  1637. "term": map[string]interface{}{
  1638. "IsDelete": 0,
  1639. },
  1640. })
  1641. // 表格名称
  1642. shouldMap := make(map[string]interface{}, 0)
  1643. if keyword != "" {
  1644. shouldMap["should"] = []interface{}{
  1645. map[string]interface{}{
  1646. "match": map[string]interface{}{
  1647. "ExcelName": keyword,
  1648. },
  1649. },
  1650. }
  1651. //shouldMap := map[string]interface{}{
  1652. // "should": []interface{}{
  1653. // map[string]interface{}{
  1654. // "match": map[string]interface{}{
  1655. // "ExcelName": keyword,
  1656. // },
  1657. // },
  1658. // },
  1659. //}
  1660. }
  1661. // 表格来源
  1662. if source > 0 {
  1663. mustMap = append(mustMap, map[string]interface{}{
  1664. "term": map[string]interface{}{
  1665. "Source": source,
  1666. },
  1667. })
  1668. }
  1669. // 创建人
  1670. if adminId > 0 {
  1671. mustMap = append(mustMap, map[string]interface{}{
  1672. "term": map[string]interface{}{
  1673. "SysUserId": adminId,
  1674. },
  1675. })
  1676. }
  1677. // 查询和排除的表格IDs
  1678. if len(queryIds) > 0 {
  1679. mustMap = append(mustMap, map[string]interface{}{
  1680. "terms": map[string]interface{}{
  1681. "ExcelInfoId": queryIds,
  1682. },
  1683. })
  1684. }
  1685. if len(exceptIds) > 0 {
  1686. mustNotMap = append(mustNotMap, map[string]interface{}{
  1687. "terms": map[string]interface{}{
  1688. "ExcelInfoId": exceptIds,
  1689. },
  1690. })
  1691. }
  1692. // 关键字匹配
  1693. mustMap = append(mustMap, map[string]interface{}{
  1694. "bool": shouldMap,
  1695. })
  1696. queryMap := map[string]interface{}{
  1697. "query": map[string]interface{}{
  1698. "bool": map[string]interface{}{
  1699. "must": mustMap,
  1700. "must_not": mustNotMap,
  1701. //"should": shouldMap,
  1702. },
  1703. },
  1704. }
  1705. //jsonBytes, _ := json.Marshal(queryMap)
  1706. //fmt.Println(string(jsonBytes))
  1707. // 根据条件数量统计
  1708. requestTotalHits := client.Count(indexName).BodyJson(queryMap)
  1709. t, e := requestTotalHits.Do(context.Background())
  1710. if e != nil {
  1711. err = fmt.Errorf("total hits err: %v", e)
  1712. return
  1713. }
  1714. total = t
  1715. // 表格名称高亮,分页
  1716. highlightKeyName := "ExcelName"
  1717. queryMap["highlight"] = map[string]interface{}{
  1718. "fields": map[string]interface{}{
  1719. highlightKeyName: map[string]interface{}{},
  1720. },
  1721. "pre_tags": "<span style=\"color:#0052D9\">",
  1722. "post_tags": "</span>",
  1723. }
  1724. queryMap["from"] = from
  1725. queryMap["size"] = size
  1726. //jsonBytes, _ := json.Marshal(queryMap)
  1727. //fmt.Println(string(jsonBytes))
  1728. request := client.Search(indexName).Source(queryMap)
  1729. searchResp, e := request.Do(context.Background())
  1730. if e != nil {
  1731. err = fmt.Errorf("search do err: %v", e)
  1732. return
  1733. }
  1734. //fmt.Println(searchResp)
  1735. if searchResp.Status != 0 {
  1736. return
  1737. }
  1738. if searchResp.Hits == nil {
  1739. return
  1740. }
  1741. searchMap := make(map[string]string)
  1742. for _, v := range searchResp.Hits.Hits {
  1743. if _, ok := searchMap[v.Id]; ok {
  1744. continue
  1745. }
  1746. j, e := v.Source.MarshalJSON()
  1747. if e != nil {
  1748. err = fmt.Errorf("hits json err: %v", e)
  1749. return
  1750. }
  1751. item := new(excel.SearchExcelInfo)
  1752. if e = json.Unmarshal(j, &item); e != nil {
  1753. err = fmt.Errorf("hits json unmarshal err: %v", e)
  1754. return
  1755. }
  1756. if len(v.Highlight[highlightKeyName]) > 0 {
  1757. item.SearchText = v.Highlight[highlightKeyName][0]
  1758. } else {
  1759. item.SearchText = item.ExcelName
  1760. }
  1761. list = append(list, item)
  1762. searchMap[v.Id] = v.Id
  1763. }
  1764. return
  1765. }
  1766. // EsAddOrEditDataSourceIndex 新增/修改es中的数据源指标
  1767. func EsAddOrEditDataSourceIndex(indexName, docId string, item *dataSourceModel.SearchDataSource) (err error) {
  1768. defer func() {
  1769. if err != nil {
  1770. tips := fmt.Sprintf("EsAddOrEditDataSourceIndex err: %v", err)
  1771. utils.FileLog.Info(tips)
  1772. }
  1773. }()
  1774. client := utils.EsClient
  1775. resp, e := client.Index().Index(indexName).Id(docId).BodyJson(item).Do(context.Background())
  1776. if e != nil {
  1777. err = fmt.Errorf("resp err, %v", e)
  1778. return
  1779. }
  1780. if resp.Status != 0 {
  1781. err = fmt.Errorf("result err, status: %d, result: %s", resp.Status, resp.Result)
  1782. return
  1783. }
  1784. err = nil
  1785. fmt.Println("data source write to es success", resp.Result)
  1786. return
  1787. }
  1788. // SearchDataSourceIndex 查询es中的数据源
  1789. func SearchDataSourceIndex(indexName, keyword string, source, subSource int, classifyIds, adminIds []int, frequency []string, from, size int) (total int64, list []*dataSourceModel.SearchDataSourceItem, err error) {
  1790. list = make([]*dataSourceModel.SearchDataSourceItem, 0)
  1791. defer func() {
  1792. if err != nil {
  1793. tips := fmt.Sprintf("SearchDataSourceIndex err: %v", err)
  1794. utils.FileLog.Info(tips)
  1795. }
  1796. }()
  1797. client := utils.EsClient
  1798. mustMap := make([]interface{}, 0)
  1799. mustNotMap := make([]interface{}, 0)
  1800. mustMap = append(mustMap, map[string]interface{}{
  1801. "term": map[string]interface{}{
  1802. "IsDeleted": 0,
  1803. },
  1804. })
  1805. // 指标编码/名称
  1806. //shouldMap := make(map[string]interface{}, 0)
  1807. if keyword != "" {
  1808. //shouldMap["should"] = []interface{}{
  1809. // map[string]interface{}{
  1810. // "match": map[string]interface{}{
  1811. // "IndexCode": keyword,
  1812. // },
  1813. // },
  1814. // map[string]interface{}{
  1815. // "match": map[string]interface{}{
  1816. // "IndexName": keyword,
  1817. // },
  1818. // },
  1819. //}
  1820. queryFields := []string{"IndexName.keyword", "IndexName.ik", "IndexName.ngram", "IndexCode.keyword", "IndexCode.ngram"}
  1821. mustMap = append(mustMap, map[string]interface{}{
  1822. "multi_match": map[string]interface{}{
  1823. "query": keyword,
  1824. "fields": queryFields,
  1825. },
  1826. })
  1827. }
  1828. // 来源/子来源
  1829. if source > 0 {
  1830. mustMap = append(mustMap, map[string]interface{}{
  1831. "term": map[string]interface{}{
  1832. "Source": source,
  1833. },
  1834. })
  1835. }
  1836. if subSource > 0 {
  1837. mustMap = append(mustMap, map[string]interface{}{
  1838. "term": map[string]interface{}{
  1839. "SubSource": subSource,
  1840. },
  1841. })
  1842. }
  1843. // 分类
  1844. if len(classifyIds) > 0 {
  1845. mustMap = append(mustMap, map[string]interface{}{
  1846. "term": map[string]interface{}{
  1847. "ClassifyId": classifyIds,
  1848. },
  1849. })
  1850. }
  1851. // 创建人
  1852. if len(adminIds) > 0 {
  1853. mustMap = append(mustMap, map[string]interface{}{
  1854. "term": map[string]interface{}{
  1855. "SysUserId": adminIds,
  1856. },
  1857. })
  1858. }
  1859. // 频度
  1860. if len(frequency) > 0 {
  1861. mustMap = append(mustMap, map[string]interface{}{
  1862. "term": map[string]interface{}{
  1863. "Frequency": frequency,
  1864. },
  1865. })
  1866. }
  1867. // 关键字匹配
  1868. //mustMap = append(mustMap, map[string]interface{}{
  1869. // "bool": shouldMap,
  1870. //})
  1871. queryMap := map[string]interface{}{
  1872. "query": map[string]interface{}{
  1873. "bool": map[string]interface{}{
  1874. "must": mustMap,
  1875. "must_not": mustNotMap,
  1876. //"should": shouldMap,
  1877. },
  1878. },
  1879. }
  1880. //tj, _ := json.Marshal(queryMap)
  1881. //utils.FileLog.Info(string(tj))
  1882. // 根据条件数量统计
  1883. requestTotalHits := client.Count(indexName).BodyJson(queryMap)
  1884. t, e := requestTotalHits.Do(context.Background())
  1885. if e != nil {
  1886. err = fmt.Errorf("total hits err: %v", e)
  1887. return
  1888. }
  1889. total = t
  1890. // 表格名称高亮,分页
  1891. //highlightKeyName := "IndexName"
  1892. highlightIk := "IndexName.ik"
  1893. highlightKeyword := "IndexName.keyword"
  1894. queryMap["highlight"] = map[string]interface{}{
  1895. "fields": map[string]interface{}{
  1896. //highlightKeyName: map[string]interface{}{},
  1897. highlightIk: map[string]interface{}{},
  1898. highlightKeyword: map[string]interface{}{},
  1899. },
  1900. "pre_tags": "<span style=\"color:#0052D9\">",
  1901. "post_tags": "</span>",
  1902. }
  1903. queryMap["from"] = from
  1904. queryMap["size"] = size
  1905. qj, _ := json.Marshal(queryMap)
  1906. utils.FileLog.Info(string(qj))
  1907. request := client.Search(indexName).Source(queryMap)
  1908. searchResp, e := request.Do(context.Background())
  1909. if e != nil {
  1910. err = fmt.Errorf("search do err: %v", e)
  1911. return
  1912. }
  1913. rj, _ := json.Marshal(searchResp)
  1914. utils.FileLog.Info(string(rj))
  1915. if searchResp.Status != 0 {
  1916. return
  1917. }
  1918. if searchResp.Hits == nil {
  1919. return
  1920. }
  1921. searchMap := make(map[string]string)
  1922. for _, v := range searchResp.Hits.Hits {
  1923. if _, ok := searchMap[v.Id]; ok {
  1924. continue
  1925. }
  1926. j, e := v.Source.MarshalJSON()
  1927. if e != nil {
  1928. err = fmt.Errorf("hits json err: %v", e)
  1929. return
  1930. }
  1931. item := new(dataSourceModel.SearchDataSourceItem)
  1932. if e = json.Unmarshal(j, &item); e != nil {
  1933. err = fmt.Errorf("hits json unmarshal err: %v", e)
  1934. return
  1935. }
  1936. //if len(v.Highlight[highlightKeyName]) > 0 {
  1937. // item.SearchText = v.Highlight[highlightKeyName][0]
  1938. //}
  1939. // 优先取keyword的高亮,其次是ik分词之后的高亮
  1940. if len(v.Highlight) > 0 {
  1941. if len(v.Highlight[highlightKeyword]) > 0 {
  1942. item.SearchText = v.Highlight[highlightKeyword][0]
  1943. }
  1944. if len(v.Highlight[highlightKeyword]) == 0 && len(v.Highlight[highlightIk]) > 0 {
  1945. item.SearchText = v.Highlight[highlightIk][0]
  1946. }
  1947. }
  1948. if item.SearchText == "" {
  1949. item.SearchText = item.IndexName
  1950. }
  1951. list = append(list, item)
  1952. searchMap[v.Id] = v.Id
  1953. }
  1954. return
  1955. }
  1956. // CreateEsIndex 创建ES索引
  1957. func CreateEsIndex(indexName, jsonMapping string) (err error) {
  1958. defer func() {
  1959. if err != nil {
  1960. tips := fmt.Sprintf("CreateEsIndex err: %v", err)
  1961. utils.FileLog.Info(tips)
  1962. }
  1963. }()
  1964. client := utils.EsClient
  1965. // 校验是否存在
  1966. exists, e := client.IndexExists(indexName).Do(context.Background())
  1967. if e != nil {
  1968. err = fmt.Errorf("check exists err: %v", e)
  1969. return
  1970. }
  1971. if exists {
  1972. fmt.Printf("索引已存在: %s, 跳过\n", indexName)
  1973. return
  1974. }
  1975. // 创建索引及映射
  1976. createIndex, e := client.CreateIndex(indexName).BodyJson(jsonMapping).Do(context.Background())
  1977. if e != nil {
  1978. err = fmt.Errorf("create index err: %v", e)
  1979. return
  1980. }
  1981. fmt.Printf("create index success: %s\n", createIndex.Index)
  1982. return
  1983. }