es_comprehensive.go 25 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738
  1. package services
  2. import (
  3. "context"
  4. "encoding/json"
  5. "fmt"
  6. "github.com/PuerkitoBio/goquery"
  7. "github.com/olivere/elastic/v7"
  8. "hongze/hongze_web_mfyx/models"
  9. "hongze/hongze_web_mfyx/utils"
  10. "strconv"
  11. //"strconv"
  12. "errors"
  13. "html"
  14. "strings"
  15. )
  16. type SearchComprehensiveItem struct {
  17. SourceId int `description:"资源ID"`
  18. IsSummary int `description:"是否是纪要"`
  19. Source string `description:"资源类型 报告 :article 、图表 :newchart、微路演 :roadshow、活动 :activity、活动视频:activityvideo、活动音频:activityvoice、专项调研活动:activityspecial 、 本周研究汇总: researchsummary 、 上周纪要汇总 :minutessummary 、晨会精华 :meetingreviewchapt 、 产品内测:productinterior"`
  20. Title string `description:"标题"`
  21. BodyText string `description:"内容"`
  22. PublishDate string `description:"发布时间"`
  23. Abstract string `description:"摘要"`
  24. Annotation string `description:"核心观点"`
  25. IndustryName string `description:"产业名称"`
  26. SubjectNames string `description:"标的名称"`
  27. Body []string
  28. }
  29. type ElasticComprehensiveDetail struct {
  30. SourceId int `description:"资源ID"`
  31. IsSummary int `description:"是否是纪要"`
  32. Source string `description:"资源类型 报告 :article 、图表 :newchart、微路演 :roadshow、活动 :activity、活动视频:activityvideo、活动音频:activityvoice、专项调研活动:activityspecial 、 本周研究汇总: researchsummary 、 上周纪要汇总 :minutessummary 、晨会精华 :meetingreviewchapt 、 产品内测:productinterior 、 产业资源包:industrialsource"`
  33. Title string `description:"标题"`
  34. BodyText string `description:"内容"`
  35. PublishDate string `description:"发布时间"`
  36. Abstract string `description:"摘要"`
  37. Annotation string `description:"核心观点"`
  38. IndustryName string `description:"产业名称"`
  39. SubjectNames string `description:"标的名称"`
  40. }
  41. func EsComprehensiveSearch(keyWord string, startSize, pageSize int) (result []*SearchComprehensiveItem, total int64, err error) {
  42. indexName := utils.IndexNameComprehensive
  43. client := utils.Client
  44. keyWordArr, err := GetIndustryMapNameSliceV3(keyWord)
  45. keyWordArr = RemoveDuplicatesAndEmpty(keyWordArr)
  46. keyWordLen := len(keyWordArr)
  47. if keyWordLen <= 0 {
  48. keyWordArr = append(keyWordArr, keyWord)
  49. keyWordLen = len(keyWordArr)
  50. }
  51. //Es 的高级查询有 自定义排序 文档一时半会儿撸不懂,先做多次查询手动过滤 2023.2.2
  52. //ikType 查询方式 ,0:查所有 、 1:查询键入词 、 2:查询除了查询键入词之外的联想词
  53. mustMap := make([]interface{}, 0)
  54. shouldMap := make(map[string]interface{}, 0)
  55. shouldMapquery := make([]interface{}, 0)
  56. mustNotMap := make([]interface{}, 0)
  57. shouldNotMap := make(map[string]interface{}, 0)
  58. shouldNotMapquery := make([]interface{}, 0)
  59. // @Param OrderColumn query int true "排序字段 ,Comprehensive综合 ,Matching匹配度 ,PublishDate 发布时间 "
  60. //keyWordWeight := GetWeight(keyWordLen)
  61. var boost int
  62. //lenkeyWordArr := len(keyWordArr)
  63. for k, v := range keyWordArr {
  64. if k > 0 {
  65. continue
  66. }
  67. if k == 0 {
  68. boost = 2 * 1000
  69. } else {
  70. boost = 1
  71. }
  72. if v != "" {
  73. shouldMapquery = append(shouldMapquery, map[string]interface{}{
  74. "function_score": map[string]interface{}{
  75. "query": map[string]interface{}{
  76. "multi_match": map[string]interface{}{
  77. //"boost": (lenkeyWordArr - k) * boost, //给查询的值赋予权重
  78. "boost": boost, //给查询的值赋予权重
  79. "fields": []interface{}{"Title"},
  80. "query": v,
  81. },
  82. },
  83. },
  84. })
  85. shouldMapquery = append(shouldMapquery, map[string]interface{}{
  86. "function_score": map[string]interface{}{
  87. "query": map[string]interface{}{
  88. "multi_match": map[string]interface{}{
  89. "boost": boost, //给查询的值赋予权重
  90. "fields": []interface{}{"Abstract"},
  91. "query": v,
  92. },
  93. },
  94. },
  95. })
  96. shouldMapquery = append(shouldMapquery, map[string]interface{}{
  97. "function_score": map[string]interface{}{
  98. "query": map[string]interface{}{
  99. "multi_match": map[string]interface{}{
  100. "boost": boost, //给查询的值赋予权重
  101. "fields": []interface{}{"Annotation"},
  102. "query": v,
  103. },
  104. },
  105. },
  106. })
  107. //shouldMapquery = append(shouldMapquery, map[string]interface{}{
  108. // "function_score": map[string]interface{}{
  109. // "query": map[string]interface{}{
  110. // "multi_match": map[string]interface{}{
  111. // //"boost": (lenkeyWordArr-k)*boost - 1, //给查询的值赋予权重
  112. // "boost": boost, //给查询的值赋予权重
  113. // "fields": []interface{}{"BodyText"},
  114. // "query": v,
  115. // },
  116. // },
  117. // },
  118. //})
  119. shouldMapquery = append(shouldMapquery, map[string]interface{}{
  120. "function_score": map[string]interface{}{
  121. "query": map[string]interface{}{
  122. "multi_match": map[string]interface{}{
  123. //"boost": (lenkeyWordArr-k)*boost - 1, //给查询的值赋予权重
  124. "boost": boost, //给查询的值赋予权重
  125. "fields": []interface{}{"IndustryName"},
  126. "query": v,
  127. },
  128. },
  129. },
  130. })
  131. shouldMapquery = append(shouldMapquery, map[string]interface{}{
  132. "function_score": map[string]interface{}{
  133. "query": map[string]interface{}{
  134. "multi_match": map[string]interface{}{
  135. //"boost": (lenkeyWordArr-k)*boost - 1, //给查询的值赋予权重
  136. "boost": boost, //给查询的值赋予权重
  137. "fields": []interface{}{"SubjectNames"},
  138. "query": v,
  139. },
  140. },
  141. },
  142. })
  143. }
  144. }
  145. shouldMap = map[string]interface{}{
  146. "should": shouldMapquery,
  147. }
  148. shouldNotMap = map[string]interface{}{
  149. "should": shouldNotMapquery,
  150. }
  151. //排序
  152. sortMap := make([]interface{}, 0)
  153. //时间
  154. sortMap = append(sortMap, map[string]interface{}{
  155. "PublishDate": map[string]interface{}{
  156. "order": "desc",
  157. },
  158. })
  159. //sortMap = append(sortMap, map[string]interface{}{
  160. // "_score": map[string]interface{}{
  161. // "order": "desc",
  162. // },
  163. //})
  164. //高亮
  165. highlightMap := make(map[string]interface{}, 0)
  166. highlightMap = map[string]interface{}{
  167. "fields": map[string]interface{}{
  168. //"BodyText": map[string]interface{}{},
  169. "Title": map[string]interface{}{},
  170. "Abstract": map[string]interface{}{},
  171. "Annotation": map[string]interface{}{},
  172. },
  173. //样式 红色
  174. "post_tags": []interface{}{"</font>"},
  175. "pre_tags": []interface{}{"<font color='red'>"},
  176. "fragment_size": 50,
  177. }
  178. mustMap = append(mustMap, map[string]interface{}{
  179. "bool": shouldMap,
  180. })
  181. mustNotMap = append(mustNotMap, map[string]interface{}{
  182. "bool": shouldNotMap,
  183. })
  184. queryMap := map[string]interface{}{
  185. "query": map[string]interface{}{
  186. "bool": map[string]interface{}{
  187. "must": mustMap,
  188. },
  189. },
  190. }
  191. queryMap["sort"] = sortMap
  192. queryMap["from"] = startSize
  193. queryMap["size"] = pageSize
  194. queryMap["highlight"] = highlightMap
  195. //jsonBytes, _ := json.Marshal(queryMap)
  196. //fmt.Println(string(jsonBytes))
  197. //utils.FileLog.Info(string(jsonBytes))
  198. request := client.Search(indexName).Source(queryMap) // sets the JSON request
  199. searchByMatch, err := request.Do(context.Background())
  200. if searchByMatch != nil {
  201. if searchByMatch.Hits != nil {
  202. for _, v := range searchByMatch.Hits.Hits {
  203. var isAppend bool
  204. articleJson, err := v.Source.MarshalJSON()
  205. if err != nil {
  206. return nil, 0, err
  207. }
  208. article := new(ElasticComprehensiveDetail)
  209. err = json.Unmarshal(articleJson, &article)
  210. if err != nil {
  211. return nil, 0, err
  212. }
  213. //fmt.Println(article.SourceId, article.Title, article.Source)
  214. searchItem := new(SearchComprehensiveItem)
  215. searchItem.SourceId = article.SourceId
  216. if len(v.Highlight["Annotation"]) > 0 {
  217. for _, vText := range v.Highlight["Annotation"] {
  218. searchItem.Body = append(searchItem.Body, vText)
  219. }
  220. }
  221. if len(v.Highlight["Abstract"]) > 0 {
  222. for _, vText := range v.Highlight["Abstract"] {
  223. searchItem.Body = append(searchItem.Body, vText)
  224. }
  225. }
  226. if len(v.Highlight["BodyText"]) > 0 {
  227. for _, vText := range v.Highlight["BodyText"] {
  228. searchItem.Body = append(searchItem.Body, vText)
  229. }
  230. }
  231. //searchItem.IsSummary = article.IsSummary
  232. //if len(searchItem.Body) == 0 {
  233. // bodyRune := []rune(article.BodyText)
  234. // bodyRuneLen := len(bodyRune)
  235. // if bodyRuneLen > 100 {
  236. // bodyRuneLen = 100
  237. // }
  238. // body := string(bodyRune[:bodyRuneLen])
  239. // searchItem.Body = []string{body}
  240. //}
  241. var title string
  242. if len(v.Highlight["Title"]) > 0 {
  243. title = v.Highlight["Title"][0]
  244. } else {
  245. title = article.Title
  246. }
  247. searchItem.Title = title
  248. searchItem.Source = article.Source
  249. searchItem.PublishDate = article.PublishDate
  250. if !isAppend {
  251. result = append(result, searchItem)
  252. }
  253. }
  254. }
  255. total = searchByMatch.Hits.TotalHits.Value
  256. }
  257. return
  258. }
  259. func GetResourceDataEsList(list []*SearchComprehensiveItem, user *models.WxUserItem) (items []*models.CygxResourceDataNewResp, err error) {
  260. var condition string
  261. var pars []interface{}
  262. uid := user.UserId
  263. titleHighlight := make(map[int]string)
  264. bodyHighlight := make(map[int][]string)
  265. yanXuanbodyHighlight := make(map[int][]string)
  266. mapItems := make(map[string]*models.CygxResourceDataNewResp)
  267. for _, v := range list {
  268. //预处理文章
  269. item := new(models.CygxResourceDataNewResp)
  270. item.SourceId = v.SourceId
  271. item.Source = v.Source
  272. item.PublishDate = utils.TimeRemoveHms2(v.PublishDate)
  273. item.BodyHighlight = v.Body
  274. item.TitleHighlight = v.Title
  275. titleHighlight[v.SourceId] = v.Title
  276. bodyHighlight[v.SourceId] = v.Body
  277. mapItems[fmt.Sprint(v.Source, v.SourceId)] = item
  278. }
  279. var articleIds []int //报告
  280. var activityIds []int //活动
  281. var yanxuanSpecialIds []int // 研选专栏
  282. var yanxuanspecialauthorIds []int //研选作者ID
  283. //Source string `description:"资源类型 报告 :article 、图表 :newchart、微路演 :roadshow、活动 :activity、活动视频:activityvideo、活动音频:activityvoice、专项调研活动:activityspecial 、 本周研究汇总: researchsummary 、 上周纪要汇总 :minutessummary 、晨会精华 :meetingreviewchapt 、 产品内测:productinterior
  284. for _, v := range list {
  285. if v.Source == "article" {
  286. articleIds = append(articleIds, v.SourceId)
  287. } else if v.Source == "activity" {
  288. activityIds = append(activityIds, v.SourceId)
  289. } else if v.Source == utils.CYGX_OBJ_YANXUANSPECIAL {
  290. yanxuanSpecialIds = append(yanxuanSpecialIds, v.SourceId)
  291. } else if v.Source == utils.CYGX_OBJ_YANXUANSPECIAL_AUTHOR {
  292. yanxuanspecialauthorIds = append(yanxuanspecialauthorIds, v.SourceId)
  293. }
  294. }
  295. detail, e := models.GetConfigByCode("city_img_url")
  296. if e != nil {
  297. err = errors.New("GetResourceDataList, Err: " + e.Error())
  298. return
  299. }
  300. detailChart, e := models.GetConfigByCode("chart_img_url")
  301. if e != nil {
  302. err = errors.New("GetResourceDataList, Err: " + e.Error())
  303. return
  304. }
  305. addressList := strings.Split(detail.ConfigValue, "{|}")
  306. mapAddress := make(map[string]string)
  307. chartList := strings.Split(detailChart.ConfigValue, "{|}")
  308. mapChart := make(map[string]string)
  309. var cityName string
  310. var chartName string
  311. var imgUrl string
  312. var imgUrlChart string
  313. for _, v := range addressList {
  314. vslice := strings.Split(v, "_")
  315. cityName = vslice[0]
  316. imgUrl = vslice[len(vslice)-1]
  317. mapAddress[cityName] = imgUrl
  318. }
  319. for _, v := range chartList {
  320. vslice := strings.Split(v, "_")
  321. chartName = vslice[0]
  322. imgUrlChart = vslice[len(vslice)-1]
  323. mapChart[chartName] = imgUrlChart
  324. }
  325. var imgUrlResp string
  326. //处理文章
  327. if len(articleIds) > 0 {
  328. pars = make([]interface{}, 0)
  329. condition = ` AND a.article_id IN (` + utils.GetOrmInReplace(len(articleIds)) + `)`
  330. pars = append(pars, articleIds)
  331. articleList, e := models.GetHomeListPublic(condition, pars, 0, len(articleIds))
  332. if e != nil {
  333. err = errors.New("GetResourceDataList, Err: " + e.Error())
  334. return
  335. }
  336. articleList, e = HandleArticleCategoryImg(articleList, user)
  337. if e != nil {
  338. err = errors.New("HandleArticleCategoryImg, Err: " + e.Error())
  339. return
  340. }
  341. for _, v := range articleList {
  342. v.Body = ""
  343. if titleHighlight[v.ArticleId] != "" {
  344. v.Title = titleHighlight[v.ArticleId]
  345. }
  346. if len(bodyHighlight[v.ArticleId]) > 0 {
  347. v.Abstract = ""
  348. v.Annotation = ""
  349. v.BodyHighlight = bodyHighlight[v.ArticleId]
  350. } else {
  351. v.BodyHighlight = make([]string, 0)
  352. }
  353. mapItems[fmt.Sprint("article", v.ArticleId)].Article = v
  354. }
  355. }
  356. //处理活动
  357. if len(activityIds) > 0 {
  358. for _, vss := range activityIds {
  359. imgUrlResp += strconv.Itoa(vss) + ","
  360. }
  361. pars = make([]interface{}, 0)
  362. condition = ` AND art.activity_id IN (` + utils.GetOrmInReplace(len(activityIds)) + `) `
  363. pars = append(pars, activityIds)
  364. activityList, e := models.GetActivityListNew(condition, pars, uid, 0, len(activityIds), 0, 0, "")
  365. if e != nil {
  366. err = errors.New("GetResourceDataList, Err: " + e.Error())
  367. return
  368. }
  369. var activityListRersp []*models.ActivityListResp
  370. //for _, v := range activityList {
  371. // activityListRersp = append(activityListRersp, ActivityButtonShow(v, user, make([]string, 0)))
  372. //}
  373. //var pArr []string
  374. activityListRersp = ActivityArrButtonShow(activityList, user, make([]string, 0))
  375. //处理不同的报名方式按钮回显
  376. mapActivitySignup, e := GetActivitySignupResp(activityIds, user)
  377. if e != nil {
  378. err = errors.New("GetActivitySignupResp, Err: " + e.Error())
  379. return
  380. }
  381. for _, v := range activityListRersp {
  382. if v == nil {
  383. continue
  384. }
  385. if v.ActivityType == 0 {
  386. if mapAddress[v.City] != "" {
  387. imgUrlResp = mapAddress[v.City]
  388. } else {
  389. imgUrlResp = mapAddress["其它"]
  390. }
  391. } else {
  392. if mapChart[v.ChartPermissionName] != "" {
  393. imgUrlResp = mapChart[v.ChartPermissionName]
  394. }
  395. }
  396. v.ImgUrl = imgUrlResp
  397. v.SourceType = mapActivitySignup[v.ActivityId]
  398. mapItems[fmt.Sprint("activity", v.ActivityId)].Activity = v
  399. }
  400. }
  401. //处理研选专栏
  402. lenyanxuanSpecialIds := len(yanxuanSpecialIds)
  403. if lenyanxuanSpecialIds > 0 {
  404. pars = make([]interface{}, 0)
  405. condition = ` AND a.id IN (` + utils.GetOrmInReplace(lenyanxuanSpecialIds) + `) `
  406. pars = append(pars, yanxuanSpecialIds)
  407. listyanxuanSpecial, e := models.GetYanxuanSpecialList(user.UserId, condition, pars, 0, 0)
  408. if e != nil {
  409. err = errors.New("GetYanxuanSpecialList, Err: " + e.Error())
  410. return
  411. }
  412. yanxuanSpecialPv := GetYanxuanSpecialRecordByYanxuanSpecialId(yanxuanSpecialIds) // 专栏Pv
  413. for _, v := range listyanxuanSpecial {
  414. v.PublishTime = utils.TimeRemoveHms2(v.PublishTime)
  415. v.Annotation, _ = GetReportContentTextSubNew(v.Content)
  416. v.Pv = yanxuanSpecialPv[v.Id]
  417. if len(yanXuanbodyHighlight[v.Id]) > 0 {
  418. v.BodyHighlight = yanXuanbodyHighlight[v.Id]
  419. } else {
  420. v.BodyHighlight = append(v.BodyHighlight, v.Annotation)
  421. }
  422. v.Annotation = "" //强制置空,兼容前端优先级
  423. v.LabelKeywordImgLink = utils.LABEL_ICO_4
  424. mapItems[fmt.Sprint(utils.CYGX_OBJ_YANXUANSPECIAL, v.Id)].YanxuanSpecial = v
  425. }
  426. }
  427. //处理研选专栏作者
  428. lenyanxuanspecialauthorIds := len(yanxuanspecialauthorIds)
  429. if lenyanxuanspecialauthorIds > 0 {
  430. pars = make([]interface{}, 0)
  431. condition = ` AND a.id IN (` + utils.GetOrmInReplace(lenyanxuanspecialauthorIds) + `) `
  432. condition += ` ORDER BY latest_publish_time DESC `
  433. pars = append(pars, yanxuanspecialauthorIds)
  434. listAuthor, e := models.GetYanxuanSpecialAuthorList("", condition, pars, 0, lenyanxuanspecialauthorIds)
  435. if e != nil {
  436. err = errors.New("GetYanxuanSpecialAuthorList, Err: " + e.Error())
  437. return
  438. }
  439. for _, v := range listAuthor {
  440. v.LatestPublishDate = v.LatestPublishTime.Format(utils.FormatDate)
  441. v.LabelKeywordImgLink = utils.LABEL_ICO_9
  442. mapItems[fmt.Sprint(utils.CYGX_OBJ_YANXUANSPECIAL_AUTHOR, v.Id)].YanxuanSpecialAuthor = v
  443. }
  444. }
  445. for _, vList := range list {
  446. for _, v := range mapItems {
  447. //如果这些类型都为空,那么就不合并
  448. if v.Article == nil && v.Activity == nil && v.YanxuanSpecial == nil && v.YanxuanSpecialAuthor == nil {
  449. continue
  450. }
  451. if v.SourceId == vList.SourceId && v.Source == vList.Source {
  452. items = append(items, v)
  453. }
  454. }
  455. }
  456. return
  457. }
  458. func SqlComprehensiveSearch(user *models.WxUserItem, keyWord string, startSize, pageSize int) (result []*SearchComprehensiveItem, total int, err error) {
  459. yanxuanActivityIds := GetYanxuanActivityIds(user, "") // 获取所有的研选活动ID
  460. yanxuanArticleIds := GetYanxuanArticleIds() //获取所有研选文章ID
  461. yanxuanArticleIds = append(yanxuanArticleIds, 0)
  462. yanxuanActivityIds = append(yanxuanActivityIds, 0)
  463. var yanxuanArticleIdsStr []string
  464. var yanxuanActivityIdsStr []string
  465. for _, v := range yanxuanArticleIds {
  466. yanxuanArticleIdsStr = append(yanxuanArticleIdsStr, strconv.Itoa(v))
  467. }
  468. for _, v := range yanxuanActivityIds {
  469. yanxuanActivityIdsStr = append(yanxuanActivityIdsStr, strconv.Itoa(v))
  470. }
  471. //yanxuanspecialIds = append(yanxuanspecialIds, 0)
  472. condition := " AND source IN ('article','activity','yanxuanspecial') " // 只有研选的文章、研选的活动、研选的专栏这三种
  473. condition += ` AND IF ( source = 'article' , source_id IN (` + strings.Join(yanxuanArticleIdsStr, ",") + `) ,1=1 ) `
  474. //pars = append(pars, yanxuanArticleIds)
  475. condition += ` AND IF ( source = 'activity' , source_id IN (` + strings.Join(yanxuanActivityIdsStr, ",") + `) ,1=1 ) `
  476. //pars = append(pars, yanxuanActivityIds)
  477. keyWord = "%" + keyWord + "%"
  478. var conditionTitle string
  479. var parsTitle []interface{}
  480. conditionTitle = " AND search_title LIKE ? " + condition
  481. parsTitle = append(parsTitle, keyWord)
  482. var conditionContentYxAuthor string // 研选专栏作者搜索条件
  483. var parsContentYxAuthor []interface{}
  484. conditionContentYxAuthor = " AND (special_name LIKE ? OR nick_name LIKE ? ) "
  485. parsContentYxAuthor = append(parsContentYxAuthor, keyWord, keyWord)
  486. totalTitle, e := models.GetResourceDataAndYanxuanSpecialAuthorCount(conditionTitle, parsTitle, conditionContentYxAuthor, parsContentYxAuthor)
  487. //totalTitle, e := models.GetResourceDataCount(conditionTitle, parsTitle)
  488. if e != nil {
  489. err = errors.New("GetResourceDataAndYanxuanSpecialAuthorCount, Err: " + e.Error())
  490. return
  491. }
  492. var conditionContent string
  493. var parsContent []interface{}
  494. conditionContent = " AND search_content LIKE ? AND search_title NOT LIKE ? " + condition
  495. parsContent = append(parsContent, keyWord, keyWord)
  496. totalContent, e := models.GetResourceDataCount(conditionContent, parsContent)
  497. if e != nil {
  498. err = errors.New("AddCygxArticleViewRecord, Err: " + e.Error())
  499. return
  500. }
  501. var searchTotal int
  502. searchTotal = (startSize/pageSize + 1) * pageSize
  503. var list []*models.CygxResourceData
  504. fmt.Println("totalTitle", totalTitle)
  505. fmt.Println("totalContent", totalContent)
  506. fmt.Println(searchTotal)
  507. if totalTitle >= searchTotal {
  508. fmt.Println("1")
  509. //全部都是标题搜索
  510. list, e = models.GetResourceDataAndYanxuanSpecialAuthorListCondition(conditionTitle, parsTitle, conditionContentYxAuthor, parsContentYxAuthor, startSize, pageSize)
  511. if e != nil && e.Error() != utils.ErrNoRow() {
  512. err = errors.New("GetResourceDataListCondition, Err: " + e.Error())
  513. return
  514. }
  515. } else if totalTitle <= searchTotal-pageSize {
  516. fmt.Println("2")
  517. //全部都是内容搜索
  518. startSize = startSize - totalTitle
  519. list, e = models.GetResourceDataListCondition(conditionContent, parsContent, startSize, pageSize)
  520. if e != nil && e.Error() != utils.ErrNoRow() {
  521. err = errors.New("GetResourceDataListCondition, Err: " + e.Error())
  522. return
  523. }
  524. } else {
  525. fmt.Println("3")
  526. //一半标题搜索,一半内容搜索
  527. //list, e = models.GetResourceDataListCondition(conditionTitle, parsTitle, startSize, pageSize)
  528. list, e = models.GetResourceDataAndYanxuanSpecialAuthorListCondition(conditionTitle, parsTitle, conditionContentYxAuthor, parsContentYxAuthor, startSize, pageSize)
  529. if e != nil && e.Error() != utils.ErrNoRow() {
  530. err = errors.New("GetResourceDataAndYanxuanSpecialAuthorListCondition, Err: " + e.Error())
  531. return
  532. }
  533. listContent, e := models.GetResourceDataListCondition(conditionContent, parsContent, 0, pageSize-totalContent%pageSize)
  534. if e != nil && e.Error() != utils.ErrNoRow() {
  535. err = errors.New("GetResourceDataListCondition, Err: " + e.Error())
  536. return
  537. }
  538. for _, v := range listContent {
  539. list = append(list, v)
  540. }
  541. }
  542. for _, v := range list {
  543. item := new(SearchComprehensiveItem)
  544. item.SourceId = v.SourceId
  545. item.Source = v.Source
  546. result = append(result, item)
  547. }
  548. total = totalTitle + totalContent
  549. return
  550. }
  551. // Es研选专栏
  552. func EsAddYanxuanSpecial(sourceId int) {
  553. var err error
  554. defer func() {
  555. if err != nil {
  556. fmt.Println("err:", err)
  557. go utils.SendAlarmMsg(fmt.Sprint("更新研选专栏失败sourceId: ", sourceId), 2)
  558. }
  559. }()
  560. detail, e := models.GetYanxuanSpecialItemById(sourceId)
  561. if e != nil {
  562. err = errors.New("GetArticleInfoOtherByArticleId" + e.Error())
  563. return
  564. }
  565. content := html.UnescapeString(detail.Content)
  566. doc, e := goquery.NewDocumentFromReader(strings.NewReader(content))
  567. if e != nil {
  568. err = errors.New("goquery.NewDocumentFromReader" + e.Error())
  569. return
  570. }
  571. bodyText := doc.Text()
  572. item := new(ElasticComprehensiveDetail)
  573. item.SourceId = detail.Id
  574. item.Source = utils.CYGX_OBJ_YANXUANSPECIAL
  575. item.Title = detail.Title
  576. item.PublishDate = detail.PublishTime
  577. item.BodyText = bodyText
  578. item.Abstract = bodyText
  579. item.IndustryName = detail.IndustryTags
  580. item.SubjectNames = detail.CompanyTags + detail.Tags
  581. if detail.Status == 3 {
  582. EsAddOrEditComprehensiveData(item) //如果发布了就新增
  583. } else {
  584. EsDeleteComprehensiveData(item) // 没有发布就删除
  585. }
  586. return
  587. }
  588. // 新增和修改数据
  589. func EsAddOrEditComprehensiveData(item *ElasticComprehensiveDetail) (err error) {
  590. indexName := utils.IndexNameComprehensive
  591. //return
  592. defer func() {
  593. if err != nil {
  594. fmt.Println(err, item.SourceId)
  595. //go utils.SendAlarmMsg("更新综合页面数据Es失败"+err.Error()+fmt.Sprint(item), 2)
  596. }
  597. }()
  598. client := utils.Client
  599. mustMap := make([]interface{}, 0)
  600. mustMap = append(mustMap, map[string]interface{}{
  601. "term": map[string]interface{}{
  602. "SourceId": item.SourceId,
  603. },
  604. })
  605. mustMap = append(mustMap, map[string]interface{}{
  606. "term": map[string]interface{}{
  607. "Source": item.Source,
  608. },
  609. })
  610. queryMap := map[string]interface{}{
  611. "query": map[string]interface{}{
  612. "bool": map[string]interface{}{
  613. "must": mustMap,
  614. },
  615. },
  616. }
  617. requestTotalHits := client.Count(indexName).BodyJson(queryMap)
  618. total, e := requestTotalHits.Do(context.Background())
  619. if e != nil {
  620. err = errors.New("requestTotalHits.Do(context.Background()), Err: " + e.Error())
  621. return
  622. }
  623. //return
  624. //根据来源以及ID ,判断内容是否存在,如果存在就新增,如果不存在就修改
  625. if total == 0 {
  626. resp, e := client.Index().Index(indexName).BodyJson(item).Do(context.Background())
  627. if e != nil {
  628. err = errors.New("client.Index().Index(indexName).BodyJson(item).Do(context.Background()), Err: " + e.Error())
  629. return
  630. }
  631. if resp.Status == 0 && resp.Result == "created" {
  632. //fmt.Println("新增成功")
  633. //err = nil
  634. return
  635. } else {
  636. //err = errors.New(fmt.Sprint(resp))
  637. err = errors.New(fmt.Sprint("articleId", item.SourceId))
  638. return
  639. }
  640. } else {
  641. //拼接需要改动的前置条件
  642. bool_query := elastic.NewBoolQuery()
  643. bool_query.Must(elastic.NewTermQuery("SourceId", item.SourceId))
  644. bool_query.Must(elastic.NewTermQuery("Source", item.Source))
  645. //设置需要改动的内容
  646. var script string
  647. script += fmt.Sprint("ctx._source['SubjectNames'] = '", item.SubjectNames, "';")
  648. script += fmt.Sprint("ctx._source['PublishDate'] = '", item.PublishDate, "';")
  649. script += fmt.Sprint("ctx._source['IsSummary'] = ", item.IsSummary, ";")
  650. script += fmt.Sprint("ctx._source['Abstract'] = '", item.Abstract, "';")
  651. script += fmt.Sprint("ctx._source['Title'] = '", item.Title, "';")
  652. script += fmt.Sprint("ctx._source['BodyText'] = '", item.BodyText, "';")
  653. script += fmt.Sprint("ctx._source['Annotation'] = '", item.Annotation, "';")
  654. script += fmt.Sprint("ctx._source['IndustryName'] = '", item.IndustryName, "'")
  655. _, e = client.UpdateByQuery(indexName).
  656. Query(bool_query).
  657. Script(elastic.NewScriptInline(script)).
  658. Refresh("true").
  659. Do(context.Background())
  660. if e != nil && e.Error() != "elastic: Error 400 (Bad Request): compile error [type=script_exception]" {
  661. //文本内容过长的时候,修改会报 400 的错误,暂时先不处理
  662. //fmt.Println("err", e.Error())
  663. ////err = e
  664. //err = errors.New("client.UpdateByQuery(indexName), Err: " + e.Error())
  665. return
  666. }
  667. }
  668. return
  669. }
  670. // 删除数据
  671. func EsDeleteComprehensiveData(item *ElasticComprehensiveDetail) (err error) {
  672. defer func() {
  673. if err != nil {
  674. fmt.Println(err)
  675. go utils.SendAlarmMsg("删除数据综合页面数据Es失败"+err.Error()+fmt.Sprint(item), 2)
  676. }
  677. }()
  678. indexName := utils.IndexNameComprehensive
  679. client := utils.Client
  680. //拼接需要删除的前置条件
  681. bool_query := elastic.NewBoolQuery()
  682. bool_query.Must(elastic.NewTermQuery("SourceId", item.SourceId))
  683. bool_query.Must(elastic.NewTermQuery("Source", item.Source))
  684. _, e := client.DeleteByQuery(indexName).
  685. Query(bool_query).
  686. Do(context.Background())
  687. if e != nil {
  688. err = errors.New(" client.DeleteByQuery(indexName), Err: " + e.Error())
  689. return
  690. }
  691. return
  692. }