123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818 |
- package services
- import (
- "context"
- "encoding/json"
- "fmt"
- "github.com/olivere/elastic/v7"
- "hongze/hongze_clpt/models"
- "hongze/hongze_clpt/utils"
- "strconv"
- "strings"
- )
- func RemoveDuplicatesAndEmpty(a []string) (ret []string) {
- a_len := len(a)
- for i := 0; i < a_len; i++ {
- if (i > 0 && a[i-1] == a[i]) || len(a[i]) == 0 {
- continue
- }
- ret = append(ret, a[i])
- }
- return
- }
- func GetArrSum(intArr []int) (sum int) {
- for _, val := range intArr {
-
- sum += val
- }
- return
- }
- func EsMultiMatchFunctionScoreQuerySort(indexName, keyWord string, startSize, pageSize, userId int, orderColumn string) (result []*models.SearchItem, total int64, err error) {
- client := utils.Client
- keyWordArr, err := GetIndustryMapNameSliceV3(keyWord)
- keyWordArr = RemoveDuplicatesAndEmpty(keyWordArr)
-
-
- n := 0
- keyWordLen := len(keyWordArr)
- if keyWordLen <= 0 {
- keyWordArr = append(keyWordArr, keyWord)
- keyWordLen = len(keyWordArr)
- }
-
- utils.FileLog.Info("SearchKeyWord:%s, userId:%s", keyWordArr, strconv.Itoa(userId))
-
- for _, v := range keyWordArr {
- if v != "" {
- matchArr := make([]elastic.Query, 0)
- boolquery := elastic.NewBoolQuery()
- bodyFunctionQuery := elastic.NewFunctionScoreQuery()
- bodyFunctionQuery2 := elastic.NewFunctionScoreQuery()
- bodyFunctionQuery3 := elastic.NewFunctionScoreQuery()
-
- multiMatch := elastic.NewMultiMatchQuery(v, "Title").Analyzer("ik_smart").Boost(100)
- bodyFunctionQuery.Query(multiMatch)
- matchArr = append(matchArr, bodyFunctionQuery)
- multiMatch = elastic.NewMultiMatchQuery(v, "BodyText").Analyzer("ik_smart").Boost(1)
- bodyFunctionQuery2.Query(multiMatch)
- matchArr = append(matchArr, bodyFunctionQuery2)
-
- bodyFunctionQuery3.Query(multiMatch)
- matchArr = append(matchArr, bodyFunctionQuery3)
- boolquery.Should(matchArr...)
-
-
-
-
- highlight := elastic.NewHighlight()
- highlight = highlight.PreTags("<font color='red'>").PostTags("</font>")
- highlight = highlight.Fields(elastic.NewHighlighterField("Title"), elastic.NewHighlighterField("BodyText"))
- request := client.Search(indexName).Highlight(highlight).Sort("PublishDate", false).From(0).Size(pageSize).Query(boolquery)
- if orderColumn == "Matching" {
- request = client.Search(indexName).Highlight(highlight).From(0).Size(pageSize).Query(boolquery)
- }
- searchByMatch, err := request.Do(context.Background())
- if err != nil {
- return nil, 0, err
- }
- if searchByMatch != nil {
- if searchByMatch.Hits != nil {
- for _, v := range searchByMatch.Hits.Hits {
- var isAppend bool
- articleJson, err := v.Source.MarshalJSON()
- if err != nil {
- return nil, 0, err
- }
- article := new(models.CygxArticleEs)
- err = json.Unmarshal(articleJson, &article)
- if err != nil {
- return nil, 0, err
- }
- searchItem := new(models.SearchItem)
- searchItem.ArticleId, _ = strconv.Atoi(v.Id)
- if len(v.Highlight["BodyText"]) > 0 {
- searchItem.Body = v.Highlight["BodyText"]
- } else {
- bodyRune := []rune(article.BodyText)
- bodyRuneLen := len(bodyRune)
- if bodyRuneLen > 100 {
- bodyRuneLen = 100
- }
- body := string(bodyRune[:bodyRuneLen])
- searchItem.Body = []string{body}
- }
- var title string
- if len(v.Highlight["Title"]) > 0 {
- title = v.Highlight["Title"][0]
- } else {
- title = article.Title
- }
- searchItem.Title = title
- searchItem.PublishDate = article.PublishDate
- searchItem.ExpertBackground = article.ExpertBackground
- searchItem.CategoryId = article.CategoryId
- for _, v_result := range result {
- if v_result.ArticleId == searchItem.ArticleId {
- isAppend = true
- }
- }
- if !isAppend {
- result = append(result, searchItem)
- }
- }
- }
-
- }
- }
- n++
- }
- total = int64(len(result))
- return
- }
- func EsMultiMatchFunctionScoreQueryTimeSort(indexName, keyWord string, startSize, pageSize, userId int) (result []*models.SearchItem, total int64, err error) {
- client := utils.Client
- keyWordArr, err := GetIndustryMapNameSliceV2(keyWord)
- keyWordArr = RemoveDuplicatesAndEmpty(keyWordArr)
- boolquery := elastic.NewBoolQuery()
- matchArr := make([]elastic.Query, 0)
-
- n := 0
- keyWordLen := len(keyWordArr)
- if keyWordLen <= 0 {
- keyWordArr = append(keyWordArr, keyWord)
- keyWordLen = len(keyWordArr)
- }
- utils.FileLog.Info("SearchKeyWord:%s, userId:%s", keyWordArr, strconv.Itoa(userId))
- for _, v := range keyWordArr {
- if v != "" {
- multiMatch := elastic.NewMultiMatchQuery(v, "Title", "BodyText")
- bodyFunctionQuery := elastic.NewFunctionScoreQuery()
- bodyFunctionQuery.Query(multiMatch)
- matchArr = append(matchArr, bodyFunctionQuery)
- }
- n++
- }
- boolquery.Should(matchArr...)
- highlight := elastic.NewHighlight()
- highlight = highlight.Fields(elastic.NewHighlighterField("Title"), elastic.NewHighlighterField("BodyText"))
- highlight = highlight.PreTags("<font color='red'>").PostTags("</font>")
- request := client.Search(indexName).Highlight(highlight).Sort("PublishDate", false).Size(pageSize).Query(boolquery)
- searchByMatch, err := request.Do(context.Background())
- if searchByMatch != nil {
- matchResult, _ := json.Marshal(searchByMatch)
- utils.FileLog.Info("%s", string(matchResult))
- fmt.Println(len(searchByMatch.Hits.Hits))
- if searchByMatch.Hits != nil {
- for _, v := range searchByMatch.Hits.Hits {
- articleJson, err := v.Source.MarshalJSON()
- utils.FileLog.Info("%s", string(articleJson))
- if err != nil {
- return nil, 0, err
- }
- article := new(models.CygxArticleEs)
- err = json.Unmarshal(articleJson, &article)
- if err != nil {
- return nil, 0, err
- }
- searchItem := new(models.SearchItem)
- searchItem.ArticleId, _ = strconv.Atoi(v.Id)
- if len(v.Highlight["BodyText"]) > 0 {
- searchItem.Body = v.Highlight["BodyText"]
- } else {
- bodyRune := []rune(article.BodyText)
- bodyRuneLen := len(bodyRune)
- if bodyRuneLen > 100 {
- bodyRuneLen = 100
- }
- body := string(bodyRune[:bodyRuneLen])
- searchItem.Body = []string{body}
- }
- var title string
- if len(v.Highlight["Title"]) > 0 {
- title = v.Highlight["Title"][0]
- } else {
- title = article.Title
- }
- searchItem.Title = title
- searchItem.PublishDate = article.PublishDate
- searchItem.ExpertBackground = article.ExpertBackground
- searchItem.CategoryId = article.CategoryId
- result = append(result, searchItem)
- }
- }
- total = searchByMatch.Hits.TotalHits.Value
- }
- return
- }
- func EsArticleSearch(keyWord string, startSize, pageSize int, orderColumn string, ikType int) (result []*models.SearchItem, total int64, err error) {
- indexName := utils.IndexName
- client := utils.Client
- keyWordArr, err := GetIndustryMapNameSliceV3(keyWord)
- keyWordArr = RemoveDuplicatesAndEmpty(keyWordArr)
- keyWordLen := len(keyWordArr)
- if keyWordLen <= 0 {
- keyWordArr = append(keyWordArr, keyWord)
- keyWordLen = len(keyWordArr)
- }
-
- if ikType == 2 && keyWordLen == 1 {
- return
- }
-
-
- mustMap := make([]interface{}, 0)
- shouldMap := make(map[string]interface{}, 0)
- shouldMapquery := make([]interface{}, 0)
- mustNotMap := make([]interface{}, 0)
- shouldNotMap := make(map[string]interface{}, 0)
- shouldNotMapquery := make([]interface{}, 0)
-
-
- var boost int
-
- for k, v := range keyWordArr {
- if k == 0 {
- boost = 2 * 1000
- } else {
- boost = 1
- }
-
- if k == 0 && ikType == 2 {
- if v != "" {
- shouldNotMapquery = append(shouldNotMapquery, map[string]interface{}{
- "function_score": map[string]interface{}{
- "query": map[string]interface{}{
- "multi_match": map[string]interface{}{
-
- "boost": boost,
- "fields": []interface{}{"Title"},
- "query": v,
- },
- },
- },
- })
- shouldNotMapquery = append(shouldNotMapquery, map[string]interface{}{
- "function_score": map[string]interface{}{
- "query": map[string]interface{}{
- "multi_match": map[string]interface{}{
- "boost": boost,
- "fields": []interface{}{"Abstract"},
- "query": v,
- },
- },
- },
- })
- shouldNotMapquery = append(shouldNotMapquery, map[string]interface{}{
- "function_score": map[string]interface{}{
- "query": map[string]interface{}{
- "multi_match": map[string]interface{}{
- "boost": boost,
- "fields": []interface{}{"Annotation"},
- "query": v,
- },
- },
- },
- })
- shouldNotMapquery = append(shouldNotMapquery, map[string]interface{}{
- "function_score": map[string]interface{}{
- "query": map[string]interface{}{
- "multi_match": map[string]interface{}{
-
- "boost": boost,
- "fields": []interface{}{"BodyText"},
- "query": v,
- },
- },
- },
- })
- }
- continue
- }
-
- if k > 0 && ikType == 1 {
- continue
- }
- if v != "" {
- shouldMapquery = append(shouldMapquery, map[string]interface{}{
- "function_score": map[string]interface{}{
- "query": map[string]interface{}{
- "multi_match": map[string]interface{}{
-
- "boost": boost,
- "fields": []interface{}{"Title"},
- "query": v,
- },
- },
- },
- })
- shouldMapquery = append(shouldMapquery, map[string]interface{}{
- "function_score": map[string]interface{}{
- "query": map[string]interface{}{
- "multi_match": map[string]interface{}{
- "boost": boost,
- "fields": []interface{}{"Abstract"},
- "query": v,
- },
- },
- },
- })
- shouldMapquery = append(shouldMapquery, map[string]interface{}{
- "function_score": map[string]interface{}{
- "query": map[string]interface{}{
- "multi_match": map[string]interface{}{
- "boost": boost,
- "fields": []interface{}{"Annotation"},
- "query": v,
- },
- },
- },
- })
- shouldMapquery = append(shouldMapquery, map[string]interface{}{
- "function_score": map[string]interface{}{
- "query": map[string]interface{}{
- "multi_match": map[string]interface{}{
-
- "boost": boost,
- "fields": []interface{}{"BodyText"},
- "query": v,
- },
- },
- },
- })
- }
- }
- shouldMap = map[string]interface{}{
- "should": shouldMapquery,
- }
- shouldNotMap = map[string]interface{}{
- "should": shouldNotMapquery,
- }
-
- sortMap := make([]interface{}, 0)
-
- sortMap = append(sortMap, map[string]interface{}{
- "PublishDate": map[string]interface{}{
- "order": "desc",
- },
- })
-
-
-
-
-
-
- highlightMap := make(map[string]interface{}, 0)
- highlightMap = map[string]interface{}{
- "fields": map[string]interface{}{
- "BodyText": map[string]interface{}{},
- "Title": map[string]interface{}{},
- "Abstract": map[string]interface{}{},
- "Annotation": map[string]interface{}{},
- },
-
- "post_tags": []interface{}{"</font>"},
- "pre_tags": []interface{}{"<font color='red'>"},
- }
- mustMap = append(mustMap, map[string]interface{}{
- "bool": shouldMap,
- })
- mustNotMap = append(mustNotMap, map[string]interface{}{
- "bool": shouldNotMap,
- })
- queryMap := map[string]interface{}{
- "query": map[string]interface{}{
- "bool": map[string]interface{}{
- "must": mustMap,
- },
- },
- }
-
- if ikType == 2 {
- queryMap = map[string]interface{}{
- "query": map[string]interface{}{
- "bool": map[string]interface{}{
- "must": mustMap,
- "must_not": mustNotMap,
- },
- },
- }
- }
- if orderColumn == "Matching" {
- queryMap["sort"] = sortMap
- }
- queryMap["from"] = startSize
- queryMap["size"] = pageSize
- queryMap["highlight"] = highlightMap
- jsonBytes, _ := json.Marshal(queryMap)
- fmt.Println(string(jsonBytes))
-
- request := client.Search(indexName).Source(queryMap)
- searchByMatch, err := request.Do(context.Background())
- if searchByMatch != nil {
- if searchByMatch.Hits != nil {
- for _, v := range searchByMatch.Hits.Hits {
- var isAppend bool
- articleJson, err := v.Source.MarshalJSON()
- if err != nil {
- return nil, 0, err
- }
- article := new(models.CygxArticleEs)
- err = json.Unmarshal(articleJson, &article)
- if err != nil {
- return nil, 0, err
- }
- searchItem := new(models.SearchItem)
- searchItem.ArticleId, _ = strconv.Atoi(v.Id)
- if len(v.Highlight["Annotation"]) > 0 {
- for _, vText := range v.Highlight["Annotation"] {
- searchItem.Body = append(searchItem.Body, vText)
- }
- }
- if len(v.Highlight["Abstract"]) > 0 {
- for _, vText := range v.Highlight["Abstract"] {
- searchItem.Body = append(searchItem.Body, vText)
- }
- }
- if len(v.Highlight["BodyText"]) > 0 {
- for _, vText := range v.Highlight["BodyText"] {
- searchItem.Body = append(searchItem.Body, vText)
- }
- }
- if len(searchItem.Body) == 0 {
- bodyRune := []rune(article.BodyText)
- bodyRuneLen := len(bodyRune)
- if bodyRuneLen > 100 {
- bodyRuneLen = 100
- }
- body := string(bodyRune[:bodyRuneLen])
- searchItem.Body = []string{body}
- }
-
-
-
-
-
-
-
-
-
-
-
- var title string
- if len(v.Highlight["Title"]) > 0 {
- title = v.Highlight["Title"][0]
- } else {
- title = article.Title
- }
- searchItem.Title = title
- searchItem.PublishDate = article.PublishDate
- searchItem.ExpertBackground = article.ExpertBackground
- searchItem.CategoryId = article.CategoryId
- for _, v_result := range result {
- if v_result.ArticleId == searchItem.ArticleId {
- isAppend = true
- }
- }
- if !isAppend {
- result = append(result, searchItem)
- }
- }
- }
- total = searchByMatch.Hits.TotalHits.Value
- }
- return
- }
- func EsArticleSearchBody(keyWord string, startSize, pageSize int, orderColumn string, searchType int) (result []*models.SearchItem, total int64, err error) {
- if keyWord == "" {
- return
- }
- indexName := utils.IndexName
- client := utils.Client
-
-
- mustMap := make([]interface{}, 0)
- shouldMap := make(map[string]interface{}, 0)
- shouldMapquery := make([]interface{}, 0)
- mustNotMap := make([]interface{}, 0)
- shouldNotMap := make(map[string]interface{}, 0)
- shouldNotMapquery := make([]interface{}, 0)
-
-
- var boost int
-
- if searchType == 1 {
- shouldMapquery = append(shouldMapquery, map[string]interface{}{
- "function_score": map[string]interface{}{
- "query": map[string]interface{}{
- "multi_match": map[string]interface{}{
- "boost": boost,
- "fields": []interface{}{"Title"},
- "query": keyWord,
- },
- },
- },
- })
- shouldMapquery = append(shouldMapquery, map[string]interface{}{
- "function_score": map[string]interface{}{
- "query": map[string]interface{}{
- "multi_match": map[string]interface{}{
- "boost": boost,
- "fields": []interface{}{"Abstract"},
- "query": keyWord,
- },
- },
- },
- })
- shouldMapquery = append(shouldMapquery, map[string]interface{}{
- "function_score": map[string]interface{}{
- "query": map[string]interface{}{
- "multi_match": map[string]interface{}{
- "boost": boost,
- "fields": []interface{}{"Annotation"},
- "query": keyWord,
- },
- },
- },
- })
- }
-
- if searchType == 2 {
- shouldNotMapquery = append(shouldNotMapquery, map[string]interface{}{
- "function_score": map[string]interface{}{
- "query": map[string]interface{}{
- "multi_match": map[string]interface{}{
- "boost": boost,
- "fields": []interface{}{"Title"},
- "query": keyWord,
- },
- },
- },
- })
- shouldNotMapquery = append(shouldNotMapquery, map[string]interface{}{
- "function_score": map[string]interface{}{
- "query": map[string]interface{}{
- "multi_match": map[string]interface{}{
- "boost": boost,
- "fields": []interface{}{"Abstract"},
- "query": keyWord,
- },
- },
- },
- })
- shouldNotMapquery = append(shouldNotMapquery, map[string]interface{}{
- "function_score": map[string]interface{}{
- "query": map[string]interface{}{
- "multi_match": map[string]interface{}{
- "boost": boost,
- "fields": []interface{}{"Annotation"},
- "query": keyWord,
- },
- },
- },
- })
-
-
-
-
-
-
-
-
-
-
-
-
- shouldMapquery = append(shouldMapquery, map[string]interface{}{
- "function_score": map[string]interface{}{
- "query": map[string]interface{}{
- "multi_match": map[string]interface{}{
-
- "boost": boost,
- "fields": []interface{}{"BodyText"},
- "query": keyWord,
- },
- },
- },
- })
- }
- shouldMap = map[string]interface{}{
- "should": shouldMapquery,
- }
- shouldNotMap = map[string]interface{}{
- "should": shouldNotMapquery,
- }
-
- sortMap := make([]interface{}, 0)
-
- sortMap = append(sortMap, map[string]interface{}{
- "PublishDate": map[string]interface{}{
- "order": "desc",
- },
- })
-
- highlightMap := make(map[string]interface{}, 0)
- highlightMap = map[string]interface{}{
- "fields": map[string]interface{}{
- "BodyText": map[string]interface{}{},
- "Title": map[string]interface{}{},
- "Abstract": map[string]interface{}{},
- "Annotation": map[string]interface{}{},
- },
-
- "post_tags": []interface{}{"</font>"},
- "pre_tags": []interface{}{"<font color='red'>"},
- }
- mustMap = append(mustMap, map[string]interface{}{
- "bool": shouldMap,
- })
- mustNotMap = append(mustNotMap, map[string]interface{}{
- "bool": shouldNotMap,
- })
- queryMap := map[string]interface{}{
- "query": map[string]interface{}{
- "bool": map[string]interface{}{
- "must": mustMap,
- },
- },
- }
-
- if searchType == 2 {
- queryMap = map[string]interface{}{
- "query": map[string]interface{}{
- "bool": map[string]interface{}{
- "must": mustMap,
- "must_not": mustNotMap,
- },
- },
- }
- }
- if orderColumn == "Matching" {
- queryMap["sort"] = sortMap
- }
- queryMap["from"] = startSize
- queryMap["size"] = pageSize
- queryMap["highlight"] = highlightMap
- jsonBytes, _ := json.Marshal(queryMap)
- fmt.Println(string(jsonBytes))
-
- request := client.Search(indexName).Source(queryMap)
- searchByMatch, err := request.Do(context.Background())
- if searchByMatch != nil {
- if searchByMatch.Hits != nil {
- for _, v := range searchByMatch.Hits.Hits {
- var isAppend bool
- articleJson, err := v.Source.MarshalJSON()
- if err != nil {
- return nil, 0, err
- }
- article := new(models.CygxArticleEs)
- err = json.Unmarshal(articleJson, &article)
- if err != nil {
- return nil, 0, err
- }
- searchItem := new(models.SearchItem)
- searchItem.ArticleId, _ = strconv.Atoi(v.Id)
- if len(v.Highlight["Annotation"]) > 0 {
- for _, vText := range v.Highlight["Annotation"] {
- searchItem.Body = append(searchItem.Body, vText)
- }
- }
- if len(v.Highlight["Abstract"]) > 0 {
- for _, vText := range v.Highlight["Abstract"] {
- searchItem.Body = append(searchItem.Body, vText)
- }
- }
- if len(v.Highlight["BodyText"]) > 0 {
- for _, vText := range v.Highlight["BodyText"] {
- searchItem.Body = append(searchItem.Body, vText)
- }
- }
- if len(searchItem.Body) == 0 {
- bodyRune := []rune(article.BodyText)
- bodyRuneLen := len(bodyRune)
- if bodyRuneLen > 100 {
- bodyRuneLen = 100
- }
- body := string(bodyRune[:bodyRuneLen])
- searchItem.Body = []string{body}
- }
-
-
-
-
-
-
-
-
-
-
-
- var title string
- if len(v.Highlight["Title"]) > 0 {
- title = v.Highlight["Title"][0]
- } else {
- title = article.Title
- }
- searchItem.Title = title
- searchItem.PublishDate = article.PublishDate
- searchItem.ExpertBackground = article.ExpertBackground
- searchItem.CategoryId = article.CategoryId
- for _, v_result := range result {
- if v_result.ArticleId == searchItem.ArticleId {
- isAppend = true
- }
- }
- if !isAppend {
- result = append(result, searchItem)
- }
- }
- }
- total = searchByMatch.Hits.TotalHits.Value
- }
- return
- }
- func KeyWordArrSqlRegexpAll(a []string) (ret string) {
- a_len := len(a)
- for i := 0; i < a_len; i++ {
- ret += a[i] + "|"
- }
- ret = strings.TrimRight(ret, "|")
-
- return
- }
- func KeyWordArrSqlRegexp(a []string) (ret string) {
- a_len := len(a)
- for i := 0; i < a_len; i++ {
- if i == 0 {
- continue
- }
- ret += a[i] + "|"
- }
- ret = strings.TrimRight(ret, "|")
- return
- }
|