package services import ( "context" "encoding/json" "fmt" "github.com/olivere/elastic/v7" "hongze/hongze_clpt/models" "hongze/hongze_clpt/utils" "strconv" "strings" ) //func NewClient() (client *elastic.Client, err error) { // //errorlog := log.New(os.Stdout, "APP", log.LstdFlags) // //file := "" // //if utils.RunMode == "release" { // // //file = `/data/rdlucklog/hongze_cygx/eslog.log` // // file = `./rdlucklog/eslog.log` // //} else { // // file = `./rdlucklog/eslog.log` // //} // //logFile, _ := os.OpenFile(file, os.O_RDWR|os.O_CREATE|os.O_APPEND, 0766) // //client, err = elastic.NewClient( // // elastic.SetURL(ES_URL), // // elastic.SetBasicAuth(ES_USERNAME, ES_PASSWORD), // // elastic.SetTraceLog(log.New(logFile, "ES-TRACE: ", 0)), // // elastic.SetSniff(false), elastic.SetErrorLog(errorlog)) // client, err = elastic.NewClient( // elastic.SetURL(ES_URL), // elastic.SetBasicAuth(ES_USERNAME, ES_PASSWORD), // elastic.SetSniff(false)) // return //} func RemoveDuplicatesAndEmpty(a []string) (ret []string) { a_len := len(a) for i := 0; i < a_len; i++ { if (i > 0 && a[i-1] == a[i]) || len(a[i]) == 0 { continue } ret = append(ret, a[i]) } return } func GetArrSum(intArr []int) (sum int) { for _, val := range intArr { //累计求和 sum += val } return } func EsMultiMatchFunctionScoreQuerySort(indexName, keyWord string, startSize, pageSize, userId int, orderColumn string) (result []*models.SearchItem, total int64, err error) { client := utils.Client keyWordArr, err := GetIndustryMapNameSliceV3(keyWord) keyWordArr = RemoveDuplicatesAndEmpty(keyWordArr) //artidArr := make([]elastic.Query, 0) //matchArr := make([]elastic.Query, 0) n := 0 keyWordLen := len(keyWordArr) if keyWordLen <= 0 { keyWordArr = append(keyWordArr, keyWord) keyWordLen = len(keyWordArr) } // @Param OrderColumn query int true "排序字段，Comprehensive综合，Matching匹配度，PublishDate 发布时间 " utils.FileLog.Info("SearchKeyWord:%s, userId:%s", keyWordArr, strconv.Itoa(userId)) //keyWordWeight := GetWeight(keyWordLen) for _, v := range keyWordArr { if v != "" { matchArr := make([]elastic.Query, 0) boolquery := elastic.NewBoolQuery() bodyFunctionQuery := elastic.NewFunctionScoreQuery() bodyFunctionQuery2 := elastic.NewFunctionScoreQuery() bodyFunctionQuery3 := elastic.NewFunctionScoreQuery() //multiMatch := elastic.NewMultiMatchQuery(v, "Title", "BodyText").Analyzer("ik_smart") multiMatch := elastic.NewMultiMatchQuery(v, "Title").Analyzer("ik_smart").Boost(100) bodyFunctionQuery.Query(multiMatch) matchArr = append(matchArr, bodyFunctionQuery) multiMatch = elastic.NewMultiMatchQuery(v, "BodyText").Analyzer("ik_smart").Boost(1) bodyFunctionQuery2.Query(multiMatch) matchArr = append(matchArr, bodyFunctionQuery2) //multiMatch = elastic.NewMultiMatchQuery(1, "IsSummary") bodyFunctionQuery3.Query(multiMatch) matchArr = append(matchArr, bodyFunctionQuery3) boolquery.Should(matchArr...) //multiMatch = elastic.NewMultiMatchQuery(v, "BodyText").Analyzer("ik_smart") //bodyFunctionQuery.Query(multiMatch) //matchArr = append(matchArr, bodyFunctionQuery) //boolquery.Should(matchArr...) highlight := elastic.NewHighlight() highlight = highlight.PreTags("").PostTags("") highlight = highlight.Fields(elastic.NewHighlighterField("Title"), elastic.NewHighlighterField("BodyText")) request := client.Search(indexName).Highlight(highlight).Sort("PublishDate", false).From(0).Size(pageSize).Query(boolquery) if orderColumn == "Matching" { request = client.Search(indexName).Highlight(highlight).From(0).Size(pageSize).Query(boolquery) } searchByMatch, err := request.Do(context.Background()) if err != nil { return nil, 0, err } if searchByMatch != nil { if searchByMatch.Hits != nil { for _, v := range searchByMatch.Hits.Hits { var isAppend bool articleJson, err := v.Source.MarshalJSON() if err != nil { return nil, 0, err } article := new(models.CygxArticleEs) err = json.Unmarshal(articleJson, &article) if err != nil { return nil, 0, err } searchItem := new(models.SearchItem) searchItem.ArticleId, _ = strconv.Atoi(v.Id) if len(v.Highlight["BodyText"]) > 0 { searchItem.Body = v.Highlight["BodyText"] } else { bodyRune := []rune(article.BodyText) bodyRuneLen := len(bodyRune) if bodyRuneLen > 100 { bodyRuneLen = 100 } body := string(bodyRune[:bodyRuneLen]) searchItem.Body = []string{body} } var title string if len(v.Highlight["Title"]) > 0 { title = v.Highlight["Title"][0] } else { title = article.Title } searchItem.Title = title searchItem.PublishDate = article.PublishDate searchItem.ExpertBackground = article.ExpertBackground searchItem.CategoryId = article.CategoryId for _, v_result := range result { if v_result.ArticleId == searchItem.ArticleId { isAppend = true } } if !isAppend { result = append(result, searchItem) } } } //total += searchByMatch.Hits.TotalHits.Value } } n++ } total = int64(len(result)) return } func EsMultiMatchFunctionScoreQueryTimeSort(indexName, keyWord string, startSize, pageSize, userId int) (result []*models.SearchItem, total int64, err error) { client := utils.Client keyWordArr, err := GetIndustryMapNameSliceV2(keyWord) keyWordArr = RemoveDuplicatesAndEmpty(keyWordArr) boolquery := elastic.NewBoolQuery() matchArr := make([]elastic.Query, 0) //matchArr2 := make([]elastic.Query, 0) n := 0 keyWordLen := len(keyWordArr) if keyWordLen <= 0 { keyWordArr = append(keyWordArr, keyWord) keyWordLen = len(keyWordArr) } utils.FileLog.Info("SearchKeyWord:%s, userId:%s", keyWordArr, strconv.Itoa(userId)) for _, v := range keyWordArr { if v != "" { multiMatch := elastic.NewMultiMatchQuery(v, "Title", "BodyText") bodyFunctionQuery := elastic.NewFunctionScoreQuery() bodyFunctionQuery.Query(multiMatch) matchArr = append(matchArr, bodyFunctionQuery) } n++ } boolquery.Should(matchArr...) highlight := elastic.NewHighlight() highlight = highlight.Fields(elastic.NewHighlighterField("Title"), elastic.NewHighlighterField("BodyText")) highlight = highlight.PreTags("").PostTags("") request := client.Search(indexName).Highlight(highlight).Sort("PublishDate", false).Size(pageSize).Query(boolquery) searchByMatch, err := request.Do(context.Background()) if searchByMatch != nil { matchResult, _ := json.Marshal(searchByMatch) utils.FileLog.Info("%s", string(matchResult)) fmt.Println(len(searchByMatch.Hits.Hits)) if searchByMatch.Hits != nil { for _, v := range searchByMatch.Hits.Hits { articleJson, err := v.Source.MarshalJSON() utils.FileLog.Info("%s", string(articleJson)) if err != nil { return nil, 0, err } article := new(models.CygxArticleEs) err = json.Unmarshal(articleJson, &article) if err != nil { return nil, 0, err } searchItem := new(models.SearchItem) searchItem.ArticleId, _ = strconv.Atoi(v.Id) if len(v.Highlight["BodyText"]) > 0 { searchItem.Body = v.Highlight["BodyText"] } else { bodyRune := []rune(article.BodyText) bodyRuneLen := len(bodyRune) if bodyRuneLen > 100 { bodyRuneLen = 100 } body := string(bodyRune[:bodyRuneLen]) searchItem.Body = []string{body} } var title string if len(v.Highlight["Title"]) > 0 { title = v.Highlight["Title"][0] } else { title = article.Title } searchItem.Title = title searchItem.PublishDate = article.PublishDate searchItem.ExpertBackground = article.ExpertBackground searchItem.CategoryId = article.CategoryId result = append(result, searchItem) } } total = searchByMatch.Hits.TotalHits.Value } return } func EsArticleSearch(keyWord string, startSize, pageSize int, orderColumn string, ikType int) (result []*models.SearchItem, total int64, err error) { indexName := utils.IndexName client := utils.Client keyWordArr, err := GetIndustryMapNameSliceV3(keyWord) keyWordArr = RemoveDuplicatesAndEmpty(keyWordArr) keyWordLen := len(keyWordArr) if keyWordLen <= 0 { keyWordArr = append(keyWordArr, keyWord) keyWordLen = len(keyWordArr) } //如果没有联想词，而且查询的还是联想词就返回 if ikType == 2 && keyWordLen == 1 { return } //Es 的高级查询有自定义排序文档一时半会儿撸不懂，先做多次查询手动过滤 2023.2.2 //ikType 查询方式，0：查所有、 1:查询键入词、 2：查询除了查询键入词之外的联想词 mustMap := make([]interface{}, 0) shouldMap := make(map[string]interface{}, 0) shouldMapquery := make([]interface{}, 0) mustNotMap := make([]interface{}, 0) shouldNotMap := make(map[string]interface{}, 0) shouldNotMapquery := make([]interface{}, 0) // @Param OrderColumn query int true "排序字段，Comprehensive综合，Matching匹配度，PublishDate 发布时间 " //keyWordWeight := GetWeight(keyWordLen) var boost int //lenkeyWordArr := len(keyWordArr) for k, v := range keyWordArr { if k == 0 { boost = 2 * 1000 } else { boost = 1 } //如果是 2：查询除了查询键入词之外的联想词 if k == 0 && ikType == 2 { if v != "" { shouldNotMapquery = append(shouldNotMapquery, map[string]interface{}{ "function_score": map[string]interface{}{ "query": map[string]interface{}{ "multi_match": map[string]interface{}{ //"boost": (lenkeyWordArr - k) * boost, //给查询的值赋予权重 "boost": boost, //给查询的值赋予权重 "fields": []interface{}{"Title"}, "query": v, }, }, }, }) shouldNotMapquery = append(shouldNotMapquery, map[string]interface{}{ "function_score": map[string]interface{}{ "query": map[string]interface{}{ "multi_match": map[string]interface{}{ "boost": boost, //给查询的值赋予权重 "fields": []interface{}{"Abstract"}, "query": v, }, }, }, }) shouldNotMapquery = append(shouldNotMapquery, map[string]interface{}{ "function_score": map[string]interface{}{ "query": map[string]interface{}{ "multi_match": map[string]interface{}{ "boost": boost, //给查询的值赋予权重 "fields": []interface{}{"Annotation"}, "query": v, }, }, }, }) shouldNotMapquery = append(shouldNotMapquery, map[string]interface{}{ "function_score": map[string]interface{}{ "query": map[string]interface{}{ "multi_match": map[string]interface{}{ //"boost": (lenkeyWordArr-k)*boost - 1, //给查询的值赋予权重 "boost": boost, //给查询的值赋予权重 "fields": []interface{}{"BodyText"}, "query": v, }, }, }, }) } continue } //如果是 1：查询键入词 if k > 0 && ikType == 1 { continue } if v != "" { shouldMapquery = append(shouldMapquery, map[string]interface{}{ "function_score": map[string]interface{}{ "query": map[string]interface{}{ "multi_match": map[string]interface{}{ //"boost": (lenkeyWordArr - k) * boost, //给查询的值赋予权重 "boost": boost, //给查询的值赋予权重 "fields": []interface{}{"Title"}, "query": v, }, }, }, }) shouldMapquery = append(shouldMapquery, map[string]interface{}{ "function_score": map[string]interface{}{ "query": map[string]interface{}{ "multi_match": map[string]interface{}{ "boost": boost, //给查询的值赋予权重 "fields": []interface{}{"Abstract"}, "query": v, }, }, }, }) shouldMapquery = append(shouldMapquery, map[string]interface{}{ "function_score": map[string]interface{}{ "query": map[string]interface{}{ "multi_match": map[string]interface{}{ "boost": boost, //给查询的值赋予权重 "fields": []interface{}{"Annotation"}, "query": v, }, }, }, }) shouldMapquery = append(shouldMapquery, map[string]interface{}{ "function_score": map[string]interface{}{ "query": map[string]interface{}{ "multi_match": map[string]interface{}{ //"boost": (lenkeyWordArr-k)*boost - 1, //给查询的值赋予权重 "boost": boost, //给查询的值赋予权重 "fields": []interface{}{"BodyText"}, "query": v, }, }, }, }) } } shouldMap = map[string]interface{}{ "should": shouldMapquery, } shouldNotMap = map[string]interface{}{ "should": shouldNotMapquery, } //排序 sortMap := make([]interface{}, 0) //时间 sortMap = append(sortMap, map[string]interface{}{ "PublishDate": map[string]interface{}{ "order": "desc", }, }) //sortMap = append(sortMap, map[string]interface{}{ // "_score": map[string]interface{}{ // "order": "desc", // }, //}) //高亮 highlightMap := make(map[string]interface{}, 0) highlightMap = map[string]interface{}{ "fields": map[string]interface{}{ "BodyText": map[string]interface{}{}, "Title": map[string]interface{}{}, "Abstract": map[string]interface{}{}, "Annotation": map[string]interface{}{}, }, //样式红色 "post_tags": []interface{}{""}, "pre_tags": []interface{}{""}, } mustMap = append(mustMap, map[string]interface{}{ "bool": shouldMap, }) mustNotMap = append(mustNotMap, map[string]interface{}{ "bool": shouldNotMap, }) queryMap := map[string]interface{}{ "query": map[string]interface{}{ "bool": map[string]interface{}{ "must": mustMap, }, }, } //把第一次键入词的筛选条件过滤掉 if ikType == 2 { queryMap = map[string]interface{}{ "query": map[string]interface{}{ "bool": map[string]interface{}{ "must": mustMap, "must_not": mustNotMap, }, }, } } if orderColumn == "Matching" { queryMap["sort"] = sortMap } queryMap["from"] = startSize queryMap["size"] = pageSize queryMap["highlight"] = highlightMap jsonBytes, _ := json.Marshal(queryMap) fmt.Println(string(jsonBytes)) //utils.FileLog.Info(string(jsonBytes)) request := client.Search(indexName).Source(queryMap) // sets the JSON request searchByMatch, err := request.Do(context.Background()) if searchByMatch != nil { if searchByMatch.Hits != nil { for _, v := range searchByMatch.Hits.Hits { var isAppend bool articleJson, err := v.Source.MarshalJSON() if err != nil { return nil, 0, err } article := new(models.CygxArticleEs) err = json.Unmarshal(articleJson, &article) if err != nil { return nil, 0, err } searchItem := new(models.SearchItem) searchItem.ArticleId, _ = strconv.Atoi(v.Id) if len(v.Highlight["Annotation"]) > 0 { for _, vText := range v.Highlight["Annotation"] { searchItem.Body = append(searchItem.Body, vText) } } if len(v.Highlight["Abstract"]) > 0 { for _, vText := range v.Highlight["Abstract"] { searchItem.Body = append(searchItem.Body, vText) } } if len(v.Highlight["BodyText"]) > 0 { for _, vText := range v.Highlight["BodyText"] { searchItem.Body = append(searchItem.Body, vText) } } if len(searchItem.Body) == 0 { bodyRune := []rune(article.BodyText) bodyRuneLen := len(bodyRune) if bodyRuneLen > 100 { bodyRuneLen = 100 } body := string(bodyRune[:bodyRuneLen]) searchItem.Body = []string{body} } //if len(v.Highlight["BodyText"]) > 0 { // searchItem.Body = v.Highlight["BodyText"] //} else { // bodyRune := []rune(article.BodyText) // bodyRuneLen := len(bodyRune) // if bodyRuneLen > 100 { // bodyRuneLen = 100 // } // body := string(bodyRune[:bodyRuneLen]) // searchItem.Body = []string{body} //} var title string if len(v.Highlight["Title"]) > 0 { title = v.Highlight["Title"][0] } else { title = article.Title } searchItem.Title = title searchItem.PublishDate = article.PublishDate searchItem.ExpertBackground = article.ExpertBackground searchItem.CategoryId = article.CategoryId for _, v_result := range result { if v_result.ArticleId == searchItem.ArticleId { isAppend = true } } if !isAppend { result = append(result, searchItem) } } } total = searchByMatch.Hits.TotalHits.Value } return } func EsArticleSearchBody(keyWord string, startSize, pageSize int, orderColumn string, searchType int) (result []*models.SearchItem, total int64, err error) { if keyWord == "" { return } indexName := utils.IndexName client := utils.Client //Es 的高级查询有自定义排序文档一时半会儿撸不懂，先做多次查询手动过滤 2023.2.2 //ikType 查询方式，0：查所有、 1:查询键入词、 2：查询除了查询键入词之外的联想词 mustMap := make([]interface{}, 0) shouldMap := make(map[string]interface{}, 0) shouldMapquery := make([]interface{}, 0) mustNotMap := make([]interface{}, 0) shouldNotMap := make(map[string]interface{}, 0) shouldNotMapquery := make([]interface{}, 0) // @Param OrderColumn query int true "排序字段，Comprehensive综合，Matching匹配度，PublishDate 发布时间 " //keyWordWeight := GetWeight(keyWordLen) var boost int //如果是 2：查询标题，摘要，核心观点的词 if searchType == 1 { shouldMapquery = append(shouldMapquery, map[string]interface{}{ "function_score": map[string]interface{}{ "query": map[string]interface{}{ "multi_match": map[string]interface{}{ "boost": boost, //给查询的值赋予权重 "fields": []interface{}{"Title"}, "query": keyWord, }, }, }, }) shouldMapquery = append(shouldMapquery, map[string]interface{}{ "function_score": map[string]interface{}{ "query": map[string]interface{}{ "multi_match": map[string]interface{}{ "boost": boost, //给查询的值赋予权重 "fields": []interface{}{"Abstract"}, "query": keyWord, }, }, }, }) shouldMapquery = append(shouldMapquery, map[string]interface{}{ "function_score": map[string]interface{}{ "query": map[string]interface{}{ "multi_match": map[string]interface{}{ "boost": boost, //给查询的值赋予权重 "fields": []interface{}{"Annotation"}, "query": keyWord, }, }, }, }) } //如果是 2：查询body的相关词 if searchType == 2 { shouldNotMapquery = append(shouldNotMapquery, map[string]interface{}{ "function_score": map[string]interface{}{ "query": map[string]interface{}{ "multi_match": map[string]interface{}{ "boost": boost, //给查询的值赋予权重 "fields": []interface{}{"Title"}, "query": keyWord, }, }, }, }) shouldNotMapquery = append(shouldNotMapquery, map[string]interface{}{ "function_score": map[string]interface{}{ "query": map[string]interface{}{ "multi_match": map[string]interface{}{ "boost": boost, //给查询的值赋予权重 "fields": []interface{}{"Abstract"}, "query": keyWord, }, }, }, }) shouldNotMapquery = append(shouldNotMapquery, map[string]interface{}{ "function_score": map[string]interface{}{ "query": map[string]interface{}{ "multi_match": map[string]interface{}{ "boost": boost, //给查询的值赋予权重 "fields": []interface{}{"Annotation"}, "query": keyWord, }, }, }, }) //shouldNotMapquery = append(shouldNotMapquery, map[string]interface{}{ // "function_score": map[string]interface{}{ // "query": map[string]interface{}{ // "multi_match": map[string]interface{}{ // //"boost": (lenkeyWordArr-k)*boost - 1, //给查询的值赋予权重 // "boost": boost, //给查询的值赋予权重 // "fields": []interface{}{"BodyText"}, // "query": keyWord, // }, // }, // }, //}) shouldMapquery = append(shouldMapquery, map[string]interface{}{ "function_score": map[string]interface{}{ "query": map[string]interface{}{ "multi_match": map[string]interface{}{ //"boost": (lenkeyWordArr-k)*boost - 1, //给查询的值赋予权重 "boost": boost, //给查询的值赋予权重 "fields": []interface{}{"BodyText"}, "query": keyWord, }, }, }, }) } shouldMap = map[string]interface{}{ "should": shouldMapquery, } shouldNotMap = map[string]interface{}{ "should": shouldNotMapquery, } //排序 sortMap := make([]interface{}, 0) //时间 sortMap = append(sortMap, map[string]interface{}{ "PublishDate": map[string]interface{}{ "order": "desc", }, }) //高亮 highlightMap := make(map[string]interface{}, 0) highlightMap = map[string]interface{}{ "fields": map[string]interface{}{ "BodyText": map[string]interface{}{}, "Title": map[string]interface{}{}, "Abstract": map[string]interface{}{}, "Annotation": map[string]interface{}{}, }, //样式红色 "post_tags": []interface{}{""}, "pre_tags": []interface{}{""}, } mustMap = append(mustMap, map[string]interface{}{ "bool": shouldMap, }) mustNotMap = append(mustNotMap, map[string]interface{}{ "bool": shouldNotMap, }) queryMap := map[string]interface{}{ "query": map[string]interface{}{ "bool": map[string]interface{}{ "must": mustMap, }, }, } //把第一次的筛选条件过滤掉 if searchType == 2 { queryMap = map[string]interface{}{ "query": map[string]interface{}{ "bool": map[string]interface{}{ "must": mustMap, "must_not": mustNotMap, }, }, } } if orderColumn == "Matching" { queryMap["sort"] = sortMap } queryMap["from"] = startSize queryMap["size"] = pageSize queryMap["highlight"] = highlightMap jsonBytes, _ := json.Marshal(queryMap) fmt.Println(string(jsonBytes)) //utils.FileLog.Info(string(jsonBytes)) request := client.Search(indexName).Source(queryMap) // sets the JSON request searchByMatch, err := request.Do(context.Background()) if searchByMatch != nil { if searchByMatch.Hits != nil { for _, v := range searchByMatch.Hits.Hits { var isAppend bool articleJson, err := v.Source.MarshalJSON() if err != nil { return nil, 0, err } article := new(models.CygxArticleEs) err = json.Unmarshal(articleJson, &article) if err != nil { return nil, 0, err } searchItem := new(models.SearchItem) searchItem.ArticleId, _ = strconv.Atoi(v.Id) if len(v.Highlight["Annotation"]) > 0 { for _, vText := range v.Highlight["Annotation"] { searchItem.Body = append(searchItem.Body, vText) } } if len(v.Highlight["Abstract"]) > 0 { for _, vText := range v.Highlight["Abstract"] { searchItem.Body = append(searchItem.Body, vText) } } if len(v.Highlight["BodyText"]) > 0 { for _, vText := range v.Highlight["BodyText"] { searchItem.Body = append(searchItem.Body, vText) } } if len(searchItem.Body) == 0 { bodyRune := []rune(article.BodyText) bodyRuneLen := len(bodyRune) if bodyRuneLen > 100 { bodyRuneLen = 100 } body := string(bodyRune[:bodyRuneLen]) searchItem.Body = []string{body} } //if len(v.Highlight["BodyText"]) > 0 { // searchItem.Body = v.Highlight["BodyText"] //} else { // bodyRune := []rune(article.BodyText) // bodyRuneLen := len(bodyRune) // if bodyRuneLen > 100 { // bodyRuneLen = 100 // } // body := string(bodyRune[:bodyRuneLen]) // searchItem.Body = []string{body} //} var title string if len(v.Highlight["Title"]) > 0 { title = v.Highlight["Title"][0] } else { title = article.Title } searchItem.Title = title searchItem.PublishDate = article.PublishDate searchItem.ExpertBackground = article.ExpertBackground searchItem.CategoryId = article.CategoryId for _, v_result := range result { if v_result.ArticleId == searchItem.ArticleId { isAppend = true } } if !isAppend { result = append(result, searchItem) } } } total = searchByMatch.Hits.TotalHits.Value } return } // KeyWordArrSqlRegexp 预处理ik联想词的模糊查询语句 func KeyWordArrSqlRegexpAll(a []string) (ret string) { a_len := len(a) for i := 0; i < a_len; i++ { ret += a[i] + "|" } ret = strings.TrimRight(ret, "|") //ret = "'" + ret + "'" return } // KeyWordArrSqlRegexp 预处理ik联想词的模糊查询语句 func KeyWordArrSqlRegexp(a []string) (ret string) { a_len := len(a) for i := 0; i < a_len; i++ { if i == 0 { continue } ret += a[i] + "|" } ret = strings.TrimRight(ret, "|") return }