package services import ( "context" "encoding/json" "errors" "fmt" "github.com/olivere/elastic/v7" "hongze/hongze_cygx/models" "hongze/hongze_cygx/utils" "sort" "strconv" "strings" "time" ) func NewClient() (client *elastic.Client, err error) { //errorlog := log.New(os.Stdout, "APP", log.LstdFlags) //file := "" //if utils.RunMode == "release" { // //file = `/data/rdlucklog/hongze_cygx/eslog.log` // file = `./rdlucklog/eslog.log` //} else { // file = `./rdlucklog/eslog.log` //} //logFile, _ := os.OpenFile(file, os.O_RDWR|os.O_CREATE|os.O_APPEND, 0766) //client, err = elastic.NewClient( // elastic.SetURL(ES_URL), // elastic.SetBasicAuth(ES_USERNAME, ES_PASSWORD), // elastic.SetTraceLog(log.New(logFile, "ES-TRACE: ", 0)), // elastic.SetSniff(false), elastic.SetErrorLog(errorlog)) client, err = elastic.NewClient( elastic.SetURL(ES_URL), elastic.SetBasicAuth(ES_USERNAME, ES_PASSWORD), elastic.SetSniff(false)) return } // 创建文章阅读记录的Es索引 func CreateIndexNameArticleHistory() { indexName := utils.IndexNameArticleHistory mappingJson := `{ "mappings": { "dynamic": true, "properties": { "ArticleId": { "type": "integer" }, "Id": { "type": "integer" }, "ArticleType": { "type": "short" }, "CompanyArticleHistoryNum": { "type": "integer" }, "CompanyName": { "type": "keyword" }, "CompanyId": { "type": "integer" }, "CreateTime": { "type": "date", "format": "yyyy-MM-dd HH:mm:ss||yyyy-MM-dd||epoch_millis" }, "Email": { "type": "keyword" }, "Mobile": { "type": "keyword" }, "PublishDate": { "type": "date", "format": "yyyy-MM-dd HH:mm:ss||yyyy-MM-dd||epoch_millis" }, "RealName": { "type": "keyword" }, "SellerName": { "type": "keyword" }, "SellerId": { "type": "integer" }, "StopTime": { "type": "integer" }, "Title": { "type": "keyword" }, "UserId": { "type": "integer" }, "UserArticleHistoryNum": { "type": "integer" } } } }` EsCreateIndex(indexName, mappingJson) } // //func init() { // AddAllArticleHistory() //} func AddAllArticleHistory(cont context.Context) (err error) { //func AddAllArticleHistory() (err error) { defer func() { if err != nil { fmt.Println(err) go utils.SendAlarmMsg("同步阅读记录到es失败;AddAllArticleHistory Err:"+err.Error(), 2) } }() var pars []interface{} var condition string var companyIds []int condition = " AND product_id = 2 AND STATUS IN ( '正式', '试用', '冻结' ) " list, e := models.GetCompanyProductList(condition, pars) if e != nil { err = errors.New("GetCompanyProductList, Err: " + e.Error()) return } for _, v := range list { companyIds = append(companyIds, v.CompanyId) } var updateUserIds string //更改过的用户ID userIdMap := make(map[int]int) //condition := ` AND r.create_time < '` + time.Now().Format(utils.FormatDate) + `' AND r.company_id IN ( // SELECT a.company_id // FROM company AS a INNER JOIN company_product AS b ON a.company_id = b.company_id // WHERE a.enabled = 1 AND b.STATUS IN ( '正式', '试用', '冻结' )) ` // //totalAll, err := models.GetCygxArticleHistoryCountByCompany(condition) //获取阅读总数 condition = " AND company_id IN (" + utils.GetOrmInReplace(len(companyIds)) + ") AND is_del = 0 " pars = append(pars, companyIds) totalAll, e := models.GetCygxArticleHistoryRecordAllCount(condition, pars) if e != nil { err = errors.New("GetCygxArticleHistoryRecordAllCount, Err: " + e.Error()) return } //更改阅读总数 e = models.UpdateConfigByCode(strconv.Itoa(totalAll), "company_article_history_num") if e != nil { err = errors.New("UpdateConfigByCode, Err: " + e.Error()) return } //处理前一天新增的数据 //condition += ` AND create_time >= ? AND is_del = 0 ` //pars = append(pars, time.Now().AddDate(0, 0, -1).Format(utils.FormatDate)) //total, err := models.GetCygxArticleHistoryRecordAllCount(condition, pars) ////fmt.Println(total) ////return //fmt.Println(total) //if err != nil { // fmt.Println("GetCygxArticleHistoryCountByCompany Err:", err.Error()) // return err //} pars = make([]interface{}, 0) condition = ` ` condition = " AND r.company_id IN (" + utils.GetOrmInReplace(len(companyIds)) + ") AND r.is_del = 0 AND r.create_time >= ? AND r.create_time <= ? " pars = append(pars, companyIds, time.Now().AddDate(0, 0, -1).Format(utils.FormatDate), time.Now().Format(utils.FormatDate)) allList, e := models.GetCygxArticleHistoryRecordByCompanyListNew(condition, pars, 0, 0) if e != nil { err = errors.New("GetCygxArticleHistoryRecordByCompanyListNew, Err: " + e.Error()) return } fmt.Println(len(allList)) //return for k, v := range allList { //fmt.Println(v) fmt.Println(v.Id, "___", k) err := EsAddArticleHistoryData(v) if err != nil { fmt.Println("EsAddOrEditData Err:", err.Error()) //return err } if _, ok := userIdMap[v.UserId]; !ok { updateUserIds += strconv.Itoa(v.UserId) + "," userIdMap[v.UserId] = v.UserId } } //for i := 0; i <= total/1000; i++ { // allList, err := models.GetCygxArticleHistoryRecordByCompanyListNew(condition, pars, 1000*i, 1000) // if err != nil { // fmt.Println("GetCygxArticleHistoryRecordByCompanyListNew Err:", err.Error()) // return err // } // for k, v := range allList { // fmt.Println(v.Id, "___", k) // //err := EsAddArticleHistoryData(v) // //if err != nil { // // fmt.Println("EsAddOrEditData Err:", err.Error()) // // return err // //} // //if _, ok := userIdMap[v.UserId]; !ok { // // updateUserIds += strconv.Itoa(v.UserId) + "," // // userIdMap[v.UserId] = v.UserId // //} // } //} //return //处理前一天新增的数据 end //处理前一天被移动的用户 startDate := time.Now().AddDate(0, 0, -1).Format(utils.FormatDate) endDate := time.Now().Format(utils.FormatDate) var mobiles []string listUpdateUser, e := models.GetWxUserOpLogList(startDate, endDate) if e != nil && e.Error() != utils.ErrNoRow() { err = errors.New("GetWxUserOpLogList, Err: " + e.Error()) return } if len(listUpdateUser) > 0 { for _, v := range listUpdateUser { mobiles = append(mobiles, v.Mobile) } } //mobiles = strings.TrimRight(mobiles, ",") if len(mobiles) > 0 { pars = make([]interface{}, 0) condition = ` AND r.mobile IN (` + utils.GetOrmInReplace(len(mobiles)) + `)` pars = append(pars, mobiles) //修改用户的阅读记录(es 自动判断,如果有他会修改数据) listUpdatePv, e := models.GetCygxArticleHistoryRecordByCompanyListNew(condition, pars, 0, 0) if e != nil && e.Error() != utils.ErrNoRow() { err = errors.New("GetCygxArticleHistoryRecordByCompanyListNew,mobilesUpdate Err: " + e.Error()) return } for _, v := range listUpdatePv { err := EsAddArticleHistoryData(v) if err != nil { fmt.Println("EsAddOrEditData Err:", err.Error()) //return err } } } //处理前一天被移动的用户 end //处理前一天被删除的用户 { listDeleteUser, e := models.GetWxUserOpLogDeleteList(startDate, endDate) if e != nil && e.Error() != utils.ErrNoRow() { err = errors.New("GetWxUserOpLogDeleteList, Err: " + e.Error()) return } var mobilesDel []string if len(listUpdateUser) > 0 { for _, v := range listDeleteUser { mobilesDel = append(mobilesDel, v.Mobile) } } if len(mobilesDel) > 0 { pars = make([]interface{}, 0) condition = ` AND r.mobile IN (` + utils.GetOrmInReplace(len(mobilesDel)) + `)` pars = append(pars, mobilesDel) listDeletePv, e := models.GetCygxArticleHistoryRecordByCompanyListNew(condition, pars, 0, 0) if e != nil && e.Error() != utils.ErrNoRow() { err = errors.New("GetCygxArticleHistoryRecordByCompanyListNew,mobilesDel Err: " + e.Error()) return } for _, v := range listDeletePv { err := EsDeleteData(utils.IndexNameArticleHistory, strconv.Itoa(v.Id)) if err != nil { fmt.Println("EsDeleteData Err:", err.Error()) //return err } } } } //处理前一天被删除的用户 end //处理新增的阅读记录的用户阅读数量、机构阅读数量(暂未找到批量修改的方法,后期优化处理 2022.7.11) //updateUserIds = strings.TrimRight(updateUserIds, ",") //if updateUserIds != "" { // condition = ` AND r.create_time >='` + time.Now().AddDate(0, 0, -1).Format(utils.FormatDate) + `' AND r.user_id IN (` + updateUserIds + `)` // total, err := models.GetCygxArticleHistoryCountByCompany(condition) // //fmt.Println(total) // //return // fmt.Println(total) // if err != nil { // fmt.Println("GetCygxArticleHistoryCountByCompany Err:", err.Error()) // return err // } // for i := 0; i <= total/1000; i++ { // allList, err := models.GetCygxArticleHistoryRecordByCompanyList(condition, 1000*i, 1000) // if err != nil { // fmt.Println("GetCygxArticleHistoryRecordByCompanyList Err:", err.Error()) // return err // } // for k, v := range allList { // fmt.Println(v.Id, "___", k) // err := EsAddArticleHistoryData(v) // if err != nil { // fmt.Println("EsAddOrEditData Err:", err.Error()) // return err // } // if _, ok := userIdMap[v.UserId]; !ok { // updateUserIds += strconv.Itoa(v.UserId) + "," // userIdMap[v.UserId] = v.UserId // } // } // } //} return } // 新增数据 func EsAddArticleHistoryData(item *models.EsUserInteraction) (err error) { defer func() { if err != nil { fmt.Println("EsAddOrEditData Err:", err.Error()) } }() client := utils.Client resp, err := client.Index().Index(utils.IndexNameArticleHistory).Id(strconv.Itoa(item.Id)).BodyJson(item).Do(context.Background()) if err != nil { fmt.Println("新增失败:", err.Error()) return err } if resp.Status == 0 && resp.Result == "created" { //fmt.Println("新增成功") err = nil return err } else { fmt.Println("AddData", resp.Status, resp.Result) } return } // indexName:索引名称 // mappingJson:表结构 func EsCreateIndex(indexName, mappingJson string) (err error) { client := utils.Client //if err != nil { // return //} //定义表结构 exists, err := client.IndexExists(indexName).Do(context.Background()) //<5> if err != nil { return } if !exists { resp, err := client.CreateIndex(indexName).BodyJson(mappingJson).Do(context.Background()) //BodyJson(bodyJson).Do(context.Background()) if err != nil { fmt.Println("CreateIndex Err:" + err.Error()) return err } fmt.Println(resp.Index, resp.ShardsAcknowledged, resp.Acknowledged) } else { fmt.Println(indexName + " 已存在") } return } // 新增和修改数据 func EsAddOrEditData(indexName, docId string, item *ElasticTestArticleDetail) (err error) { defer func() { if err != nil { fmt.Println("EsAddOrEditData Err:", err.Error()) } }() client := utils.Client searchById, err := client.Get().Index(indexName).Id(docId).Do(context.Background()) if err != nil && !strings.Contains(err.Error(), "404") { fmt.Println("Get Err" + err.Error()) return } if searchById != nil && searchById.Found { resp, err := client.Update().Index(indexName).Id(docId).Doc(map[string]interface{}{ "BodyText": item.BodyText, "Title": item.Title, "PublishDate": item.PublishDate, "CategoryId": item.CategoryId, "ExpertBackground": item.ExpertBackground, "Abstract": item.Abstract, "Annotation": item.Annotation, }).Do(context.Background()) if err != nil { return err } if resp.Status == 0 { fmt.Println("修改成功") } else { fmt.Println("EditData", resp.Status, resp.Result) } client.CloseIndex(indexName) } else { resp, err := client.Index().Index(indexName).Id(docId).BodyJson(item).Do(context.Background()) if err != nil { fmt.Println("新增失败:", err.Error()) return err } if resp.Status == 0 && resp.Result == "created" { fmt.Println("新增成功") err = nil } else { fmt.Println("AddData", resp.Status, resp.Result) } } return } // 新增和修改数据 func EsAddOrEditDataV4(indexName, docId string, item *ElasticTestArticleDetailV4) (err error) { defer func() { if err != nil { fmt.Println("EsAddOrEditData Err:", err.Error()) } }() client := utils.Client //if err != nil { // return //} searchById, err := client.Get().Index(indexName).Id(docId).Do(context.Background()) if err != nil && !strings.Contains(err.Error(), "404") { fmt.Println("Get Err" + err.Error()) return } if searchById != nil && searchById.Found { resp, err := client.Update().Index(indexName).Id(docId).Doc(map[string]interface{}{ "BodyText": item.BodyText, "Title": item.Title, "PublishDate": item.PublishDate, "IsSummary": item.IsSummary, "IsReport": item.IsReport, }).Do(context.Background()) if err != nil { return err } fmt.Println(resp.Status, resp.Result) if resp.Status == 0 { fmt.Println("修改成功") } else { fmt.Println("EditData", resp.Status, resp.Result) } } else { resp, err := client.Index().Index(indexName).Id(docId).BodyJson(item).Do(context.Background()) if err != nil { fmt.Println("新增失败:", err.Error()) return err } if resp.Status == 0 && resp.Result == "created" { fmt.Println("新增成功") err = nil } else { fmt.Println("AddData", resp.Status, resp.Result) } } return } // 删除数据 func EsDeleteData(indexName, docId string) (err error) { client := utils.Client //if err != nil { // return //} resp, err := client.Delete().Index(indexName).Id(docId).Do(context.Background()) if err != nil { return } if resp.Status == 0 { fmt.Println("删除成功") } else { fmt.Println("AddData", resp.Status, resp.Result) } return } func GetWeight(length int) []int { steep := 10 intArr := make([]int, 0) for i := 1; i <= length; i++ { if i == 1 { intArr = append(intArr, 1) } else { min := GetArrSum(intArr) maxVal := utils.GetRandInt(min, min+steep) intArr = append(intArr, maxVal) } } //数组排序 sort.Slice(intArr, func(i, j int) bool { return intArr[i] > intArr[j] }) return intArr } func GetArrSum(intArr []int) (sum int) { for _, val := range intArr { //累计求和 sum += val } return } func EsMultiMatchFunctionScoreQuerySort(indexName, keyWord string, startSize, pageSize, userId int, orderColumn string) (result []*models.SearchItem, total int64, err error) { client := utils.Client keyWordArr, err := GetIndustryMapNameSliceV3(keyWord) keyWordArr = RemoveDuplicatesAndEmpty(keyWordArr) //artidArr := make([]elastic.Query, 0) //matchArr := make([]elastic.Query, 0) n := 0 keyWordLen := len(keyWordArr) if keyWordLen <= 0 { keyWordArr = append(keyWordArr, keyWord) keyWordLen = len(keyWordArr) } // @Param OrderColumn query int true "排序字段 ,Comprehensive综合 ,Matching匹配度 ,PublishDate 发布时间 " utils.FileLog.Info("SearchKeyWord:%s, userId:%s", keyWordArr, strconv.Itoa(userId)) //keyWordWeight := GetWeight(keyWordLen) for _, v := range keyWordArr { if v != "" { matchArr := make([]elastic.Query, 0) boolquery := elastic.NewBoolQuery() bodyFunctionQuery := elastic.NewFunctionScoreQuery() bodyFunctionQuery2 := elastic.NewFunctionScoreQuery() bodyFunctionQuery3 := elastic.NewFunctionScoreQuery() //multiMatch := elastic.NewMultiMatchQuery(v, "Title", "BodyText").Analyzer("ik_smart") multiMatch := elastic.NewMultiMatchQuery(v, "Title").Analyzer("ik_smart").Boost(100) bodyFunctionQuery.Query(multiMatch) matchArr = append(matchArr, bodyFunctionQuery) multiMatch = elastic.NewMultiMatchQuery(v, "BodyText").Analyzer("ik_smart").Boost(1) bodyFunctionQuery2.Query(multiMatch) matchArr = append(matchArr, bodyFunctionQuery2) //multiMatch = elastic.NewMultiMatchQuery(1, "IsSummary") bodyFunctionQuery3.Query(multiMatch) matchArr = append(matchArr, bodyFunctionQuery3) boolquery.Should(matchArr...) //multiMatch = elastic.NewMultiMatchQuery(v, "BodyText").Analyzer("ik_smart") //bodyFunctionQuery.Query(multiMatch) //matchArr = append(matchArr, bodyFunctionQuery) //boolquery.Should(matchArr...) highlight := elastic.NewHighlight() highlight = highlight.PreTags("").PostTags("") highlight = highlight.Fields(elastic.NewHighlighterField("Title"), elastic.NewHighlighterField("BodyText")) request := client.Search(indexName).Highlight(highlight).Sort("PublishDate", false).From(0).Size(pageSize).Query(boolquery) if orderColumn == "Matching" { request = client.Search(indexName).Highlight(highlight).From(0).Size(pageSize).Query(boolquery) } searchByMatch, err := request.Do(context.Background()) if err != nil { return nil, 0, err } if searchByMatch != nil { if searchByMatch.Hits != nil { for _, v := range searchByMatch.Hits.Hits { var isAppend bool articleJson, err := v.Source.MarshalJSON() if err != nil { return nil, 0, err } article := new(models.CygxArticleEs) err = json.Unmarshal(articleJson, &article) if err != nil { return nil, 0, err } searchItem := new(models.SearchItem) searchItem.ArticleId, _ = strconv.Atoi(v.Id) if len(v.Highlight["BodyText"]) > 0 { searchItem.Body = v.Highlight["BodyText"] } else { bodyRune := []rune(article.BodyText) bodyRuneLen := len(bodyRune) if bodyRuneLen > 100 { bodyRuneLen = 100 } body := string(bodyRune[:bodyRuneLen]) searchItem.Body = []string{body} } var title string if len(v.Highlight["Title"]) > 0 { title = v.Highlight["Title"][0] } else { title = article.Title } searchItem.Title = title searchItem.PublishDate = article.PublishDate searchItem.ExpertBackground = article.ExpertBackground searchItem.CategoryId = article.CategoryId for _, v_result := range result { if v_result.ArticleId == searchItem.ArticleId { isAppend = true } } if !isAppend { result = append(result, searchItem) } } } //total += searchByMatch.Hits.TotalHits.Value } } n++ } total = int64(len(result)) return } func EsMultiMatchFunctionScoreQueryTimeSort(indexName, keyWord string, startSize, pageSize, userId int) (result []*models.SearchItem, total int64, err error) { client := utils.Client keyWordArr, err := GetIndustryMapNameSliceV2(keyWord) keyWordArr = RemoveDuplicatesAndEmpty(keyWordArr) boolquery := elastic.NewBoolQuery() matchArr := make([]elastic.Query, 0) //matchArr2 := make([]elastic.Query, 0) n := 0 keyWordLen := len(keyWordArr) if keyWordLen <= 0 { keyWordArr = append(keyWordArr, keyWord) keyWordLen = len(keyWordArr) } utils.FileLog.Info("SearchKeyWord:%s, userId:%s", keyWordArr, strconv.Itoa(userId)) for _, v := range keyWordArr { if v != "" { multiMatch := elastic.NewMultiMatchQuery(v, "Title", "BodyText") bodyFunctionQuery := elastic.NewFunctionScoreQuery() bodyFunctionQuery.Query(multiMatch) matchArr = append(matchArr, bodyFunctionQuery) } n++ } boolquery.Should(matchArr...) highlight := elastic.NewHighlight() highlight = highlight.Fields(elastic.NewHighlighterField("Title"), elastic.NewHighlighterField("BodyText")) highlight = highlight.PreTags("").PostTags("") request := client.Search(indexName).Highlight(highlight).Sort("PublishDate", false).Size(pageSize).Query(boolquery) searchByMatch, err := request.Do(context.Background()) if searchByMatch != nil { matchResult, _ := json.Marshal(searchByMatch) utils.FileLog.Info("%s", string(matchResult)) fmt.Println(len(searchByMatch.Hits.Hits)) if searchByMatch.Hits != nil { for _, v := range searchByMatch.Hits.Hits { articleJson, err := v.Source.MarshalJSON() utils.FileLog.Info("%s", string(articleJson)) if err != nil { return nil, 0, err } article := new(models.CygxArticleEs) err = json.Unmarshal(articleJson, &article) if err != nil { return nil, 0, err } searchItem := new(models.SearchItem) searchItem.ArticleId, _ = strconv.Atoi(v.Id) if len(v.Highlight["BodyText"]) > 0 { searchItem.Body = v.Highlight["BodyText"] } else { bodyRune := []rune(article.BodyText) bodyRuneLen := len(bodyRune) if bodyRuneLen > 100 { bodyRuneLen = 100 } body := string(bodyRune[:bodyRuneLen]) searchItem.Body = []string{body} } var title string if len(v.Highlight["Title"]) > 0 { title = v.Highlight["Title"][0] } else { title = article.Title } searchItem.Title = title searchItem.PublishDate = article.PublishDate searchItem.ExpertBackground = article.ExpertBackground searchItem.CategoryId = article.CategoryId result = append(result, searchItem) } } total = searchByMatch.Hits.TotalHits.Value } return } func EsSearchReport(indexName, keyWord string, startSize, pageSize, userId int) (result []*models.SearchItem, total int64, err error) { client := utils.Client keyWordArr, err := GetIndustryMapNameSliceV3(keyWord) keyWordArr = RemoveDuplicatesAndEmpty(keyWordArr) n := 0 keyWordLen := len(keyWordArr) if keyWordLen <= 0 { keyWordArr = append(keyWordArr, keyWord) keyWordLen = len(keyWordArr) } for _, v := range keyWordArr { fmt.Println(v) } utils.FileLog.Info("SearchKeyWord:%s, userId:%s", keyWordArr, strconv.Itoa(userId)) for _, v := range keyWordArr { if v != "" { matchArr := make([]elastic.Query, 0) boolquery := elastic.NewBoolQuery() bodyFunctionQuery := elastic.NewFunctionScoreQuery() bodyFunctionQuery2 := elastic.NewFunctionScoreQuery() multiMatch := elastic.NewMultiMatchQuery(v, "Title").Analyzer("ik_smart") bodyFunctionQuery.Query(multiMatch) matchArr = append(matchArr, bodyFunctionQuery) multiMatch = elastic.NewMultiMatchQuery(1, "IsReport") bodyFunctionQuery2.Query(multiMatch) matchArr = append(matchArr, bodyFunctionQuery2) boolquery.Must(matchArr...) highlight := elastic.NewHighlight() highlight = highlight.PreTags("").PostTags("") highlight = highlight.Fields(elastic.NewHighlighterField("Title")) request := client.Search(indexName).Highlight(highlight).Sort("PublishDate", false).From(0).Size(pageSize).Query(boolquery) searchByMatch, err := request.Do(context.Background()) if err != nil { return nil, 0, err } if searchByMatch != nil { if searchByMatch.Hits != nil { for _, v := range searchByMatch.Hits.Hits { var isAppend bool articleJson, err := v.Source.MarshalJSON() if err != nil { return nil, 0, err } article := new(models.CygxArticleEs) err = json.Unmarshal(articleJson, &article) if err != nil { return nil, 0, err } searchItem := new(models.SearchItem) searchItem.ArticleId, _ = strconv.Atoi(v.Id) var title string if len(v.Highlight["Title"]) > 0 { title = v.Highlight["Title"][0] } else { title = article.Title } searchItem.Title = title searchItem.PublishDate = article.PublishDate for _, v_result := range result { if v_result.ArticleId == searchItem.ArticleId { isAppend = true } } if !isAppend { result = append(result, searchItem) } } } } } n++ } total = int64(len(result)) return } // 分页 func EsMultiMatchFunctionScoreQueryTimeSortPage(indexName, keyWord string, startSize, pageSize, userId int) (result []*models.SearchItem, total int64, err error) { client := utils.Client keyWordArr, err := GetIndustryMapNameSliceV2(keyWord) keyWordArr = RemoveDuplicatesAndEmpty(keyWordArr) boolquery := elastic.NewBoolQuery() matchArr := make([]elastic.Query, 0) n := 0 keyWordLen := len(keyWordArr) if keyWordLen <= 0 { keyWordArr = append(keyWordArr, keyWord) keyWordLen = len(keyWordArr) } for _, v := range keyWordArr { if v != "" { multiMatch := elastic.NewMultiMatchQuery(v, "Title", "BodyText") bodyFunctionQuery := elastic.NewFunctionScoreQuery() bodyFunctionQuery.Query(multiMatch) matchArr = append(matchArr, bodyFunctionQuery) } n++ } boolquery.Should(matchArr...) highlight := elastic.NewHighlight() highlight = highlight.Fields(elastic.NewHighlighterField("Title"), elastic.NewHighlighterField("BodyText")) highlight = highlight.PreTags("").PostTags("") request := client.Search(indexName).Highlight(highlight).Sort("PublishDate", false).From(startSize).Size(pageSize).Query(boolquery) searchByMatch, err := request.Do(context.Background()) if searchByMatch != nil { if searchByMatch.Hits != nil { for _, v := range searchByMatch.Hits.Hits { articleJson, err := v.Source.MarshalJSON() if err != nil { return nil, 0, err } article := new(models.CygxArticleEs) err = json.Unmarshal(articleJson, &article) if err != nil { return nil, 0, err } searchItem := new(models.SearchItem) searchItem.ArticleId, _ = strconv.Atoi(v.Id) if len(v.Highlight["BodyText"]) > 0 { searchItem.Body = v.Highlight["BodyText"] } else { bodyRune := []rune(article.BodyText) bodyRuneLen := len(bodyRune) if bodyRuneLen > 100 { bodyRuneLen = 100 } body := string(bodyRune[:bodyRuneLen]) searchItem.Body = []string{body} } var title string if len(v.Highlight["Title"]) > 0 { title = v.Highlight["Title"][0] } else { title = article.Title } searchItem.Title = title searchItem.PublishDate = article.PublishDate searchItem.ExpertBackground = article.ExpertBackground searchItem.CategoryId = article.CategoryId result = append(result, searchItem) } } total = searchByMatch.Hits.TotalHits.Value } return } func EsMultiMatchFunctionScoreQuerySortPage(indexName, keyWord string, startSize, pageSize, userId int, orderColumn string) (result []*models.SearchItem, total int64, err error) { client := utils.Client keyWordArr, err := GetIndustryMapNameSliceV3(keyWord) keyWordArr = RemoveDuplicatesAndEmpty(keyWordArr) keyWordLen := len(keyWordArr) if keyWordLen <= 0 { keyWordArr = append(keyWordArr, keyWord) keyWordLen = len(keyWordArr) } var keyWords string for _, v := range keyWordArr { keyWords += v + " " } // @Param OrderColumn query int true "排序字段 ,Comprehensive综合 ,Matching匹配度 ,PublishDate 发布时间 " //keyWordWeight := GetWeight(keyWordLen) matchArr := make([]elastic.Query, 0) boolquery := elastic.NewBoolQuery() bodyFunctionQuery := elastic.NewFunctionScoreQuery() bodyFunctionQuery2 := elastic.NewFunctionScoreQuery() bodyFunctionQuery3 := elastic.NewFunctionScoreQuery() multiMatch := elastic.NewMultiMatchQuery(keyWords, "Title").Analyzer("ik_smart").Boost(100) bodyFunctionQuery.Query(multiMatch) matchArr = append(matchArr, bodyFunctionQuery) multiMatch = elastic.NewMultiMatchQuery(keyWords, "BodyText").Analyzer("ik_smart").Boost(1) bodyFunctionQuery2.Query(multiMatch) matchArr = append(matchArr, bodyFunctionQuery2) bodyFunctionQuery3.Query(multiMatch) matchArr = append(matchArr, bodyFunctionQuery3) boolquery.Should(matchArr...) highlight := elastic.NewHighlight() highlight = highlight.PreTags("").PostTags("") highlight = highlight.Fields(elastic.NewHighlighterField("Title"), elastic.NewHighlighterField("BodyText")) request := client.Search(indexName).Highlight(highlight).Sort("PublishDate", false).From(startSize).Size(pageSize).Query(boolquery) if orderColumn == "Matching" { request = client.Search(indexName).Highlight(highlight).From(startSize).Size(pageSize).Query(boolquery) } searchByMatch, err := request.Do(context.Background()) if err != nil { return nil, 0, err } if searchByMatch != nil { if searchByMatch.Hits != nil { for _, v := range searchByMatch.Hits.Hits { var isAppend bool articleJson, err := v.Source.MarshalJSON() if err != nil { return nil, 0, err } article := new(models.CygxArticleEs) err = json.Unmarshal(articleJson, &article) if err != nil { return nil, 0, err } searchItem := new(models.SearchItem) searchItem.ArticleId, _ = strconv.Atoi(v.Id) if len(v.Highlight["BodyText"]) > 0 { searchItem.Body = v.Highlight["BodyText"] } else { bodyRune := []rune(article.BodyText) bodyRuneLen := len(bodyRune) if bodyRuneLen > 100 { bodyRuneLen = 100 } body := string(bodyRune[:bodyRuneLen]) searchItem.Body = []string{body} } var title string if len(v.Highlight["Title"]) > 0 { title = v.Highlight["Title"][0] } else { title = article.Title } searchItem.Title = title searchItem.PublishDate = article.PublishDate searchItem.ExpertBackground = article.ExpertBackground searchItem.CategoryId = article.CategoryId for _, v_result := range result { if v_result.ArticleId == searchItem.ArticleId { isAppend = true } } if !isAppend { result = append(result, searchItem) } } } } total += searchByMatch.Hits.TotalHits.Value return } //func init23423() { // EsArticleSearch("立高食品", 0, 10, "34") //} func EsArticleSearch(keyWord string, startSize, pageSize int, orderColumn string, ikType int) (result []*models.SearchItem, total int64, err error) { indexName := utils.IndexName client := utils.Client keyWordArr, err := GetIndustryMapNameSliceV3(keyWord) keyWordArr = RemoveDuplicatesAndEmpty(keyWordArr) keyWordLen := len(keyWordArr) if keyWordLen <= 0 { keyWordArr = append(keyWordArr, keyWord) keyWordLen = len(keyWordArr) } fmt.Println(keyWordArr) //如果没有联想词,而且查询的还是联想词就返回 if ikType == 2 && keyWordLen == 1 { return } //Es 的高级查询有 自定义排序 文档一时半会儿撸不懂,先做多次查询手动过滤 2023.2.2 //ikType 查询方式 ,0:查所有 、 1:查询键入词 、 2:查询除了查询键入词之外的联想词 mustMap := make([]interface{}, 0) shouldMap := make(map[string]interface{}, 0) shouldMapquery := make([]interface{}, 0) mustNotMap := make([]interface{}, 0) shouldNotMap := make(map[string]interface{}, 0) shouldNotMapquery := make([]interface{}, 0) // @Param OrderColumn query int true "排序字段 ,Comprehensive综合 ,Matching匹配度 ,PublishDate 发布时间 " //keyWordWeight := GetWeight(keyWordLen) var boost int //lenkeyWordArr := len(keyWordArr) for k, v := range keyWordArr { if k == 0 { boost = 2 * 1000 } else { boost = 1 } //如果是 2:查询除了查询键入词之外的联想词 if k == 0 && ikType == 2 { if v != "" { shouldNotMapquery = append(shouldNotMapquery, map[string]interface{}{ "function_score": map[string]interface{}{ "query": map[string]interface{}{ "multi_match": map[string]interface{}{ //"boost": (lenkeyWordArr - k) * boost, //给查询的值赋予权重 "boost": boost, //给查询的值赋予权重 "fields": []interface{}{"Title"}, "query": v, }, }, }, }) shouldNotMapquery = append(shouldNotMapquery, map[string]interface{}{ "function_score": map[string]interface{}{ "query": map[string]interface{}{ "multi_match": map[string]interface{}{ "boost": boost, //给查询的值赋予权重 "fields": []interface{}{"Abstract"}, "query": v, }, }, }, }) shouldNotMapquery = append(shouldNotMapquery, map[string]interface{}{ "function_score": map[string]interface{}{ "query": map[string]interface{}{ "multi_match": map[string]interface{}{ "boost": boost, //给查询的值赋予权重 "fields": []interface{}{"Annotation"}, "query": v, }, }, }, }) shouldNotMapquery = append(shouldNotMapquery, map[string]interface{}{ "function_score": map[string]interface{}{ "query": map[string]interface{}{ "multi_match": map[string]interface{}{ //"boost": (lenkeyWordArr-k)*boost - 1, //给查询的值赋予权重 "boost": boost, //给查询的值赋予权重 "fields": []interface{}{"BodyText"}, "query": v, }, }, }, }) } continue } //如果是 1:查询键入词 if k > 0 && ikType == 1 { continue } if v != "" { shouldMapquery = append(shouldMapquery, map[string]interface{}{ "function_score": map[string]interface{}{ "query": map[string]interface{}{ "multi_match": map[string]interface{}{ //"boost": (lenkeyWordArr - k) * boost, //给查询的值赋予权重 "boost": boost, //给查询的值赋予权重 "fields": []interface{}{"Title"}, "query": v, }, }, }, }) shouldMapquery = append(shouldMapquery, map[string]interface{}{ "function_score": map[string]interface{}{ "query": map[string]interface{}{ "multi_match": map[string]interface{}{ "boost": boost, //给查询的值赋予权重 "fields": []interface{}{"Abstract"}, "query": v, }, }, }, }) shouldMapquery = append(shouldMapquery, map[string]interface{}{ "function_score": map[string]interface{}{ "query": map[string]interface{}{ "multi_match": map[string]interface{}{ "boost": boost, //给查询的值赋予权重 "fields": []interface{}{"Annotation"}, "query": v, }, }, }, }) shouldMapquery = append(shouldMapquery, map[string]interface{}{ "function_score": map[string]interface{}{ "query": map[string]interface{}{ "multi_match": map[string]interface{}{ //"boost": (lenkeyWordArr-k)*boost - 1, //给查询的值赋予权重 "boost": boost, //给查询的值赋予权重 "fields": []interface{}{"BodyText"}, "query": v, }, }, }, }) } } shouldMap = map[string]interface{}{ "should": shouldMapquery, } shouldNotMap = map[string]interface{}{ "should": shouldNotMapquery, } //排序 sortMap := make([]interface{}, 0) //时间 sortMap = append(sortMap, map[string]interface{}{ "PublishDate": map[string]interface{}{ "order": "desc", }, }) //sortMap = append(sortMap, map[string]interface{}{ // "_score": map[string]interface{}{ // "order": "desc", // }, //}) //高亮 highlightMap := make(map[string]interface{}, 0) highlightMap = map[string]interface{}{ "fields": map[string]interface{}{ "BodyText": map[string]interface{}{}, "Title": map[string]interface{}{}, "Abstract": map[string]interface{}{}, "Annotation": map[string]interface{}{}, }, //样式 红色 "post_tags": []interface{}{""}, "pre_tags": []interface{}{""}, "fragment_size": 50, } mustMap = append(mustMap, map[string]interface{}{ "bool": shouldMap, }) mustNotMap = append(mustNotMap, map[string]interface{}{ "bool": shouldNotMap, }) queryMap := map[string]interface{}{ "query": map[string]interface{}{ "bool": map[string]interface{}{ "must": mustMap, }, }, } //把第一次键入词的筛选条件过滤掉 if ikType == 2 { queryMap = map[string]interface{}{ "query": map[string]interface{}{ "bool": map[string]interface{}{ "must": mustMap, "must_not": mustNotMap, }, }, } } if orderColumn == "Matching" { queryMap["sort"] = sortMap } queryMap["from"] = startSize queryMap["size"] = pageSize queryMap["highlight"] = highlightMap jsonBytes, _ := json.Marshal(queryMap) fmt.Println(string(jsonBytes)) //utils.FileLog.Info(string(jsonBytes)) request := client.Search(indexName).Source(queryMap) // sets the JSON request searchByMatch, err := request.Do(context.Background()) if searchByMatch != nil { if searchByMatch.Hits != nil { for _, v := range searchByMatch.Hits.Hits { var isAppend bool articleJson, err := v.Source.MarshalJSON() if err != nil { return nil, 0, err } article := new(models.CygxArticleEs) err = json.Unmarshal(articleJson, &article) if err != nil { return nil, 0, err } searchItem := new(models.SearchItem) searchItem.ArticleId, _ = strconv.Atoi(v.Id) if len(v.Highlight["Annotation"]) > 0 { for _, vText := range v.Highlight["Annotation"] { searchItem.Body = append(searchItem.Body, vText) } } if len(v.Highlight["Abstract"]) > 0 { for _, vText := range v.Highlight["Abstract"] { searchItem.Body = append(searchItem.Body, vText) } } if len(v.Highlight["BodyText"]) > 0 { for _, vText := range v.Highlight["BodyText"] { searchItem.Body = append(searchItem.Body, vText) } } if len(searchItem.Body) == 0 { bodyRune := []rune(article.BodyText) bodyRuneLen := len(bodyRune) if bodyRuneLen > 100 { bodyRuneLen = 100 } body := string(bodyRune[:bodyRuneLen]) searchItem.Body = []string{body} } //if len(v.Highlight["BodyText"]) > 0 { // searchItem.Body = v.Highlight["BodyText"] //} else { // bodyRune := []rune(article.BodyText) // bodyRuneLen := len(bodyRune) // if bodyRuneLen > 100 { // bodyRuneLen = 100 // } // body := string(bodyRune[:bodyRuneLen]) // searchItem.Body = []string{body} //} var title string if len(v.Highlight["Title"]) > 0 { title = v.Highlight["Title"][0] } else { title = article.Title } searchItem.Title = title searchItem.PublishDate = article.PublishDate searchItem.ExpertBackground = article.ExpertBackground searchItem.CategoryId = article.CategoryId for _, v_result := range result { if v_result.ArticleId == searchItem.ArticleId { isAppend = true } } if !isAppend { result = append(result, searchItem) } } } total = searchByMatch.Hits.TotalHits.Value } return } func EsArticleSearchBody(keyWord string, startSize, pageSize int, orderColumn string, searchType int) (result []*models.SearchItem, total int64, err error) { if keyWord == "" { return } indexName := utils.IndexName client := utils.Client //Es 的高级查询有 自定义排序 文档一时半会儿撸不懂,先做多次查询手动过滤 2023.2.2 //ikType 查询方式 ,0:查所有 、 1:查询键入词 、 2:查询除了查询键入词之外的联想词 mustMap := make([]interface{}, 0) shouldMap := make(map[string]interface{}, 0) shouldMapquery := make([]interface{}, 0) mustNotMap := make([]interface{}, 0) shouldNotMap := make(map[string]interface{}, 0) shouldNotMapquery := make([]interface{}, 0) // @Param OrderColumn query int true "排序字段 ,Comprehensive综合 ,Matching匹配度 ,PublishDate 发布时间 " //keyWordWeight := GetWeight(keyWordLen) var boost int //如果是 2:查询标题,摘要,核心观点的词 if searchType == 1 { shouldMapquery = append(shouldMapquery, map[string]interface{}{ "function_score": map[string]interface{}{ "query": map[string]interface{}{ "multi_match": map[string]interface{}{ "boost": boost, //给查询的值赋予权重 "fields": []interface{}{"Title"}, "query": keyWord, }, }, }, }) shouldMapquery = append(shouldMapquery, map[string]interface{}{ "function_score": map[string]interface{}{ "query": map[string]interface{}{ "multi_match": map[string]interface{}{ "boost": boost, //给查询的值赋予权重 "fields": []interface{}{"Abstract"}, "query": keyWord, }, }, }, }) shouldMapquery = append(shouldMapquery, map[string]interface{}{ "function_score": map[string]interface{}{ "query": map[string]interface{}{ "multi_match": map[string]interface{}{ "boost": boost, //给查询的值赋予权重 "fields": []interface{}{"Annotation"}, "query": keyWord, }, }, }, }) } //如果是 2:查询body的相关词 if searchType == 2 { shouldNotMapquery = append(shouldNotMapquery, map[string]interface{}{ "function_score": map[string]interface{}{ "query": map[string]interface{}{ "multi_match": map[string]interface{}{ "boost": boost, //给查询的值赋予权重 "fields": []interface{}{"Title"}, "query": keyWord, }, }, }, }) shouldNotMapquery = append(shouldNotMapquery, map[string]interface{}{ "function_score": map[string]interface{}{ "query": map[string]interface{}{ "multi_match": map[string]interface{}{ "boost": boost, //给查询的值赋予权重 "fields": []interface{}{"Abstract"}, "query": keyWord, }, }, }, }) shouldNotMapquery = append(shouldNotMapquery, map[string]interface{}{ "function_score": map[string]interface{}{ "query": map[string]interface{}{ "multi_match": map[string]interface{}{ "boost": boost, //给查询的值赋予权重 "fields": []interface{}{"Annotation"}, "query": keyWord, }, }, }, }) //shouldNotMapquery = append(shouldNotMapquery, map[string]interface{}{ // "function_score": map[string]interface{}{ // "query": map[string]interface{}{ // "multi_match": map[string]interface{}{ // //"boost": (lenkeyWordArr-k)*boost - 1, //给查询的值赋予权重 // "boost": boost, //给查询的值赋予权重 // "fields": []interface{}{"BodyText"}, // "query": keyWord, // }, // }, // }, //}) shouldMapquery = append(shouldMapquery, map[string]interface{}{ "function_score": map[string]interface{}{ "query": map[string]interface{}{ "multi_match": map[string]interface{}{ //"boost": (lenkeyWordArr-k)*boost - 1, //给查询的值赋予权重 "boost": boost, //给查询的值赋予权重 "fields": []interface{}{"BodyText"}, "query": keyWord, }, }, }, }) } shouldMap = map[string]interface{}{ "should": shouldMapquery, } shouldNotMap = map[string]interface{}{ "should": shouldNotMapquery, } //排序 sortMap := make([]interface{}, 0) //时间 sortMap = append(sortMap, map[string]interface{}{ "PublishDate": map[string]interface{}{ "order": "desc", }, }) //高亮 highlightMap := make(map[string]interface{}, 0) highlightMap = map[string]interface{}{ "fields": map[string]interface{}{ "BodyText": map[string]interface{}{}, "Title": map[string]interface{}{}, "Abstract": map[string]interface{}{}, "Annotation": map[string]interface{}{}, }, //样式 红色 "post_tags": []interface{}{""}, "pre_tags": []interface{}{""}, "fragment_size": 50, } mustMap = append(mustMap, map[string]interface{}{ "bool": shouldMap, }) mustNotMap = append(mustNotMap, map[string]interface{}{ "bool": shouldNotMap, }) queryMap := map[string]interface{}{ "query": map[string]interface{}{ "bool": map[string]interface{}{ "must": mustMap, }, }, } //把第一次的筛选条件过滤掉 if searchType == 2 { queryMap = map[string]interface{}{ "query": map[string]interface{}{ "bool": map[string]interface{}{ "must": mustMap, "must_not": mustNotMap, }, }, } } if orderColumn == "Matching" { queryMap["sort"] = sortMap } queryMap["from"] = startSize queryMap["size"] = pageSize queryMap["highlight"] = highlightMap jsonBytes, _ := json.Marshal(queryMap) fmt.Println(string(jsonBytes)) //utils.FileLog.Info(string(jsonBytes)) request := client.Search(indexName).Source(queryMap) // sets the JSON request searchByMatch, err := request.Do(context.Background()) if searchByMatch != nil { if searchByMatch.Hits != nil { for _, v := range searchByMatch.Hits.Hits { var isAppend bool articleJson, err := v.Source.MarshalJSON() if err != nil { return nil, 0, err } article := new(models.CygxArticleEs) err = json.Unmarshal(articleJson, &article) if err != nil { return nil, 0, err } searchItem := new(models.SearchItem) searchItem.ArticleId, _ = strconv.Atoi(v.Id) if len(v.Highlight["Annotation"]) > 0 { for _, vText := range v.Highlight["Annotation"] { searchItem.Body = append(searchItem.Body, vText) } } if len(v.Highlight["Abstract"]) > 0 { for _, vText := range v.Highlight["Abstract"] { searchItem.Body = append(searchItem.Body, vText) } } if len(v.Highlight["BodyText"]) > 0 { for _, vText := range v.Highlight["BodyText"] { searchItem.Body = append(searchItem.Body, vText) } } if len(searchItem.Body) == 0 { bodyRune := []rune(article.BodyText) bodyRuneLen := len(bodyRune) if bodyRuneLen > 100 { bodyRuneLen = 100 } body := string(bodyRune[:bodyRuneLen]) searchItem.Body = []string{body} } //if len(v.Highlight["BodyText"]) > 0 { // searchItem.Body = v.Highlight["BodyText"] //} else { // bodyRune := []rune(article.BodyText) // bodyRuneLen := len(bodyRune) // if bodyRuneLen > 100 { // bodyRuneLen = 100 // } // body := string(bodyRune[:bodyRuneLen]) // searchItem.Body = []string{body} //} var title string if len(v.Highlight["Title"]) > 0 { title = v.Highlight["Title"][0] } else { title = article.Title } searchItem.Title = title searchItem.PublishDate = article.PublishDate searchItem.ExpertBackground = article.ExpertBackground searchItem.CategoryId = article.CategoryId for _, v_result := range result { if v_result.ArticleId == searchItem.ArticleId { isAppend = true } } if !isAppend { result = append(result, searchItem) } } } total = searchByMatch.Hits.TotalHits.Value } return }