package services
import (
"context"
"encoding/json"
"errors"
"fmt"
"github.com/olivere/elastic/v7"
"hongze/hongze_cygx/models"
"hongze/hongze_cygx/utils"
"sort"
"strconv"
"strings"
"time"
)
func NewClient() (client *elastic.Client, err error) {
//errorlog := log.New(os.Stdout, "APP", log.LstdFlags)
//file := ""
//if utils.RunMode == "release" {
// //file = `/data/rdlucklog/hongze_cygx/eslog.log`
// file = `./rdlucklog/eslog.log`
//} else {
// file = `./rdlucklog/eslog.log`
//}
//logFile, _ := os.OpenFile(file, os.O_RDWR|os.O_CREATE|os.O_APPEND, 0766)
//client, err = elastic.NewClient(
// elastic.SetURL(ES_URL),
// elastic.SetBasicAuth(ES_USERNAME, ES_PASSWORD),
// elastic.SetTraceLog(log.New(logFile, "ES-TRACE: ", 0)),
// elastic.SetSniff(false), elastic.SetErrorLog(errorlog))
client, err = elastic.NewClient(
elastic.SetURL(ES_URL),
elastic.SetBasicAuth(ES_USERNAME, ES_PASSWORD),
elastic.SetSniff(false))
return
}
// 创建文章阅读记录的Es索引
func CreateIndexNameArticleHistory() {
indexName := utils.IndexNameArticleHistory
mappingJson := `{
"mappings": {
"dynamic": true,
"properties": {
"ArticleId": {
"type": "integer"
},
"Id": {
"type": "integer"
},
"ArticleType": {
"type": "short"
},
"CompanyArticleHistoryNum": {
"type": "integer"
},
"CompanyName": {
"type": "keyword"
},
"CompanyId": {
"type": "integer"
},
"CreateTime": {
"type": "date",
"format": "yyyy-MM-dd HH:mm:ss||yyyy-MM-dd||epoch_millis"
},
"Email": {
"type": "keyword"
},
"Mobile": {
"type": "keyword"
},
"PublishDate": {
"type": "date",
"format": "yyyy-MM-dd HH:mm:ss||yyyy-MM-dd||epoch_millis"
},
"RealName": {
"type": "keyword"
},
"SellerName": {
"type": "keyword"
},
"SellerId": {
"type": "integer"
},
"StopTime": {
"type": "integer"
},
"Title": {
"type": "keyword"
},
"UserId": {
"type": "integer"
},
"UserArticleHistoryNum": {
"type": "integer"
}
}
}
}`
EsCreateIndex(indexName, mappingJson)
}
//
//func init() {
// AddAllArticleHistory()
//}
func AddAllArticleHistory(cont context.Context) (err error) {
//func AddAllArticleHistory() (err error) {
defer func() {
if err != nil {
fmt.Println(err)
go utils.SendAlarmMsg("同步阅读记录到es失败;AddAllArticleHistory Err:"+err.Error(), 2)
}
}()
var pars []interface{}
var condition string
var companyIds []int
condition = " AND product_id = 2 AND STATUS IN ( '正式', '试用', '冻结' ) "
list, e := models.GetCompanyProductList(condition, pars)
if e != nil {
err = errors.New("GetCompanyProductList, Err: " + e.Error())
return
}
for _, v := range list {
companyIds = append(companyIds, v.CompanyId)
}
var updateUserIds string //更改过的用户ID
userIdMap := make(map[int]int)
//condition := ` AND r.create_time < '` + time.Now().Format(utils.FormatDate) + `' AND r.company_id IN (
// SELECT a.company_id
// FROM company AS a INNER JOIN company_product AS b ON a.company_id = b.company_id
// WHERE a.enabled = 1 AND b.STATUS IN ( '正式', '试用', '冻结' )) `
//
//totalAll, err := models.GetCygxArticleHistoryCountByCompany(condition)
//获取阅读总数
condition = " AND company_id IN (" + utils.GetOrmInReplace(len(companyIds)) + ") AND is_del = 0 "
pars = append(pars, companyIds)
totalAll, e := models.GetCygxArticleHistoryRecordAllCount(condition, pars)
if e != nil {
err = errors.New("GetCygxArticleHistoryRecordAllCount, Err: " + e.Error())
return
}
//更改阅读总数
e = models.UpdateConfigByCode(strconv.Itoa(totalAll), "company_article_history_num")
if e != nil {
err = errors.New("UpdateConfigByCode, Err: " + e.Error())
return
}
//处理前一天新增的数据
//condition += ` AND create_time >= ? AND is_del = 0 `
//pars = append(pars, time.Now().AddDate(0, 0, -1).Format(utils.FormatDate))
//total, err := models.GetCygxArticleHistoryRecordAllCount(condition, pars)
////fmt.Println(total)
////return
//fmt.Println(total)
//if err != nil {
// fmt.Println("GetCygxArticleHistoryCountByCompany Err:", err.Error())
// return err
//}
pars = make([]interface{}, 0)
condition = ` `
condition = " AND r.company_id IN (" + utils.GetOrmInReplace(len(companyIds)) + ") AND r.is_del = 0 AND r.create_time >= ? AND r.create_time <= ? "
pars = append(pars, companyIds, time.Now().AddDate(0, 0, -1).Format(utils.FormatDate), time.Now().Format(utils.FormatDate))
allList, e := models.GetCygxArticleHistoryRecordByCompanyListNew(condition, pars, 0, 0)
if e != nil {
err = errors.New("GetCygxArticleHistoryRecordByCompanyListNew, Err: " + e.Error())
return
}
fmt.Println(len(allList))
//return
for k, v := range allList {
//fmt.Println(v)
fmt.Println(v.Id, "___", k)
err := EsAddArticleHistoryData(v)
if err != nil {
fmt.Println("EsAddOrEditData Err:", err.Error())
//return err
}
if _, ok := userIdMap[v.UserId]; !ok {
updateUserIds += strconv.Itoa(v.UserId) + ","
userIdMap[v.UserId] = v.UserId
}
}
//for i := 0; i <= total/1000; i++ {
// allList, err := models.GetCygxArticleHistoryRecordByCompanyListNew(condition, pars, 1000*i, 1000)
// if err != nil {
// fmt.Println("GetCygxArticleHistoryRecordByCompanyListNew Err:", err.Error())
// return err
// }
// for k, v := range allList {
// fmt.Println(v.Id, "___", k)
// //err := EsAddArticleHistoryData(v)
// //if err != nil {
// // fmt.Println("EsAddOrEditData Err:", err.Error())
// // return err
// //}
// //if _, ok := userIdMap[v.UserId]; !ok {
// // updateUserIds += strconv.Itoa(v.UserId) + ","
// // userIdMap[v.UserId] = v.UserId
// //}
// }
//}
//return
//处理前一天新增的数据 end
//处理前一天被移动的用户
startDate := time.Now().AddDate(0, 0, -1).Format(utils.FormatDate)
endDate := time.Now().Format(utils.FormatDate)
var mobiles []string
listUpdateUser, e := models.GetWxUserOpLogList(startDate, endDate)
if e != nil && e.Error() != utils.ErrNoRow() {
err = errors.New("GetWxUserOpLogList, Err: " + e.Error())
return
}
if len(listUpdateUser) > 0 {
for _, v := range listUpdateUser {
mobiles = append(mobiles, v.Mobile)
}
}
//mobiles = strings.TrimRight(mobiles, ",")
if len(mobiles) > 0 {
pars = make([]interface{}, 0)
condition = ` AND r.mobile IN (` + utils.GetOrmInReplace(len(mobiles)) + `)`
pars = append(pars, mobiles)
//修改用户的阅读记录(es 自动判断,如果有他会修改数据)
listUpdatePv, e := models.GetCygxArticleHistoryRecordByCompanyListNew(condition, pars, 0, 0)
if e != nil && e.Error() != utils.ErrNoRow() {
err = errors.New("GetCygxArticleHistoryRecordByCompanyListNew,mobilesUpdate Err: " + e.Error())
return
}
for _, v := range listUpdatePv {
err := EsAddArticleHistoryData(v)
if err != nil {
fmt.Println("EsAddOrEditData Err:", err.Error())
//return err
}
}
}
//处理前一天被移动的用户 end
//处理前一天被删除的用户
{
listDeleteUser, e := models.GetWxUserOpLogDeleteList(startDate, endDate)
if e != nil && e.Error() != utils.ErrNoRow() {
err = errors.New("GetWxUserOpLogDeleteList, Err: " + e.Error())
return
}
var mobilesDel []string
if len(listUpdateUser) > 0 {
for _, v := range listDeleteUser {
mobilesDel = append(mobilesDel, v.Mobile)
}
}
if len(mobilesDel) > 0 {
pars = make([]interface{}, 0)
condition = ` AND r.mobile IN (` + utils.GetOrmInReplace(len(mobilesDel)) + `)`
pars = append(pars, mobilesDel)
listDeletePv, e := models.GetCygxArticleHistoryRecordByCompanyListNew(condition, pars, 0, 0)
if e != nil && e.Error() != utils.ErrNoRow() {
err = errors.New("GetCygxArticleHistoryRecordByCompanyListNew,mobilesDel Err: " + e.Error())
return
}
for _, v := range listDeletePv {
err := EsDeleteData(utils.IndexNameArticleHistory, strconv.Itoa(v.Id))
if err != nil {
fmt.Println("EsDeleteData Err:", err.Error())
//return err
}
}
}
}
//处理前一天被删除的用户 end
//处理新增的阅读记录的用户阅读数量、机构阅读数量(暂未找到批量修改的方法,后期优化处理 2022.7.11)
//updateUserIds = strings.TrimRight(updateUserIds, ",")
//if updateUserIds != "" {
// condition = ` AND r.create_time >='` + time.Now().AddDate(0, 0, -1).Format(utils.FormatDate) + `' AND r.user_id IN (` + updateUserIds + `)`
// total, err := models.GetCygxArticleHistoryCountByCompany(condition)
// //fmt.Println(total)
// //return
// fmt.Println(total)
// if err != nil {
// fmt.Println("GetCygxArticleHistoryCountByCompany Err:", err.Error())
// return err
// }
// for i := 0; i <= total/1000; i++ {
// allList, err := models.GetCygxArticleHistoryRecordByCompanyList(condition, 1000*i, 1000)
// if err != nil {
// fmt.Println("GetCygxArticleHistoryRecordByCompanyList Err:", err.Error())
// return err
// }
// for k, v := range allList {
// fmt.Println(v.Id, "___", k)
// err := EsAddArticleHistoryData(v)
// if err != nil {
// fmt.Println("EsAddOrEditData Err:", err.Error())
// return err
// }
// if _, ok := userIdMap[v.UserId]; !ok {
// updateUserIds += strconv.Itoa(v.UserId) + ","
// userIdMap[v.UserId] = v.UserId
// }
// }
// }
//}
return
}
// 新增数据
func EsAddArticleHistoryData(item *models.EsUserInteraction) (err error) {
defer func() {
if err != nil {
fmt.Println("EsAddOrEditData Err:", err.Error())
}
}()
client := utils.Client
resp, err := client.Index().Index(utils.IndexNameArticleHistory).Id(strconv.Itoa(item.Id)).BodyJson(item).Do(context.Background())
if err != nil {
fmt.Println("新增失败:", err.Error())
return err
}
if resp.Status == 0 && resp.Result == "created" {
//fmt.Println("新增成功")
err = nil
return err
} else {
fmt.Println("AddData", resp.Status, resp.Result)
}
return
}
// indexName:索引名称
// mappingJson:表结构
func EsCreateIndex(indexName, mappingJson string) (err error) {
client := utils.Client
//if err != nil {
// return
//}
//定义表结构
exists, err := client.IndexExists(indexName).Do(context.Background()) //<5>
if err != nil {
return
}
if !exists {
resp, err := client.CreateIndex(indexName).BodyJson(mappingJson).Do(context.Background())
//BodyJson(bodyJson).Do(context.Background())
if err != nil {
fmt.Println("CreateIndex Err:" + err.Error())
return err
}
fmt.Println(resp.Index, resp.ShardsAcknowledged, resp.Acknowledged)
} else {
fmt.Println(indexName + " 已存在")
}
return
}
// 新增和修改数据
func EsAddOrEditData(indexName, docId string, item *ElasticTestArticleDetail) (err error) {
defer func() {
if err != nil {
fmt.Println("EsAddOrEditData Err:", err.Error())
}
}()
client := utils.Client
searchById, err := client.Get().Index(indexName).Id(docId).Do(context.Background())
if err != nil && !strings.Contains(err.Error(), "404") {
fmt.Println("Get Err" + err.Error())
return
}
if searchById != nil && searchById.Found {
resp, err := client.Update().Index(indexName).Id(docId).Doc(map[string]interface{}{
"BodyText": item.BodyText,
"Title": item.Title,
"PublishDate": item.PublishDate,
"CategoryId": item.CategoryId,
"ExpertBackground": item.ExpertBackground,
"Abstract": item.Abstract,
"Annotation": item.Annotation,
}).Do(context.Background())
if err != nil {
return err
}
if resp.Status == 0 {
fmt.Println("修改成功")
} else {
fmt.Println("EditData", resp.Status, resp.Result)
}
client.CloseIndex(indexName)
} else {
resp, err := client.Index().Index(indexName).Id(docId).BodyJson(item).Do(context.Background())
if err != nil {
fmt.Println("新增失败:", err.Error())
return err
}
if resp.Status == 0 && resp.Result == "created" {
fmt.Println("新增成功")
err = nil
} else {
fmt.Println("AddData", resp.Status, resp.Result)
}
}
return
}
// 新增和修改数据
func EsAddOrEditDataV4(indexName, docId string, item *ElasticTestArticleDetailV4) (err error) {
defer func() {
if err != nil {
fmt.Println("EsAddOrEditData Err:", err.Error())
}
}()
client := utils.Client
//if err != nil {
// return
//}
searchById, err := client.Get().Index(indexName).Id(docId).Do(context.Background())
if err != nil && !strings.Contains(err.Error(), "404") {
fmt.Println("Get Err" + err.Error())
return
}
if searchById != nil && searchById.Found {
resp, err := client.Update().Index(indexName).Id(docId).Doc(map[string]interface{}{
"BodyText": item.BodyText,
"Title": item.Title,
"PublishDate": item.PublishDate,
"IsSummary": item.IsSummary,
"IsReport": item.IsReport,
}).Do(context.Background())
if err != nil {
return err
}
fmt.Println(resp.Status, resp.Result)
if resp.Status == 0 {
fmt.Println("修改成功")
} else {
fmt.Println("EditData", resp.Status, resp.Result)
}
} else {
resp, err := client.Index().Index(indexName).Id(docId).BodyJson(item).Do(context.Background())
if err != nil {
fmt.Println("新增失败:", err.Error())
return err
}
if resp.Status == 0 && resp.Result == "created" {
fmt.Println("新增成功")
err = nil
} else {
fmt.Println("AddData", resp.Status, resp.Result)
}
}
return
}
// 删除数据
func EsDeleteData(indexName, docId string) (err error) {
client := utils.Client
//if err != nil {
// return
//}
resp, err := client.Delete().Index(indexName).Id(docId).Do(context.Background())
if err != nil {
return
}
if resp.Status == 0 {
fmt.Println("删除成功")
} else {
fmt.Println("AddData", resp.Status, resp.Result)
}
return
}
func GetWeight(length int) []int {
steep := 10
intArr := make([]int, 0)
for i := 1; i <= length; i++ {
if i == 1 {
intArr = append(intArr, 1)
} else {
min := GetArrSum(intArr)
maxVal := utils.GetRandInt(min, min+steep)
intArr = append(intArr, maxVal)
}
}
//数组排序
sort.Slice(intArr, func(i, j int) bool {
return intArr[i] > intArr[j]
})
return intArr
}
func GetArrSum(intArr []int) (sum int) {
for _, val := range intArr {
//累计求和
sum += val
}
return
}
func EsMultiMatchFunctionScoreQuerySort(indexName, keyWord string, startSize, pageSize, userId int, orderColumn string) (result []*models.SearchItem, total int64, err error) {
client := utils.Client
keyWordArr, err := GetIndustryMapNameSliceV3(keyWord)
keyWordArr = RemoveDuplicatesAndEmpty(keyWordArr)
//artidArr := make([]elastic.Query, 0)
//matchArr := make([]elastic.Query, 0)
n := 0
keyWordLen := len(keyWordArr)
if keyWordLen <= 0 {
keyWordArr = append(keyWordArr, keyWord)
keyWordLen = len(keyWordArr)
}
// @Param OrderColumn query int true "排序字段 ,Comprehensive综合 ,Matching匹配度 ,PublishDate 发布时间 "
utils.FileLog.Info("SearchKeyWord:%s, userId:%s", keyWordArr, strconv.Itoa(userId))
//keyWordWeight := GetWeight(keyWordLen)
for _, v := range keyWordArr {
if v != "" {
matchArr := make([]elastic.Query, 0)
boolquery := elastic.NewBoolQuery()
bodyFunctionQuery := elastic.NewFunctionScoreQuery()
bodyFunctionQuery2 := elastic.NewFunctionScoreQuery()
bodyFunctionQuery3 := elastic.NewFunctionScoreQuery()
//multiMatch := elastic.NewMultiMatchQuery(v, "Title", "BodyText").Analyzer("ik_smart")
multiMatch := elastic.NewMultiMatchQuery(v, "Title").Analyzer("ik_smart").Boost(100)
bodyFunctionQuery.Query(multiMatch)
matchArr = append(matchArr, bodyFunctionQuery)
multiMatch = elastic.NewMultiMatchQuery(v, "BodyText").Analyzer("ik_smart").Boost(1)
bodyFunctionQuery2.Query(multiMatch)
matchArr = append(matchArr, bodyFunctionQuery2)
//multiMatch = elastic.NewMultiMatchQuery(1, "IsSummary")
bodyFunctionQuery3.Query(multiMatch)
matchArr = append(matchArr, bodyFunctionQuery3)
boolquery.Should(matchArr...)
//multiMatch = elastic.NewMultiMatchQuery(v, "BodyText").Analyzer("ik_smart")
//bodyFunctionQuery.Query(multiMatch)
//matchArr = append(matchArr, bodyFunctionQuery)
//boolquery.Should(matchArr...)
highlight := elastic.NewHighlight()
highlight = highlight.PreTags("").PostTags("")
highlight = highlight.Fields(elastic.NewHighlighterField("Title"), elastic.NewHighlighterField("BodyText"))
request := client.Search(indexName).Highlight(highlight).Sort("PublishDate", false).From(0).Size(pageSize).Query(boolquery)
if orderColumn == "Matching" {
request = client.Search(indexName).Highlight(highlight).From(0).Size(pageSize).Query(boolquery)
}
searchByMatch, err := request.Do(context.Background())
if err != nil {
return nil, 0, err
}
if searchByMatch != nil {
if searchByMatch.Hits != nil {
for _, v := range searchByMatch.Hits.Hits {
var isAppend bool
articleJson, err := v.Source.MarshalJSON()
if err != nil {
return nil, 0, err
}
article := new(models.CygxArticleEs)
err = json.Unmarshal(articleJson, &article)
if err != nil {
return nil, 0, err
}
searchItem := new(models.SearchItem)
searchItem.ArticleId, _ = strconv.Atoi(v.Id)
if len(v.Highlight["BodyText"]) > 0 {
searchItem.Body = v.Highlight["BodyText"]
} else {
bodyRune := []rune(article.BodyText)
bodyRuneLen := len(bodyRune)
if bodyRuneLen > 100 {
bodyRuneLen = 100
}
body := string(bodyRune[:bodyRuneLen])
searchItem.Body = []string{body}
}
var title string
if len(v.Highlight["Title"]) > 0 {
title = v.Highlight["Title"][0]
} else {
title = article.Title
}
searchItem.Title = title
searchItem.PublishDate = article.PublishDate
searchItem.ExpertBackground = article.ExpertBackground
searchItem.CategoryId = article.CategoryId
for _, v_result := range result {
if v_result.ArticleId == searchItem.ArticleId {
isAppend = true
}
}
if !isAppend {
result = append(result, searchItem)
}
}
}
//total += searchByMatch.Hits.TotalHits.Value
}
}
n++
}
total = int64(len(result))
return
}
func EsMultiMatchFunctionScoreQueryTimeSort(indexName, keyWord string, startSize, pageSize, userId int) (result []*models.SearchItem, total int64, err error) {
client := utils.Client
keyWordArr, err := GetIndustryMapNameSliceV2(keyWord)
keyWordArr = RemoveDuplicatesAndEmpty(keyWordArr)
boolquery := elastic.NewBoolQuery()
matchArr := make([]elastic.Query, 0)
//matchArr2 := make([]elastic.Query, 0)
n := 0
keyWordLen := len(keyWordArr)
if keyWordLen <= 0 {
keyWordArr = append(keyWordArr, keyWord)
keyWordLen = len(keyWordArr)
}
utils.FileLog.Info("SearchKeyWord:%s, userId:%s", keyWordArr, strconv.Itoa(userId))
for _, v := range keyWordArr {
if v != "" {
multiMatch := elastic.NewMultiMatchQuery(v, "Title", "BodyText")
bodyFunctionQuery := elastic.NewFunctionScoreQuery()
bodyFunctionQuery.Query(multiMatch)
matchArr = append(matchArr, bodyFunctionQuery)
}
n++
}
boolquery.Should(matchArr...)
highlight := elastic.NewHighlight()
highlight = highlight.Fields(elastic.NewHighlighterField("Title"), elastic.NewHighlighterField("BodyText"))
highlight = highlight.PreTags("").PostTags("")
request := client.Search(indexName).Highlight(highlight).Sort("PublishDate", false).Size(pageSize).Query(boolquery)
searchByMatch, err := request.Do(context.Background())
if searchByMatch != nil {
matchResult, _ := json.Marshal(searchByMatch)
utils.FileLog.Info("%s", string(matchResult))
fmt.Println(len(searchByMatch.Hits.Hits))
if searchByMatch.Hits != nil {
for _, v := range searchByMatch.Hits.Hits {
articleJson, err := v.Source.MarshalJSON()
utils.FileLog.Info("%s", string(articleJson))
if err != nil {
return nil, 0, err
}
article := new(models.CygxArticleEs)
err = json.Unmarshal(articleJson, &article)
if err != nil {
return nil, 0, err
}
searchItem := new(models.SearchItem)
searchItem.ArticleId, _ = strconv.Atoi(v.Id)
if len(v.Highlight["BodyText"]) > 0 {
searchItem.Body = v.Highlight["BodyText"]
} else {
bodyRune := []rune(article.BodyText)
bodyRuneLen := len(bodyRune)
if bodyRuneLen > 100 {
bodyRuneLen = 100
}
body := string(bodyRune[:bodyRuneLen])
searchItem.Body = []string{body}
}
var title string
if len(v.Highlight["Title"]) > 0 {
title = v.Highlight["Title"][0]
} else {
title = article.Title
}
searchItem.Title = title
searchItem.PublishDate = article.PublishDate
searchItem.ExpertBackground = article.ExpertBackground
searchItem.CategoryId = article.CategoryId
result = append(result, searchItem)
}
}
total = searchByMatch.Hits.TotalHits.Value
}
return
}
func EsSearchReport(indexName, keyWord string, startSize, pageSize, userId int) (result []*models.SearchItem, total int64, err error) {
client := utils.Client
keyWordArr, err := GetIndustryMapNameSliceV3(keyWord)
keyWordArr = RemoveDuplicatesAndEmpty(keyWordArr)
n := 0
keyWordLen := len(keyWordArr)
if keyWordLen <= 0 {
keyWordArr = append(keyWordArr, keyWord)
keyWordLen = len(keyWordArr)
}
for _, v := range keyWordArr {
fmt.Println(v)
}
utils.FileLog.Info("SearchKeyWord:%s, userId:%s", keyWordArr, strconv.Itoa(userId))
for _, v := range keyWordArr {
if v != "" {
matchArr := make([]elastic.Query, 0)
boolquery := elastic.NewBoolQuery()
bodyFunctionQuery := elastic.NewFunctionScoreQuery()
bodyFunctionQuery2 := elastic.NewFunctionScoreQuery()
multiMatch := elastic.NewMultiMatchQuery(v, "Title").Analyzer("ik_smart")
bodyFunctionQuery.Query(multiMatch)
matchArr = append(matchArr, bodyFunctionQuery)
multiMatch = elastic.NewMultiMatchQuery(1, "IsReport")
bodyFunctionQuery2.Query(multiMatch)
matchArr = append(matchArr, bodyFunctionQuery2)
boolquery.Must(matchArr...)
highlight := elastic.NewHighlight()
highlight = highlight.PreTags("").PostTags("")
highlight = highlight.Fields(elastic.NewHighlighterField("Title"))
request := client.Search(indexName).Highlight(highlight).Sort("PublishDate", false).From(0).Size(pageSize).Query(boolquery)
searchByMatch, err := request.Do(context.Background())
if err != nil {
return nil, 0, err
}
if searchByMatch != nil {
if searchByMatch.Hits != nil {
for _, v := range searchByMatch.Hits.Hits {
var isAppend bool
articleJson, err := v.Source.MarshalJSON()
if err != nil {
return nil, 0, err
}
article := new(models.CygxArticleEs)
err = json.Unmarshal(articleJson, &article)
if err != nil {
return nil, 0, err
}
searchItem := new(models.SearchItem)
searchItem.ArticleId, _ = strconv.Atoi(v.Id)
var title string
if len(v.Highlight["Title"]) > 0 {
title = v.Highlight["Title"][0]
} else {
title = article.Title
}
searchItem.Title = title
searchItem.PublishDate = article.PublishDate
for _, v_result := range result {
if v_result.ArticleId == searchItem.ArticleId {
isAppend = true
}
}
if !isAppend {
result = append(result, searchItem)
}
}
}
}
}
n++
}
total = int64(len(result))
return
}
// 分页
func EsMultiMatchFunctionScoreQueryTimeSortPage(indexName, keyWord string, startSize, pageSize, userId int) (result []*models.SearchItem, total int64, err error) {
client := utils.Client
keyWordArr, err := GetIndustryMapNameSliceV2(keyWord)
keyWordArr = RemoveDuplicatesAndEmpty(keyWordArr)
boolquery := elastic.NewBoolQuery()
matchArr := make([]elastic.Query, 0)
n := 0
keyWordLen := len(keyWordArr)
if keyWordLen <= 0 {
keyWordArr = append(keyWordArr, keyWord)
keyWordLen = len(keyWordArr)
}
for _, v := range keyWordArr {
if v != "" {
multiMatch := elastic.NewMultiMatchQuery(v, "Title", "BodyText")
bodyFunctionQuery := elastic.NewFunctionScoreQuery()
bodyFunctionQuery.Query(multiMatch)
matchArr = append(matchArr, bodyFunctionQuery)
}
n++
}
boolquery.Should(matchArr...)
highlight := elastic.NewHighlight()
highlight = highlight.Fields(elastic.NewHighlighterField("Title"), elastic.NewHighlighterField("BodyText"))
highlight = highlight.PreTags("").PostTags("")
request := client.Search(indexName).Highlight(highlight).Sort("PublishDate", false).From(startSize).Size(pageSize).Query(boolquery)
searchByMatch, err := request.Do(context.Background())
if searchByMatch != nil {
if searchByMatch.Hits != nil {
for _, v := range searchByMatch.Hits.Hits {
articleJson, err := v.Source.MarshalJSON()
if err != nil {
return nil, 0, err
}
article := new(models.CygxArticleEs)
err = json.Unmarshal(articleJson, &article)
if err != nil {
return nil, 0, err
}
searchItem := new(models.SearchItem)
searchItem.ArticleId, _ = strconv.Atoi(v.Id)
if len(v.Highlight["BodyText"]) > 0 {
searchItem.Body = v.Highlight["BodyText"]
} else {
bodyRune := []rune(article.BodyText)
bodyRuneLen := len(bodyRune)
if bodyRuneLen > 100 {
bodyRuneLen = 100
}
body := string(bodyRune[:bodyRuneLen])
searchItem.Body = []string{body}
}
var title string
if len(v.Highlight["Title"]) > 0 {
title = v.Highlight["Title"][0]
} else {
title = article.Title
}
searchItem.Title = title
searchItem.PublishDate = article.PublishDate
searchItem.ExpertBackground = article.ExpertBackground
searchItem.CategoryId = article.CategoryId
result = append(result, searchItem)
}
}
total = searchByMatch.Hits.TotalHits.Value
}
return
}
func EsMultiMatchFunctionScoreQuerySortPage(indexName, keyWord string, startSize, pageSize, userId int, orderColumn string) (result []*models.SearchItem, total int64, err error) {
client := utils.Client
keyWordArr, err := GetIndustryMapNameSliceV3(keyWord)
keyWordArr = RemoveDuplicatesAndEmpty(keyWordArr)
keyWordLen := len(keyWordArr)
if keyWordLen <= 0 {
keyWordArr = append(keyWordArr, keyWord)
keyWordLen = len(keyWordArr)
}
var keyWords string
for _, v := range keyWordArr {
keyWords += v + " "
}
// @Param OrderColumn query int true "排序字段 ,Comprehensive综合 ,Matching匹配度 ,PublishDate 发布时间 "
//keyWordWeight := GetWeight(keyWordLen)
matchArr := make([]elastic.Query, 0)
boolquery := elastic.NewBoolQuery()
bodyFunctionQuery := elastic.NewFunctionScoreQuery()
bodyFunctionQuery2 := elastic.NewFunctionScoreQuery()
bodyFunctionQuery3 := elastic.NewFunctionScoreQuery()
multiMatch := elastic.NewMultiMatchQuery(keyWords, "Title").Analyzer("ik_smart").Boost(100)
bodyFunctionQuery.Query(multiMatch)
matchArr = append(matchArr, bodyFunctionQuery)
multiMatch = elastic.NewMultiMatchQuery(keyWords, "BodyText").Analyzer("ik_smart").Boost(1)
bodyFunctionQuery2.Query(multiMatch)
matchArr = append(matchArr, bodyFunctionQuery2)
bodyFunctionQuery3.Query(multiMatch)
matchArr = append(matchArr, bodyFunctionQuery3)
boolquery.Should(matchArr...)
highlight := elastic.NewHighlight()
highlight = highlight.PreTags("").PostTags("")
highlight = highlight.Fields(elastic.NewHighlighterField("Title"), elastic.NewHighlighterField("BodyText"))
request := client.Search(indexName).Highlight(highlight).Sort("PublishDate", false).From(startSize).Size(pageSize).Query(boolquery)
if orderColumn == "Matching" {
request = client.Search(indexName).Highlight(highlight).From(startSize).Size(pageSize).Query(boolquery)
}
searchByMatch, err := request.Do(context.Background())
if err != nil {
return nil, 0, err
}
if searchByMatch != nil {
if searchByMatch.Hits != nil {
for _, v := range searchByMatch.Hits.Hits {
var isAppend bool
articleJson, err := v.Source.MarshalJSON()
if err != nil {
return nil, 0, err
}
article := new(models.CygxArticleEs)
err = json.Unmarshal(articleJson, &article)
if err != nil {
return nil, 0, err
}
searchItem := new(models.SearchItem)
searchItem.ArticleId, _ = strconv.Atoi(v.Id)
if len(v.Highlight["BodyText"]) > 0 {
searchItem.Body = v.Highlight["BodyText"]
} else {
bodyRune := []rune(article.BodyText)
bodyRuneLen := len(bodyRune)
if bodyRuneLen > 100 {
bodyRuneLen = 100
}
body := string(bodyRune[:bodyRuneLen])
searchItem.Body = []string{body}
}
var title string
if len(v.Highlight["Title"]) > 0 {
title = v.Highlight["Title"][0]
} else {
title = article.Title
}
searchItem.Title = title
searchItem.PublishDate = article.PublishDate
searchItem.ExpertBackground = article.ExpertBackground
searchItem.CategoryId = article.CategoryId
for _, v_result := range result {
if v_result.ArticleId == searchItem.ArticleId {
isAppend = true
}
}
if !isAppend {
result = append(result, searchItem)
}
}
}
}
total += searchByMatch.Hits.TotalHits.Value
return
}
//func init23423() {
// EsArticleSearch("立高食品", 0, 10, "34")
//}
func EsArticleSearch(keyWord string, startSize, pageSize int, orderColumn string, ikType int) (result []*models.SearchItem, total int64, err error) {
indexName := utils.IndexName
client := utils.Client
keyWordArr, err := GetIndustryMapNameSliceV3(keyWord)
keyWordArr = RemoveDuplicatesAndEmpty(keyWordArr)
keyWordLen := len(keyWordArr)
if keyWordLen <= 0 {
keyWordArr = append(keyWordArr, keyWord)
keyWordLen = len(keyWordArr)
}
fmt.Println(keyWordArr)
//如果没有联想词,而且查询的还是联想词就返回
if ikType == 2 && keyWordLen == 1 {
return
}
//Es 的高级查询有 自定义排序 文档一时半会儿撸不懂,先做多次查询手动过滤 2023.2.2
//ikType 查询方式 ,0:查所有 、 1:查询键入词 、 2:查询除了查询键入词之外的联想词
mustMap := make([]interface{}, 0)
shouldMap := make(map[string]interface{}, 0)
shouldMapquery := make([]interface{}, 0)
mustNotMap := make([]interface{}, 0)
shouldNotMap := make(map[string]interface{}, 0)
shouldNotMapquery := make([]interface{}, 0)
// @Param OrderColumn query int true "排序字段 ,Comprehensive综合 ,Matching匹配度 ,PublishDate 发布时间 "
//keyWordWeight := GetWeight(keyWordLen)
var boost int
//lenkeyWordArr := len(keyWordArr)
for k, v := range keyWordArr {
if k == 0 {
boost = 2 * 1000
} else {
boost = 1
}
//如果是 2:查询除了查询键入词之外的联想词
if k == 0 && ikType == 2 {
if v != "" {
shouldNotMapquery = append(shouldNotMapquery, map[string]interface{}{
"function_score": map[string]interface{}{
"query": map[string]interface{}{
"multi_match": map[string]interface{}{
//"boost": (lenkeyWordArr - k) * boost, //给查询的值赋予权重
"boost": boost, //给查询的值赋予权重
"fields": []interface{}{"Title"},
"query": v,
},
},
},
})
shouldNotMapquery = append(shouldNotMapquery, map[string]interface{}{
"function_score": map[string]interface{}{
"query": map[string]interface{}{
"multi_match": map[string]interface{}{
"boost": boost, //给查询的值赋予权重
"fields": []interface{}{"Abstract"},
"query": v,
},
},
},
})
shouldNotMapquery = append(shouldNotMapquery, map[string]interface{}{
"function_score": map[string]interface{}{
"query": map[string]interface{}{
"multi_match": map[string]interface{}{
"boost": boost, //给查询的值赋予权重
"fields": []interface{}{"Annotation"},
"query": v,
},
},
},
})
shouldNotMapquery = append(shouldNotMapquery, map[string]interface{}{
"function_score": map[string]interface{}{
"query": map[string]interface{}{
"multi_match": map[string]interface{}{
//"boost": (lenkeyWordArr-k)*boost - 1, //给查询的值赋予权重
"boost": boost, //给查询的值赋予权重
"fields": []interface{}{"BodyText"},
"query": v,
},
},
},
})
}
continue
}
//如果是 1:查询键入词
if k > 0 && ikType == 1 {
continue
}
if v != "" {
shouldMapquery = append(shouldMapquery, map[string]interface{}{
"function_score": map[string]interface{}{
"query": map[string]interface{}{
"multi_match": map[string]interface{}{
//"boost": (lenkeyWordArr - k) * boost, //给查询的值赋予权重
"boost": boost, //给查询的值赋予权重
"fields": []interface{}{"Title"},
"query": v,
},
},
},
})
shouldMapquery = append(shouldMapquery, map[string]interface{}{
"function_score": map[string]interface{}{
"query": map[string]interface{}{
"multi_match": map[string]interface{}{
"boost": boost, //给查询的值赋予权重
"fields": []interface{}{"Abstract"},
"query": v,
},
},
},
})
shouldMapquery = append(shouldMapquery, map[string]interface{}{
"function_score": map[string]interface{}{
"query": map[string]interface{}{
"multi_match": map[string]interface{}{
"boost": boost, //给查询的值赋予权重
"fields": []interface{}{"Annotation"},
"query": v,
},
},
},
})
shouldMapquery = append(shouldMapquery, map[string]interface{}{
"function_score": map[string]interface{}{
"query": map[string]interface{}{
"multi_match": map[string]interface{}{
//"boost": (lenkeyWordArr-k)*boost - 1, //给查询的值赋予权重
"boost": boost, //给查询的值赋予权重
"fields": []interface{}{"BodyText"},
"query": v,
},
},
},
})
}
}
shouldMap = map[string]interface{}{
"should": shouldMapquery,
}
shouldNotMap = map[string]interface{}{
"should": shouldNotMapquery,
}
//排序
sortMap := make([]interface{}, 0)
//时间
sortMap = append(sortMap, map[string]interface{}{
"PublishDate": map[string]interface{}{
"order": "desc",
},
})
//sortMap = append(sortMap, map[string]interface{}{
// "_score": map[string]interface{}{
// "order": "desc",
// },
//})
//高亮
highlightMap := make(map[string]interface{}, 0)
highlightMap = map[string]interface{}{
"fields": map[string]interface{}{
"BodyText": map[string]interface{}{},
"Title": map[string]interface{}{},
"Abstract": map[string]interface{}{},
"Annotation": map[string]interface{}{},
},
//样式 红色
"post_tags": []interface{}{""},
"pre_tags": []interface{}{""},
"fragment_size": 50,
}
mustMap = append(mustMap, map[string]interface{}{
"bool": shouldMap,
})
mustNotMap = append(mustNotMap, map[string]interface{}{
"bool": shouldNotMap,
})
queryMap := map[string]interface{}{
"query": map[string]interface{}{
"bool": map[string]interface{}{
"must": mustMap,
},
},
}
//把第一次键入词的筛选条件过滤掉
if ikType == 2 {
queryMap = map[string]interface{}{
"query": map[string]interface{}{
"bool": map[string]interface{}{
"must": mustMap,
"must_not": mustNotMap,
},
},
}
}
if orderColumn == "Matching" {
queryMap["sort"] = sortMap
}
queryMap["from"] = startSize
queryMap["size"] = pageSize
queryMap["highlight"] = highlightMap
jsonBytes, _ := json.Marshal(queryMap)
fmt.Println(string(jsonBytes))
//utils.FileLog.Info(string(jsonBytes))
request := client.Search(indexName).Source(queryMap) // sets the JSON request
searchByMatch, err := request.Do(context.Background())
if searchByMatch != nil {
if searchByMatch.Hits != nil {
for _, v := range searchByMatch.Hits.Hits {
var isAppend bool
articleJson, err := v.Source.MarshalJSON()
if err != nil {
return nil, 0, err
}
article := new(models.CygxArticleEs)
err = json.Unmarshal(articleJson, &article)
if err != nil {
return nil, 0, err
}
searchItem := new(models.SearchItem)
searchItem.ArticleId, _ = strconv.Atoi(v.Id)
if len(v.Highlight["Annotation"]) > 0 {
for _, vText := range v.Highlight["Annotation"] {
searchItem.Body = append(searchItem.Body, vText)
}
}
if len(v.Highlight["Abstract"]) > 0 {
for _, vText := range v.Highlight["Abstract"] {
searchItem.Body = append(searchItem.Body, vText)
}
}
if len(v.Highlight["BodyText"]) > 0 {
for _, vText := range v.Highlight["BodyText"] {
searchItem.Body = append(searchItem.Body, vText)
}
}
if len(searchItem.Body) == 0 {
bodyRune := []rune(article.BodyText)
bodyRuneLen := len(bodyRune)
if bodyRuneLen > 100 {
bodyRuneLen = 100
}
body := string(bodyRune[:bodyRuneLen])
searchItem.Body = []string{body}
}
//if len(v.Highlight["BodyText"]) > 0 {
// searchItem.Body = v.Highlight["BodyText"]
//} else {
// bodyRune := []rune(article.BodyText)
// bodyRuneLen := len(bodyRune)
// if bodyRuneLen > 100 {
// bodyRuneLen = 100
// }
// body := string(bodyRune[:bodyRuneLen])
// searchItem.Body = []string{body}
//}
var title string
if len(v.Highlight["Title"]) > 0 {
title = v.Highlight["Title"][0]
} else {
title = article.Title
}
searchItem.Title = title
searchItem.PublishDate = article.PublishDate
searchItem.ExpertBackground = article.ExpertBackground
searchItem.CategoryId = article.CategoryId
for _, v_result := range result {
if v_result.ArticleId == searchItem.ArticleId {
isAppend = true
}
}
if !isAppend {
result = append(result, searchItem)
}
}
}
total = searchByMatch.Hits.TotalHits.Value
}
return
}
func EsArticleSearchBody(keyWord string, startSize, pageSize int, orderColumn string, searchType int) (result []*models.SearchItem, total int64, err error) {
if keyWord == "" {
return
}
indexName := utils.IndexName
client := utils.Client
//Es 的高级查询有 自定义排序 文档一时半会儿撸不懂,先做多次查询手动过滤 2023.2.2
//ikType 查询方式 ,0:查所有 、 1:查询键入词 、 2:查询除了查询键入词之外的联想词
mustMap := make([]interface{}, 0)
shouldMap := make(map[string]interface{}, 0)
shouldMapquery := make([]interface{}, 0)
mustNotMap := make([]interface{}, 0)
shouldNotMap := make(map[string]interface{}, 0)
shouldNotMapquery := make([]interface{}, 0)
// @Param OrderColumn query int true "排序字段 ,Comprehensive综合 ,Matching匹配度 ,PublishDate 发布时间 "
//keyWordWeight := GetWeight(keyWordLen)
var boost int
//如果是 2:查询标题,摘要,核心观点的词
if searchType == 1 {
shouldMapquery = append(shouldMapquery, map[string]interface{}{
"function_score": map[string]interface{}{
"query": map[string]interface{}{
"multi_match": map[string]interface{}{
"boost": boost, //给查询的值赋予权重
"fields": []interface{}{"Title"},
"query": keyWord,
},
},
},
})
shouldMapquery = append(shouldMapquery, map[string]interface{}{
"function_score": map[string]interface{}{
"query": map[string]interface{}{
"multi_match": map[string]interface{}{
"boost": boost, //给查询的值赋予权重
"fields": []interface{}{"Abstract"},
"query": keyWord,
},
},
},
})
shouldMapquery = append(shouldMapquery, map[string]interface{}{
"function_score": map[string]interface{}{
"query": map[string]interface{}{
"multi_match": map[string]interface{}{
"boost": boost, //给查询的值赋予权重
"fields": []interface{}{"Annotation"},
"query": keyWord,
},
},
},
})
}
//如果是 2:查询body的相关词
if searchType == 2 {
shouldNotMapquery = append(shouldNotMapquery, map[string]interface{}{
"function_score": map[string]interface{}{
"query": map[string]interface{}{
"multi_match": map[string]interface{}{
"boost": boost, //给查询的值赋予权重
"fields": []interface{}{"Title"},
"query": keyWord,
},
},
},
})
shouldNotMapquery = append(shouldNotMapquery, map[string]interface{}{
"function_score": map[string]interface{}{
"query": map[string]interface{}{
"multi_match": map[string]interface{}{
"boost": boost, //给查询的值赋予权重
"fields": []interface{}{"Abstract"},
"query": keyWord,
},
},
},
})
shouldNotMapquery = append(shouldNotMapquery, map[string]interface{}{
"function_score": map[string]interface{}{
"query": map[string]interface{}{
"multi_match": map[string]interface{}{
"boost": boost, //给查询的值赋予权重
"fields": []interface{}{"Annotation"},
"query": keyWord,
},
},
},
})
//shouldNotMapquery = append(shouldNotMapquery, map[string]interface{}{
// "function_score": map[string]interface{}{
// "query": map[string]interface{}{
// "multi_match": map[string]interface{}{
// //"boost": (lenkeyWordArr-k)*boost - 1, //给查询的值赋予权重
// "boost": boost, //给查询的值赋予权重
// "fields": []interface{}{"BodyText"},
// "query": keyWord,
// },
// },
// },
//})
shouldMapquery = append(shouldMapquery, map[string]interface{}{
"function_score": map[string]interface{}{
"query": map[string]interface{}{
"multi_match": map[string]interface{}{
//"boost": (lenkeyWordArr-k)*boost - 1, //给查询的值赋予权重
"boost": boost, //给查询的值赋予权重
"fields": []interface{}{"BodyText"},
"query": keyWord,
},
},
},
})
}
shouldMap = map[string]interface{}{
"should": shouldMapquery,
}
shouldNotMap = map[string]interface{}{
"should": shouldNotMapquery,
}
//排序
sortMap := make([]interface{}, 0)
//时间
sortMap = append(sortMap, map[string]interface{}{
"PublishDate": map[string]interface{}{
"order": "desc",
},
})
//高亮
highlightMap := make(map[string]interface{}, 0)
highlightMap = map[string]interface{}{
"fields": map[string]interface{}{
"BodyText": map[string]interface{}{},
"Title": map[string]interface{}{},
"Abstract": map[string]interface{}{},
"Annotation": map[string]interface{}{},
},
//样式 红色
"post_tags": []interface{}{""},
"pre_tags": []interface{}{""},
"fragment_size": 50,
}
mustMap = append(mustMap, map[string]interface{}{
"bool": shouldMap,
})
mustNotMap = append(mustNotMap, map[string]interface{}{
"bool": shouldNotMap,
})
queryMap := map[string]interface{}{
"query": map[string]interface{}{
"bool": map[string]interface{}{
"must": mustMap,
},
},
}
//把第一次的筛选条件过滤掉
if searchType == 2 {
queryMap = map[string]interface{}{
"query": map[string]interface{}{
"bool": map[string]interface{}{
"must": mustMap,
"must_not": mustNotMap,
},
},
}
}
if orderColumn == "Matching" {
queryMap["sort"] = sortMap
}
queryMap["from"] = startSize
queryMap["size"] = pageSize
queryMap["highlight"] = highlightMap
jsonBytes, _ := json.Marshal(queryMap)
fmt.Println(string(jsonBytes))
//utils.FileLog.Info(string(jsonBytes))
request := client.Search(indexName).Source(queryMap) // sets the JSON request
searchByMatch, err := request.Do(context.Background())
if searchByMatch != nil {
if searchByMatch.Hits != nil {
for _, v := range searchByMatch.Hits.Hits {
var isAppend bool
articleJson, err := v.Source.MarshalJSON()
if err != nil {
return nil, 0, err
}
article := new(models.CygxArticleEs)
err = json.Unmarshal(articleJson, &article)
if err != nil {
return nil, 0, err
}
searchItem := new(models.SearchItem)
searchItem.ArticleId, _ = strconv.Atoi(v.Id)
if len(v.Highlight["Annotation"]) > 0 {
for _, vText := range v.Highlight["Annotation"] {
searchItem.Body = append(searchItem.Body, vText)
}
}
if len(v.Highlight["Abstract"]) > 0 {
for _, vText := range v.Highlight["Abstract"] {
searchItem.Body = append(searchItem.Body, vText)
}
}
if len(v.Highlight["BodyText"]) > 0 {
for _, vText := range v.Highlight["BodyText"] {
searchItem.Body = append(searchItem.Body, vText)
}
}
if len(searchItem.Body) == 0 {
bodyRune := []rune(article.BodyText)
bodyRuneLen := len(bodyRune)
if bodyRuneLen > 100 {
bodyRuneLen = 100
}
body := string(bodyRune[:bodyRuneLen])
searchItem.Body = []string{body}
}
//if len(v.Highlight["BodyText"]) > 0 {
// searchItem.Body = v.Highlight["BodyText"]
//} else {
// bodyRune := []rune(article.BodyText)
// bodyRuneLen := len(bodyRune)
// if bodyRuneLen > 100 {
// bodyRuneLen = 100
// }
// body := string(bodyRune[:bodyRuneLen])
// searchItem.Body = []string{body}
//}
var title string
if len(v.Highlight["Title"]) > 0 {
title = v.Highlight["Title"][0]
} else {
title = article.Title
}
searchItem.Title = title
searchItem.PublishDate = article.PublishDate
searchItem.ExpertBackground = article.ExpertBackground
searchItem.CategoryId = article.CategoryId
for _, v_result := range result {
if v_result.ArticleId == searchItem.ArticleId {
isAppend = true
}
}
if !isAppend {
result = append(result, searchItem)
}
}
}
total = searchByMatch.Hits.TotalHits.Value
}
return
}