package services
import (
"context"
"encoding/json"
"fmt"
"github.com/PuerkitoBio/goquery"
"github.com/olivere/elastic/v7"
"hongze/hongze_web_mfyx/models"
"hongze/hongze_web_mfyx/utils"
"strconv"
//"strconv"
"errors"
"html"
"strings"
)
type SearchComprehensiveItem struct {
SourceId int `description:"资源ID"`
IsSummary int `description:"是否是纪要"`
Source string `description:"资源类型 报告 ：article 、图表 ：newchart、微路演 ：roadshow、活动 ：activity、活动视频：activityvideo、活动音频：activityvoice、专项调研活动：activityspecial 、 本周研究汇总： researchsummary 、 上周纪要汇总 ：minutessummary 、晨会精华 ：meetingreviewchapt 、 产品内测：productinterior"`
Title string `description:"标题"`
BodyText string `description:"内容"`
PublishDate string `description:"发布时间"`
Abstract string `description:"摘要"`
Annotation string `description:"核心观点"`
IndustryName string `description:"产业名称"`
SubjectNames string `description:"标的名称"`
Body []string
}
type ElasticComprehensiveDetail struct {
SourceId int `description:"资源ID"`
IsSummary int `description:"是否是纪要"`
Source string `description:"资源类型 报告 ：article 、图表 ：newchart、微路演 ：roadshow、活动 ：activity、活动视频：activityvideo、活动音频：activityvoice、专项调研活动：activityspecial 、 本周研究汇总： researchsummary 、 上周纪要汇总 ：minutessummary 、晨会精华 ：meetingreviewchapt 、 产品内测：productinterior 、 产业资源包：industrialsource"`
Title string `description:"标题"`
BodyText string `description:"内容"`
PublishDate string `description:"发布时间"`
Abstract string `description:"摘要"`
Annotation string `description:"核心观点"`
IndustryName string `description:"产业名称"`
SubjectNames string `description:"标的名称"`
}
func EsComprehensiveSearch(keyWord string, startSize, pageSize int) (result []*SearchComprehensiveItem, total int64, err error) {
indexName := utils.IndexNameComprehensive
client := utils.Client
keyWordArr, err := GetIndustryMapNameSliceV3(keyWord)
keyWordArr = RemoveDuplicatesAndEmpty(keyWordArr)
keyWordLen := len(keyWordArr)
if keyWordLen <= 0 {
keyWordArr = append(keyWordArr, keyWord)
keyWordLen = len(keyWordArr)
}
//Es 的高级查询有 自定义排序 文档一时半会儿撸不懂，先做多次查询手动过滤 2023.2.2
//ikType 查询方式 ，0：查所有 、 1:查询键入词 、 2：查询除了查询键入词之外的联想词
mustMap := make([]interface{}, 0)
shouldMap := make(map[string]interface{}, 0)
shouldMapquery := make([]interface{}, 0)
mustNotMap := make([]interface{}, 0)
shouldNotMap := make(map[string]interface{}, 0)
shouldNotMapquery := make([]interface{}, 0)
// @Param OrderColumn query int true "排序字段 ，Comprehensive综合 ，Matching匹配度 ，PublishDate 发布时间 "
//keyWordWeight := GetWeight(keyWordLen)
var boost int
//lenkeyWordArr := len(keyWordArr)
for k, v := range keyWordArr {
if k > 0 {
continue
}
if k == 0 {
boost = 2 * 1000
} else {
boost = 1
}
if v != "" {
shouldMapquery = append(shouldMapquery, map[string]interface{}{
"function_score": map[string]interface{}{
"query": map[string]interface{}{
"multi_match": map[string]interface{}{
//"boost": (lenkeyWordArr - k) * boost, //给查询的值赋予权重
"boost": boost, //给查询的值赋予权重
"fields": []interface{}{"Title"},
"query": v,
},
},
},
})
shouldMapquery = append(shouldMapquery, map[string]interface{}{
"function_score": map[string]interface{}{
"query": map[string]interface{}{
"multi_match": map[string]interface{}{
"boost": boost, //给查询的值赋予权重
"fields": []interface{}{"Abstract"},
"query": v,
},
},
},
})
shouldMapquery = append(shouldMapquery, map[string]interface{}{
"function_score": map[string]interface{}{
"query": map[string]interface{}{
"multi_match": map[string]interface{}{
"boost": boost, //给查询的值赋予权重
"fields": []interface{}{"Annotation"},
"query": v,
},
},
},
})
//shouldMapquery = append(shouldMapquery, map[string]interface{}{
// "function_score": map[string]interface{}{
// "query": map[string]interface{}{
// "multi_match": map[string]interface{}{
// //"boost": (lenkeyWordArr-k)*boost - 1, //给查询的值赋予权重
// "boost": boost, //给查询的值赋予权重
// "fields": []interface{}{"BodyText"},
// "query": v,
// },
// },
// },
//})
shouldMapquery = append(shouldMapquery, map[string]interface{}{
"function_score": map[string]interface{}{
"query": map[string]interface{}{
"multi_match": map[string]interface{}{
//"boost": (lenkeyWordArr-k)*boost - 1, //给查询的值赋予权重
"boost": boost, //给查询的值赋予权重
"fields": []interface{}{"IndustryName"},
"query": v,
},
},
},
})
shouldMapquery = append(shouldMapquery, map[string]interface{}{
"function_score": map[string]interface{}{
"query": map[string]interface{}{
"multi_match": map[string]interface{}{
//"boost": (lenkeyWordArr-k)*boost - 1, //给查询的值赋予权重
"boost": boost, //给查询的值赋予权重
"fields": []interface{}{"SubjectNames"},
"query": v,
},
},
},
})
}
}
shouldMap = map[string]interface{}{
"should": shouldMapquery,
}
shouldNotMap = map[string]interface{}{
"should": shouldNotMapquery,
}
//排序
sortMap := make([]interface{}, 0)
//时间
sortMap = append(sortMap, map[string]interface{}{
"PublishDate": map[string]interface{}{
"order": "desc",
},
})
//sortMap = append(sortMap, map[string]interface{}{
// "_score": map[string]interface{}{
// "order": "desc",
// },
//})
//高亮
highlightMap := make(map[string]interface{}, 0)
highlightMap = map[string]interface{}{
"fields": map[string]interface{}{
//"BodyText": map[string]interface{}{},
"Title": map[string]interface{}{},
"Abstract": map[string]interface{}{},
"Annotation": map[string]interface{}{},
},
//样式 红色
"post_tags": []interface{}{"</font>"},
"pre_tags": []interface{}{"<font color='red'>"},
"fragment_size": 50,
}
mustMap = append(mustMap, map[string]interface{}{
"bool": shouldMap,
})
mustNotMap = append(mustNotMap, map[string]interface{}{
"bool": shouldNotMap,
})
queryMap := map[string]interface{}{
"query": map[string]interface{}{
"bool": map[string]interface{}{
"must": mustMap,
},
},
}
queryMap["sort"] = sortMap
queryMap["from"] = startSize
queryMap["size"] = pageSize
queryMap["highlight"] = highlightMap
//jsonBytes, _ := json.Marshal(queryMap)
//fmt.Println(string(jsonBytes))
//utils.FileLog.Info(string(jsonBytes))
request := client.Search(indexName).Source(queryMap) // sets the JSON request
searchByMatch, err := request.Do(context.Background())
if searchByMatch != nil {
if searchByMatch.Hits != nil {
for _, v := range searchByMatch.Hits.Hits {
var isAppend bool
articleJson, err := v.Source.MarshalJSON()
if err != nil {
return nil, 0, err
}
article := new(ElasticComprehensiveDetail)
err = json.Unmarshal(articleJson, &article)
if err != nil {
return nil, 0, err
}
//fmt.Println(article.SourceId, article.Title, article.Source)
searchItem := new(SearchComprehensiveItem)
searchItem.SourceId = article.SourceId
if len(v.Highlight["Annotation"]) > 0 {
for _, vText := range v.Highlight["Annotation"] {
searchItem.Body = append(searchItem.Body, vText)
}
}
if len(v.Highlight["Abstract"]) > 0 {
for _, vText := range v.Highlight["Abstract"] {
searchItem.Body = append(searchItem.Body, vText)
}
}
if len(v.Highlight["BodyText"]) > 0 {
for _, vText := range v.Highlight["BodyText"] {
searchItem.Body = append(searchItem.Body, vText)
}
}
//searchItem.IsSummary = article.IsSummary
//if len(searchItem.Body) == 0 {
// bodyRune := []rune(article.BodyText)
// bodyRuneLen := len(bodyRune)
// if bodyRuneLen > 100 {
// bodyRuneLen = 100
// }
// body := string(bodyRune[:bodyRuneLen])
// searchItem.Body = []string{body}
//}
var title string
if len(v.Highlight["Title"]) > 0 {
title = v.Highlight["Title"][0]
} else {
title = article.Title
}
searchItem.Title = title
searchItem.Source = article.Source
searchItem.PublishDate = article.PublishDate
if !isAppend {
result = append(result, searchItem)
}
}
}
total = searchByMatch.Hits.TotalHits.Value
}
return
}
func GetResourceDataEsList(list []*SearchComprehensiveItem, user *models.WxUserItem) (items []*models.CygxResourceDataNewResp, err error) {
var condition string
var pars []interface{}
uid := user.UserId
titleHighlight := make(map[int]string)
bodyHighlight := make(map[int][]string)
yanXuanbodyHighlight := make(map[int][]string)
mapItems := make(map[string]*models.CygxResourceDataNewResp)
for _, v := range list {
//预处理文章
item := new(models.CygxResourceDataNewResp)
item.SourceId = v.SourceId
item.Source = v.Source
item.PublishDate = utils.TimeRemoveHms2(v.PublishDate)
item.BodyHighlight = v.Body
item.TitleHighlight = v.Title
titleHighlight[v.SourceId] = v.Title
bodyHighlight[v.SourceId] = v.Body
mapItems[fmt.Sprint(v.Source, v.SourceId)] = item
}
var articleIds []int //报告
var newchartIds []int //图表
var roadshowIds []string //微路演
var activityIds []int //活动
var activityvideoIds []string // 活动视频
var activityvoiceIds []string //活动音频
var activityspecialIds []int //专项调研活动
var researchsummaryIds []int //本周研究汇总
var minutessummaryIds []int //上周纪要汇总
var meetingreviewchaptIds []int //晨会精华
var productinteriorIds []int //产品内测
var industrialResourceIdsHz []int // 弘则产业资源包
var industrialResourceIdsYx []int // 研选产业资源包
var yanxuanSpecialIds []int // 研选专栏
var askserieVideoIds []string //问答系列视频
var reportselectionIds []int //报告精选
//Source string `description:"资源类型 报告 ：article 、图表 ：newchart、微路演 ：roadshow、活动 ：activity、活动视频：activityvideo、活动音频：activityvoice、专项调研活动：activityspecial 、 本周研究汇总： researchsummary 、 上周纪要汇总 ：minutessummary 、晨会精华 ：meetingreviewchapt 、 产品内测：productinterior
for _, v := range list {
if v.Source == "article" {
articleIds = append(articleIds, v.SourceId)
} else if v.Source == "newchart" {
newchartIds = append(newchartIds, v.SourceId)
} else if v.Source == "roadshow" {
roadshowIds = append(roadshowIds, strconv.Itoa(v.SourceId))
} else if v.Source == "activity" {
activityIds = append(activityIds, v.SourceId)
} else if v.Source == "activityvideo" {
activityvideoIds = append(activityvideoIds, strconv.Itoa(v.SourceId))
} else if v.Source == "activityvoice" {
activityvoiceIds = append(activityvoiceIds, strconv.Itoa(v.SourceId))
} else if v.Source == "activityspecial" {
activityspecialIds = append(activityspecialIds, v.SourceId)
} else if v.Source == "researchsummary" {
researchsummaryIds = append(researchsummaryIds, v.SourceId)
} else if v.Source == "minutessummary" {
minutessummaryIds = append(minutessummaryIds, v.SourceId)
} else if v.Source == "meetingreviewchapt" {
meetingreviewchaptIds = append(meetingreviewchaptIds, v.SourceId)
} else if v.Source == "productinterior" {
productinteriorIds = append(productinteriorIds, v.SourceId)
} else if v.Source == "industrialsourceHz" {
industrialResourceIdsHz = append(industrialResourceIdsHz, v.SourceId)
} else if v.Source == "industrialsourceYx" {
industrialResourceIdsYx = append(industrialResourceIdsYx, v.SourceId)
} else if v.Source == utils.CYGX_OBJ_YANXUANSPECIAL {
yanxuanSpecialIds = append(yanxuanSpecialIds, v.SourceId)
yanXuanbodyHighlight[v.SourceId] = v.Body
} else if v.Source == utils.CYGX_OBJ_ASKSERIEVIDEO {
askserieVideoIds = append(askserieVideoIds, strconv.Itoa(v.SourceId))
} else if v.Source == "reportselection" {
reportselectionIds = append(reportselectionIds, v.SourceId)
}
}
detail, e := models.GetConfigByCode("city_img_url")
if e != nil {
err = errors.New("GetResourceDataList, Err: " + e.Error())
return
}
detailChart, e := models.GetConfigByCode("chart_img_url")
if e != nil {
err = errors.New("GetResourceDataList, Err: " + e.Error())
return
}
addressList := strings.Split(detail.ConfigValue, "{|}")
mapAddress := make(map[string]string)
chartList := strings.Split(detailChart.ConfigValue, "{|}")
mapChart := make(map[string]string)
var cityName string
var chartName string
var imgUrl string
var imgUrlChart string
for _, v := range addressList {
vslice := strings.Split(v, "_")
cityName = vslice[0]
imgUrl = vslice[len(vslice)-1]
mapAddress[cityName] = imgUrl
}
for _, v := range chartList {
vslice := strings.Split(v, "_")
chartName = vslice[0]
imgUrlChart = vslice[len(vslice)-1]
mapChart[chartName] = imgUrlChart
}
var imgUrlResp string
//处理文章
if len(articleIds) > 0 {
pars = make([]interface{}, 0)
condition = ` AND a.article_id IN (` + utils.GetOrmInReplace(len(articleIds)) + `)`
pars = append(pars, articleIds)
articleList, e := models.GetHomeListPublic(condition, pars, 0, len(articleIds))
if e != nil {
err = errors.New("GetResourceDataList, Err: " + e.Error())
return
}
articleList, e = HandleArticleCategoryImg(articleList, user)
if e != nil {
err = errors.New("HandleArticleCategoryImg, Err: " + e.Error())
return
}
for _, v := range articleList {
v.Body = ""
if titleHighlight[v.ArticleId] != "" {
v.Title = titleHighlight[v.ArticleId]
}
if len(bodyHighlight[v.ArticleId]) > 0 {
v.Abstract = ""
v.Annotation = ""
v.BodyHighlight = bodyHighlight[v.ArticleId]
} else {
v.BodyHighlight = make([]string, 0)
}
mapItems[fmt.Sprint("article", v.ArticleId)].Article = v
}
}
//处理活动
if len(activityIds) > 0 {
for _, vss := range activityIds {
imgUrlResp += strconv.Itoa(vss) + ","
}
pars = make([]interface{}, 0)
condition = ` AND art.activity_id IN (` + utils.GetOrmInReplace(len(activityIds)) + `) `
pars = append(pars, activityIds)
activityList, e := models.GetActivityListNew(condition, pars, uid, 0, len(activityIds), 0, 0, "")
if e != nil {
err = errors.New("GetResourceDataList, Err: " + e.Error())
return
}
var activityListRersp []*models.ActivityListResp
//for _, v := range activityList {
// activityListRersp = append(activityListRersp, ActivityButtonShow(v, user, make([]string, 0)))
//}
//var pArr []string
activityListRersp = ActivityArrButtonShow(activityList, user, make([]string, 0))
//处理不同的报名方式按钮回显
mapActivitySignup, e := GetActivitySignupResp(activityIds, user)
if e != nil {
err = errors.New("GetActivitySignupResp, Err: " + e.Error())
return
}
for _, v := range activityListRersp {
if v == nil {
continue
}
if v.ActivityType == 0 {
if mapAddress[v.City] != "" {
imgUrlResp = mapAddress[v.City]
} else {
imgUrlResp = mapAddress["其它"]
}
} else {
if mapChart[v.ChartPermissionName] != "" {
imgUrlResp = mapChart[v.ChartPermissionName]
}
}
v.ImgUrl = imgUrlResp
v.SourceType = mapActivitySignup[v.ActivityId]
mapItems[fmt.Sprint("activity", v.ActivityId)].Activity = v
}
}
//处理路演 处理路活动视频 处理路活音频
if len(roadshowIds)+len(activityvideoIds)+len(activityvoiceIds)+len(askserieVideoIds) > 0 {
audioIdstr := strings.Join(activityvoiceIds, ",")
activityVideoIdsStr := strings.Join(activityvideoIds, ",")
roadshowIdsStr := strings.Join(roadshowIds, ",")
askserieVideoIdsStr := strings.Join(askserieVideoIds, ",")
list, _, e := GetMicroRoadShowMycollectV12(len(roadshowIds)+len(activityvideoIds)+len(activityvoiceIds)+len(askserieVideoIds), 0, audioIdstr, activityVideoIdsStr, roadshowIdsStr, askserieVideoIdsStr, user)
if e != nil {
err = errors.New("GetMicroRoadShowMycollectV12, Err: " + e.Error())
return
}
// 用户权限
authInfo, permissionArr, e := GetUserRaiPermissionInfo(user.UserId, user.CompanyId)
if e != nil {
err = errors.New("GetUserRaiPermissionInfo, Err: " + e.Error())
return
}
// 获取默认图配置
audioMap, videoMap, audioShareMap, videoShareMap, e := GetMicroRoadShowDefaultImgConfig()
if e != nil {
err = errors.New("GetMicroRoadShowDefaultImgConfig, Err: " + e.Error())
return
}
//Source string `description:"资源类型 报告 ：article 、图表 ：newchart、微路演 ：roadshow、活动 ：activity、活动视频：activityvideo、活动音频：activityvoice、专项调研活动：activityspecial"`
for i := range list {
// 权限
au := new(models.UserPermissionAuthInfo)
au.SellerName = authInfo.SellerName
au.SellerMobile = authInfo.SellerMobile
au.HasPermission = authInfo.HasPermission
au.OperationMode = authInfo.OperationMode
if au.HasPermission == 1 {
// 非宏观权限进一步判断是否有权限
if list[i].ChartPermissionId != utils.HONG_GUAN_ID && !utils.InArrayByStr(permissionArr, list[i].ChartPermissionName) {
au.HasPermission = 2
}
}
// 无权限的弹框提示
if au.HasPermission != 1 {
if au.OperationMode == UserPermissionOperationModeCall {
if list[i].Type == 1 {
au.PopupMsg = UserPermissionPopupMsgCallActivity
} else {
au.PopupMsg = UserPermissionPopupMsgCallMicroVideo
}
} else {
if list[i].Type == 1 {
au.PopupMsg = UserPermissionPopupMsgApplyActivity
} else {
au.PopupMsg = UserPermissionPopupMsgApplyMicroVideo
}
}
}
list[i].AuthInfo = au
list[i].PublishTime = utils.StrTimeToTime(list[i].PublishTime).Format(utils.FormatDate)
// 默认图
if list[i].BackgroundImg == "" {
if list[i].Type == 1 {
list[i].BackgroundImg = audioMap[list[i].ChartPermissionId]
} else {
list[i].BackgroundImg = videoMap[list[i].ChartPermissionId]
}
}
// 分享图
if list[i].ShareImg == "" {
if list[i].Type == 1 {
list[i].ShareImg = audioShareMap[list[i].ChartPermissionId]
} else {
list[i].ShareImg = videoShareMap[list[i].ChartPermissionId]
}
}
}
//Type int `description:"类型: 1-音频; 2-活动视频； 3-产业视频"`
for _, item := range list {
if item.Type == 1 {
mapItems[fmt.Sprint("activityvoice", item.Id)].Activityvoice = item
} else if item.Type == 2 {
mapItems[fmt.Sprint("activityvideo", item.Id)].Activityvideo = item
} else if item.Type == 3 {
mapItems[fmt.Sprint("roadshow", item.Id)].Roadshow = item
} else if item.Type == 4 {
mapItems[fmt.Sprint(utils.CYGX_OBJ_ASKSERIEVIDEO, item.Id)].AskserieVideo = item
}
}
}
//处理研选专栏
lenyanxuanSpecialIds := len(yanxuanSpecialIds)
if lenyanxuanSpecialIds > 0 {
pars = make([]interface{}, 0)
condition = ` AND a.id IN (` + utils.GetOrmInReplace(lenyanxuanSpecialIds) + `) `
pars = append(pars, yanxuanSpecialIds)
listyanxuanSpecial, e := models.GetYanxuanSpecialList(user.UserId, condition, pars, 0, 0)
if e != nil {
err = errors.New("GetYanxuanSpecialList, Err: " + e.Error())
return
}
yanxuanSpecialPv := GetYanxuanSpecialRecordByYanxuanSpecialId(yanxuanSpecialIds) // 专栏Pv
for _, v := range listyanxuanSpecial {
v.PublishTime = utils.TimeRemoveHms2(v.PublishTime)
v.Annotation, _ = GetReportContentTextSubNew(v.Content)
v.Pv = yanxuanSpecialPv[v.Id]
if len(yanXuanbodyHighlight[v.Id]) > 0 {
v.BodyHighlight = yanXuanbodyHighlight[v.Id]
} else {
v.BodyHighlight = append(v.BodyHighlight, v.Annotation)
}
v.Annotation = "" //强制置空，兼容前端优先级
v.LabelKeywordImgLink = utils.LABEL_ICO_4
mapItems[fmt.Sprint(utils.CYGX_OBJ_YANXUANSPECIAL, v.Id)].YanxuanSpecial = v
}
}
for _, vList := range list {
for _, v := range mapItems {
//如果这些类型都为空，那么就不合并
if v.Article == nil && v.Newchart == nil && v.Roadshow == nil && v.Activity == nil && v.Activityvideo == nil && v.Activityvoice == nil && v.Activityspecial == nil && v.Researchsummary == nil && v.Minutessummary == nil && v.Meetingreviewchapt == nil && v.ProductInterior == nil && v.IndustrialResource == nil && v.YanxuanSpecial == nil && v.AskserieVideo == nil && v.ReportSelection == nil {
continue
}
if v.SourceId == vList.SourceId && v.Source == vList.Source {
items = append(items, v)
}
}
}
for _, v := range items {
if v.IndustrialResource != nil {
v.Source = "industrialsource"
}
}
return
}
func SqlComprehensiveSearch(user *models.WxUserItem, keyWord string, startSize, pageSize int) (result []*SearchComprehensiveItem, total int, err error) {
yanxuanActivityIds := GetYanxuanActivityIds(user, 0) // 获取所有的研选活动ID
yanxuanArticleIds := GetYanxuanArticleIds() //获取所有研选文章ID
yanxuanArticleIds = append(yanxuanArticleIds, 0)
yanxuanActivityIds = append(yanxuanActivityIds, 0)
var yanxuanArticleIdsStr []string
var yanxuanActivityIdsStr []string
for _, v := range yanxuanArticleIds {
yanxuanArticleIdsStr = append(yanxuanArticleIdsStr, strconv.Itoa(v))
}
for _, v := range yanxuanActivityIds {
yanxuanActivityIdsStr = append(yanxuanActivityIdsStr, strconv.Itoa(v))
}
//yanxuanspecialIds = append(yanxuanspecialIds, 0)
condition := " AND source IN ('article','activity','yanxuanspecial') " // 只有研选的文章、研选的活动、研选的专栏这三种
condition += ` AND IF ( source = 'article' , source_id IN (` + strings.Join(yanxuanArticleIdsStr, ",") + `) ,1=1 ) `
//pars = append(pars, yanxuanArticleIds)
condition += ` AND IF ( source = 'activity' , source_id IN (` + strings.Join(yanxuanActivityIdsStr, ",") + `) ,1=1 ) `
//pars = append(pars, yanxuanActivityIds)
keyWord = "%" + keyWord + "%"
var conditionTitle string
var parsTitle []interface{}
conditionTitle = " AND search_title LIKE ? " + condition
parsTitle = append(parsTitle, keyWord)
totalTitle, e := models.GetResourceDataCount(conditionTitle, parsTitle)
if e != nil {
err = errors.New("GetResourceDataCount, Err: " + e.Error())
return
}
var conditionContent string
var parsContent []interface{}
conditionContent = " AND search_content LIKE ? AND search_title NOT LIKE ? " + condition
parsContent = append(parsContent, keyWord, keyWord)
totalContent, e := models.GetResourceDataCount(conditionContent, parsContent)
if e != nil {
err = errors.New("AddCygxArticleViewRecord, Err: " + e.Error())
return
}
var searchTotal int
searchTotal = (startSize/pageSize + 1) * pageSize
var list []*models.CygxResourceData
fmt.Println("totalTitle", totalTitle)
fmt.Println("totalContent", totalContent)
fmt.Println(searchTotal)
if totalTitle >= searchTotal {
fmt.Println("1")
//全部都是标题搜索
list, e = models.GetResourceDataListCondition(conditionTitle, parsTitle, startSize, pageSize)
if e != nil && e.Error() != utils.ErrNoRow() {
err = errors.New("GetResourceDataListCondition, Err: " + e.Error())
return
}
} else if totalTitle <= searchTotal-pageSize {
fmt.Println("2")
//全部都是内容搜索
startSize = startSize - totalTitle
list, e = models.GetResourceDataListCondition(conditionContent, parsContent, startSize, pageSize)
if e != nil && e.Error() != utils.ErrNoRow() {
err = errors.New("GetResourceDataListCondition, Err: " + e.Error())
return
}
} else {
fmt.Println("3")
//一半标题搜索，一半内容搜索
list, e = models.GetResourceDataListCondition(conditionTitle, parsTitle, startSize, pageSize)
if e != nil && e.Error() != utils.ErrNoRow() {
err = errors.New("GetResourceDataListCondition, Err: " + e.Error())
return
}
listContent, e := models.GetResourceDataListCondition(conditionContent, parsContent, 0, pageSize-totalContent%pageSize)
if e != nil && e.Error() != utils.ErrNoRow() {
err = errors.New("GetResourceDataListCondition, Err: " + e.Error())
return
}
for _, v := range listContent {
list = append(list, v)
}
}
for _, v := range list {
item := new(SearchComprehensiveItem)
item.SourceId = v.SourceId
item.Source = v.Source
result = append(result, item)
}
total = totalTitle + totalContent
return
}
// Es研选专栏
func EsAddYanxuanSpecial(sourceId int) {
var err error
defer func() {
if err != nil {
fmt.Println("err：", err)
go utils.SendAlarmMsg(fmt.Sprint("更新研选专栏失败sourceId: ", sourceId), 2)
}
}()
detail, e := models.GetYanxuanSpecialItemById(sourceId)
if e != nil {
err = errors.New("GetArticleInfoOtherByArticleId" + e.Error())
return
}
content := html.UnescapeString(detail.Content)
doc, e := goquery.NewDocumentFromReader(strings.NewReader(content))
if e != nil {
err = errors.New("goquery.NewDocumentFromReader" + e.Error())
return
}
bodyText := doc.Text()
item := new(ElasticComprehensiveDetail)
item.SourceId = detail.Id
item.Source = utils.CYGX_OBJ_YANXUANSPECIAL
item.Title = detail.Title
item.PublishDate = detail.PublishTime
item.BodyText = bodyText
item.Abstract = bodyText
item.IndustryName = detail.IndustryTags
item.SubjectNames = detail.CompanyTags + detail.Tags
if detail.Status == 3 {
EsAddOrEditComprehensiveData(item) //如果发布了就新增
} else {
EsDeleteComprehensiveData(item) // 没有发布就删除
}
return
}
// 新增和修改数据
func EsAddOrEditComprehensiveData(item *ElasticComprehensiveDetail) (err error) {
indexName := utils.IndexNameComprehensive
//return
defer func() {
if err != nil {
fmt.Println(err, item.SourceId)
//go utils.SendAlarmMsg("更新综合页面数据Es失败"+err.Error()+fmt.Sprint(item), 2)
}
}()
client := utils.Client
mustMap := make([]interface{}, 0)
mustMap = append(mustMap, map[string]interface{}{
"term": map[string]interface{}{
"SourceId": item.SourceId,
},
})
mustMap = append(mustMap, map[string]interface{}{
"term": map[string]interface{}{
"Source": item.Source,
},
})
queryMap := map[string]interface{}{
"query": map[string]interface{}{
"bool": map[string]interface{}{
"must": mustMap,
},
},
}
requestTotalHits := client.Count(indexName).BodyJson(queryMap)
total, e := requestTotalHits.Do(context.Background())
if e != nil {
err = errors.New("requestTotalHits.Do(context.Background()), Err: " + e.Error())
return
}
//return
//根据来源以及ID ，判断内容是否存在，如果存在就新增，如果不存在就修改
if total == 0 {
resp, e := client.Index().Index(indexName).BodyJson(item).Do(context.Background())
if e != nil {
err = errors.New("client.Index().Index(indexName).BodyJson(item).Do(context.Background()), Err: " + e.Error())
return
}
if resp.Status == 0 && resp.Result == "created" {
//fmt.Println("新增成功")
//err = nil
return
} else {
//err = errors.New(fmt.Sprint(resp))
err = errors.New(fmt.Sprint("articleId", item.SourceId))
return
}
} else {
//拼接需要改动的前置条件
bool_query := elastic.NewBoolQuery()
bool_query.Must(elastic.NewTermQuery("SourceId", item.SourceId))
bool_query.Must(elastic.NewTermQuery("Source", item.Source))
//设置需要改动的内容
var script string
script += fmt.Sprint("ctx._source['SubjectNames'] = '", item.SubjectNames, "';")
script += fmt.Sprint("ctx._source['PublishDate'] = '", item.PublishDate, "';")
script += fmt.Sprint("ctx._source['IsSummary'] = ", item.IsSummary, ";")
script += fmt.Sprint("ctx._source['Abstract'] = '", item.Abstract, "';")
script += fmt.Sprint("ctx._source['Title'] = '", item.Title, "';")
script += fmt.Sprint("ctx._source['BodyText'] = '", item.BodyText, "';")
script += fmt.Sprint("ctx._source['Annotation'] = '", item.Annotation, "';")
script += fmt.Sprint("ctx._source['IndustryName'] = '", item.IndustryName, "'")
_, e = client.UpdateByQuery(indexName).
Query(bool_query).
Script(elastic.NewScriptInline(script)).
Refresh("true").
Do(context.Background())
if e != nil && e.Error() != "elastic: Error 400 (Bad Request): compile error [type=script_exception]" {
//文本内容过长的时候，修改会报 400 的错误，暂时先不处理
//fmt.Println("err", e.Error())
////err = e
//err = errors.New("client.UpdateByQuery(indexName), Err: " + e.Error())
return
}
}
return
}
// 删除数据
func EsDeleteComprehensiveData(item *ElasticComprehensiveDetail) (err error) {
defer func() {
if err != nil {
fmt.Println(err)
go utils.SendAlarmMsg("删除数据综合页面数据Es失败"+err.Error()+fmt.Sprint(item), 2)
}
}()
indexName := utils.IndexNameComprehensive
client := utils.Client
//拼接需要删除的前置条件
bool_query := elastic.NewBoolQuery()
bool_query.Must(elastic.NewTermQuery("SourceId", item.SourceId))
bool_query.Must(elastic.NewTermQuery("Source", item.Source))
_, e := client.DeleteByQuery(indexName).
Query(bool_query).
Do(context.Background())
if e != nil {
err = errors.New(" client.DeleteByQuery(indexName), Err: " + e.Error())
return
}
return
}