package services import ( "context" "encoding/json" "fmt" "github.com/PuerkitoBio/goquery" "github.com/olivere/elastic/v7" "hongze/hongze_web_mfyx/models" "hongze/hongze_web_mfyx/utils" "strconv" //"strconv" "errors" "html" "strings" ) type SearchComprehensiveItem struct { SourceId int `description:"资源ID"` IsSummary int `description:"是否是纪要"` Source string `description:"资源类型 报告 ：article 、图表 ：newchart、微路演 ：roadshow、活动 ：activity、活动视频：activityvideo、活动音频：activityvoice、专项调研活动：activityspecial 、 本周研究汇总： researchsummary 、 上周纪要汇总 ：minutessummary 、晨会精华 ：meetingreviewchapt 、 产品内测：productinterior"` Title string `description:"标题"` BodyText string `description:"内容"` PublishDate string `description:"发布时间"` Abstract string `description:"摘要"` Annotation string `description:"核心观点"` IndustryName string `description:"产业名称"` SubjectNames string `description:"标的名称"` Body []string } type ElasticComprehensiveDetail struct { SourceId int `description:"资源ID"` IsSummary int `description:"是否是纪要"` Source string `description:"资源类型 报告 ：article 、图表 ：newchart、微路演 ：roadshow、活动 ：activity、活动视频：activityvideo、活动音频：activityvoice、专项调研活动：activityspecial 、 本周研究汇总： researchsummary 、 上周纪要汇总 ：minutessummary 、晨会精华 ：meetingreviewchapt 、 产品内测：productinterior 、 产业资源包：industrialsource"` Title string `description:"标题"` BodyText string `description:"内容"` PublishDate string `description:"发布时间"` Abstract string `description:"摘要"` Annotation string `description:"核心观点"` IndustryName string `description:"产业名称"` SubjectNames string `description:"标的名称"` } func EsComprehensiveSearch(keyWord string, startSize, pageSize int) (result []*SearchComprehensiveItem, total int64, err error) { indexName := utils.IndexNameComprehensive client := utils.Client keyWordArr, err := GetIndustryMapNameSliceV3(keyWord) keyWordArr = RemoveDuplicatesAndEmpty(keyWordArr) keyWordLen := len(keyWordArr) if keyWordLen <= 0 { keyWordArr = append(keyWordArr, keyWord) keyWordLen = len(keyWordArr) } //Es 的高级查询有 自定义排序 文档一时半会儿撸不懂，先做多次查询手动过滤 2023.2.2 //ikType 查询方式 ，0：查所有 、 1:查询键入词 、 2：查询除了查询键入词之外的联想词 mustMap := make([]interface{}, 0) shouldMap := make(map[string]interface{}, 0) shouldMapquery := make([]interface{}, 0) mustNotMap := make([]interface{}, 0) shouldNotMap := make(map[string]interface{}, 0) shouldNotMapquery := make([]interface{}, 0) // @Param OrderColumn query int true "排序字段 ，Comprehensive综合 ，Matching匹配度 ，PublishDate 发布时间 " //keyWordWeight := GetWeight(keyWordLen) var boost int //lenkeyWordArr := len(keyWordArr) for k, v := range keyWordArr { if k > 0 { continue } if k == 0 { boost = 2 * 1000 } else { boost = 1 } if v != "" { shouldMapquery = append(shouldMapquery, map[string]interface{}{ "function_score": map[string]interface{}{ "query": map[string]interface{}{ "multi_match": map[string]interface{}{ //"boost": (lenkeyWordArr - k) * boost, //给查询的值赋予权重 "boost": boost, //给查询的值赋予权重 "fields": []interface{}{"Title"}, "query": v, }, }, }, }) shouldMapquery = append(shouldMapquery, map[string]interface{}{ "function_score": map[string]interface{}{ "query": map[string]interface{}{ "multi_match": map[string]interface{}{ "boost": boost, //给查询的值赋予权重 "fields": []interface{}{"Abstract"}, "query": v, }, }, }, }) shouldMapquery = append(shouldMapquery, map[string]interface{}{ "function_score": map[string]interface{}{ "query": map[string]interface{}{ "multi_match": map[string]interface{}{ "boost": boost, //给查询的值赋予权重 "fields": []interface{}{"Annotation"}, "query": v, }, }, }, }) //shouldMapquery = append(shouldMapquery, map[string]interface{}{ // "function_score": map[string]interface{}{ // "query": map[string]interface{}{ // "multi_match": map[string]interface{}{ // //"boost": (lenkeyWordArr-k)*boost - 1, //给查询的值赋予权重 // "boost": boost, //给查询的值赋予权重 // "fields": []interface{}{"BodyText"}, // "query": v, // }, // }, // }, //}) shouldMapquery = append(shouldMapquery, map[string]interface{}{ "function_score": map[string]interface{}{ "query": map[string]interface{}{ "multi_match": map[string]interface{}{ //"boost": (lenkeyWordArr-k)*boost - 1, //给查询的值赋予权重 "boost": boost, //给查询的值赋予权重 "fields": []interface{}{"IndustryName"}, "query": v, }, }, }, }) shouldMapquery = append(shouldMapquery, map[string]interface{}{ "function_score": map[string]interface{}{ "query": map[string]interface{}{ "multi_match": map[string]interface{}{ //"boost": (lenkeyWordArr-k)*boost - 1, //给查询的值赋予权重 "boost": boost, //给查询的值赋予权重 "fields": []interface{}{"SubjectNames"}, "query": v, }, }, }, }) } } shouldMap = map[string]interface{}{ "should": shouldMapquery, } shouldNotMap = map[string]interface{}{ "should": shouldNotMapquery, } //排序 sortMap := make([]interface{}, 0) //时间 sortMap = append(sortMap, map[string]interface{}{ "PublishDate": map[string]interface{}{ "order": "desc", }, }) //sortMap = append(sortMap, map[string]interface{}{ // "_score": map[string]interface{}{ // "order": "desc", // }, //}) //高亮 highlightMap := make(map[string]interface{}, 0) highlightMap = map[string]interface{}{ "fields": map[string]interface{}{ //"BodyText": map[string]interface{}{}, "Title": map[string]interface{}{}, "Abstract": map[string]interface{}{}, "Annotation": map[string]interface{}{}, }, //样式 红色 "post_tags": []interface{}{""}, "pre_tags": []interface{}{""}, "fragment_size": 50, } mustMap = append(mustMap, map[string]interface{}{ "bool": shouldMap, }) mustNotMap = append(mustNotMap, map[string]interface{}{ "bool": shouldNotMap, }) queryMap := map[string]interface{}{ "query": map[string]interface{}{ "bool": map[string]interface{}{ "must": mustMap, }, }, } queryMap["sort"] = sortMap queryMap["from"] = startSize queryMap["size"] = pageSize queryMap["highlight"] = highlightMap //jsonBytes, _ := json.Marshal(queryMap) //fmt.Println(string(jsonBytes)) //utils.FileLog.Info(string(jsonBytes)) request := client.Search(indexName).Source(queryMap) // sets the JSON request searchByMatch, err := request.Do(context.Background()) if searchByMatch != nil { if searchByMatch.Hits != nil { for _, v := range searchByMatch.Hits.Hits { var isAppend bool articleJson, err := v.Source.MarshalJSON() if err != nil { return nil, 0, err } article := new(ElasticComprehensiveDetail) err = json.Unmarshal(articleJson, &article) if err != nil { return nil, 0, err } //fmt.Println(article.SourceId, article.Title, article.Source) searchItem := new(SearchComprehensiveItem) searchItem.SourceId = article.SourceId if len(v.Highlight["Annotation"]) > 0 { for _, vText := range v.Highlight["Annotation"] { searchItem.Body = append(searchItem.Body, vText) } } if len(v.Highlight["Abstract"]) > 0 { for _, vText := range v.Highlight["Abstract"] { searchItem.Body = append(searchItem.Body, vText) } } if len(v.Highlight["BodyText"]) > 0 { for _, vText := range v.Highlight["BodyText"] { searchItem.Body = append(searchItem.Body, vText) } } //searchItem.IsSummary = article.IsSummary //if len(searchItem.Body) == 0 { // bodyRune := []rune(article.BodyText) // bodyRuneLen := len(bodyRune) // if bodyRuneLen > 100 { // bodyRuneLen = 100 // } // body := string(bodyRune[:bodyRuneLen]) // searchItem.Body = []string{body} //} var title string if len(v.Highlight["Title"]) > 0 { title = v.Highlight["Title"][0] } else { title = article.Title } searchItem.Title = title searchItem.Source = article.Source searchItem.PublishDate = article.PublishDate if !isAppend { result = append(result, searchItem) } } } total = searchByMatch.Hits.TotalHits.Value } return } func GetResourceDataEsList(list []*SearchComprehensiveItem, user *models.WxUserItem) (items []*models.CygxResourceDataNewResp, err error) { var condition string var pars []interface{} uid := user.UserId titleHighlight := make(map[int]string) bodyHighlight := make(map[int][]string) yanXuanbodyHighlight := make(map[int][]string) mapItems := make(map[string]*models.CygxResourceDataNewResp) for _, v := range list { //预处理文章 item := new(models.CygxResourceDataNewResp) item.SourceId = v.SourceId item.Source = v.Source item.PublishDate = utils.TimeRemoveHms2(v.PublishDate) item.BodyHighlight = v.Body item.TitleHighlight = v.Title titleHighlight[v.SourceId] = v.Title bodyHighlight[v.SourceId] = v.Body mapItems[fmt.Sprint(v.Source, v.SourceId)] = item } var articleIds []int //报告 var activityIds []int //活动 var yanxuanSpecialIds []int // 研选专栏 var yanxuanspecialauthorIds []int //研选作者ID var activityvideoIds []string // 活动视频 var activityvoiceIds []string //活动音频 var roadshowIds []string //微路演 var askserieVideoIds []string //问答系列视频 //Source string `description:"资源类型 报告 ：article 、图表 ：newchart、微路演 ：roadshow、活动 ：activity、活动视频：activityvideo、活动音频：activityvoice、专项调研活动：activityspecial 、 本周研究汇总： researchsummary 、 上周纪要汇总 ：minutessummary 、晨会精华 ：meetingreviewchapt 、 产品内测：productinterior for _, v := range list { if v.Source == "article" { articleIds = append(articleIds, v.SourceId) } else if v.Source == "activity" { activityIds = append(activityIds, v.SourceId) } else if v.Source == "activityvideo" { activityvideoIds = append(activityvideoIds, strconv.Itoa(v.SourceId)) } else if v.Source == "activityvoice" { activityvoiceIds = append(activityvoiceIds, strconv.Itoa(v.SourceId)) } else if v.Source == utils.CYGX_OBJ_YANXUANSPECIAL { yanxuanSpecialIds = append(yanxuanSpecialIds, v.SourceId) } else if v.Source == utils.CYGX_OBJ_YANXUANSPECIAL_AUTHOR { yanxuanspecialauthorIds = append(yanxuanspecialauthorIds, v.SourceId) } } detail, e := models.GetConfigByCode("city_img_url") if e != nil { err = errors.New("GetResourceDataList, Err: " + e.Error()) return } detailChart, e := models.GetConfigByCode("chart_img_url") if e != nil { err = errors.New("GetResourceDataList, Err: " + e.Error()) return } addressList := strings.Split(detail.ConfigValue, "{|}") mapAddress := make(map[string]string) chartList := strings.Split(detailChart.ConfigValue, "{|}") mapChart := make(map[string]string) var cityName string var chartName string var imgUrl string var imgUrlChart string for _, v := range addressList { vslice := strings.Split(v, "_") cityName = vslice[0] imgUrl = vslice[len(vslice)-1] mapAddress[cityName] = imgUrl } for _, v := range chartList { vslice := strings.Split(v, "_") chartName = vslice[0] imgUrlChart = vslice[len(vslice)-1] mapChart[chartName] = imgUrlChart } var imgUrlResp string //处理文章 if len(articleIds) > 0 { pars = make([]interface{}, 0) condition = ` AND a.article_id IN (` + utils.GetOrmInReplace(len(articleIds)) + `)` pars = append(pars, articleIds) articleList, e := models.GetHomeListPublic(condition, pars, 0, len(articleIds)) if e != nil { err = errors.New("GetResourceDataList, Err: " + e.Error()) return } articleList, e = HandleArticleCategoryImg(articleList, user) if e != nil { err = errors.New("HandleArticleCategoryImg, Err: " + e.Error()) return } for _, v := range articleList { v.Body = "" if titleHighlight[v.ArticleId] != "" { v.Title = titleHighlight[v.ArticleId] } if len(bodyHighlight[v.ArticleId]) > 0 { v.Abstract = "" v.Annotation = "" v.BodyHighlight = bodyHighlight[v.ArticleId] } else { v.BodyHighlight = make([]string, 0) } mapItems[fmt.Sprint("article", v.ArticleId)].Article = v } } //处理活动 if len(activityIds) > 0 { for _, vss := range activityIds { imgUrlResp += strconv.Itoa(vss) + "," } pars = make([]interface{}, 0) condition = ` AND art.activity_id IN (` + utils.GetOrmInReplace(len(activityIds)) + `) ` pars = append(pars, activityIds) activityList, e := models.GetActivityListNew(condition, pars, uid, 0, len(activityIds), 0, 0, "") if e != nil { err = errors.New("GetResourceDataList, Err: " + e.Error()) return } var activityListRersp []*models.ActivityListResp //for _, v := range activityList { // activityListRersp = append(activityListRersp, ActivityButtonShow(v, user, make([]string, 0))) //} //var pArr []string activityListRersp = ActivityArrButtonShow(activityList, user, make([]string, 0)) //处理不同的报名方式按钮回显 mapActivitySignup, e := GetActivitySignupResp(activityIds, user) if e != nil { err = errors.New("GetActivitySignupResp, Err: " + e.Error()) return } for _, v := range activityListRersp { if v == nil { continue } if v.ActivityType == 0 { if mapAddress[v.City] != "" { imgUrlResp = mapAddress[v.City] } else { imgUrlResp = mapAddress["其它"] } } else { if mapChart[v.ChartPermissionName] != "" { imgUrlResp = mapChart[v.ChartPermissionName] } } v.ImgUrl = imgUrlResp v.SourceType = mapActivitySignup[v.ActivityId] mapItems[fmt.Sprint("activity", v.ActivityId)].Activity = v } } //处理研选专栏 lenyanxuanSpecialIds := len(yanxuanSpecialIds) if lenyanxuanSpecialIds > 0 { pars = make([]interface{}, 0) condition = ` AND a.id IN (` + utils.GetOrmInReplace(lenyanxuanSpecialIds) + `) ` pars = append(pars, yanxuanSpecialIds) listyanxuanSpecial, e := models.GetYanxuanSpecialList(user.UserId, condition, pars, 0, 0) if e != nil { err = errors.New("GetYanxuanSpecialList, Err: " + e.Error()) return } yanxuanSpecialPv := GetYanxuanSpecialRecordByYanxuanSpecialId(yanxuanSpecialIds) // 专栏Pv for _, v := range listyanxuanSpecial { v.PublishTime = utils.TimeRemoveHms2(v.PublishTime) v.Annotation, _ = GetReportContentTextSubNew(v.Content) v.Pv = yanxuanSpecialPv[v.Id] if len(yanXuanbodyHighlight[v.Id]) > 0 { v.BodyHighlight = yanXuanbodyHighlight[v.Id] } else { v.BodyHighlight = append(v.BodyHighlight, v.Annotation) } v.Annotation = "" //强制置空，兼容前端优先级 v.LabelKeywordImgLink = utils.LABEL_ICO_4 mapItems[fmt.Sprint(utils.CYGX_OBJ_YANXUANSPECIAL, v.Id)].YanxuanSpecial = v } } //处理研选专栏作者 lenyanxuanspecialauthorIds := len(yanxuanspecialauthorIds) if lenyanxuanspecialauthorIds > 0 { pars = make([]interface{}, 0) condition = ` AND a.id IN (` + utils.GetOrmInReplace(lenyanxuanspecialauthorIds) + `) ` condition += ` ORDER BY latest_publish_time DESC ` pars = append(pars, yanxuanspecialauthorIds) listAuthor, e := models.GetYanxuanSpecialAuthorList("", condition, pars, 0, lenyanxuanspecialauthorIds) if e != nil { err = errors.New("GetYanxuanSpecialAuthorList, Err: " + e.Error()) return } for _, v := range listAuthor { v.LatestPublishDate = v.LatestPublishTime.Format(utils.FormatDate) v.LabelKeywordImgLink = utils.LABEL_ICO_9 mapItems[fmt.Sprint(utils.CYGX_OBJ_YANXUANSPECIAL_AUTHOR, v.Id)].YanxuanSpecialAuthor = v } } if len(roadshowIds)+len(activityvideoIds)+len(activityvoiceIds)+len(askserieVideoIds) > 0 { audioIdstr := strings.Join(activityvoiceIds, ",") activityVideoIdsStr := strings.Join(activityvideoIds, ",") roadshowIdsStr := strings.Join(roadshowIds, ",") askserieVideoIdsStr := strings.Join(askserieVideoIds, ",") listv, _, e := GetMicroRoadShowMycollectV12(len(roadshowIds)+len(activityvideoIds)+len(activityvoiceIds)+len(askserieVideoIds), 0, audioIdstr, activityVideoIdsStr, roadshowIdsStr, askserieVideoIdsStr, user) if e != nil { err = errors.New("GetMicroRoadShowMycollect, Err: " + e.Error()) return } // 用户权限 authInfo, permissionArr, e := GetUserRaiPermissionInfo(user.UserId, user.CompanyId) if e != nil { err = errors.New("GetUserRaiPermissionInfo, Err: " + e.Error()) return } // 获取默认图配置 audioMap, videoMap, audioShareMap, videoShareMap, e := GetMicroRoadShowDefaultImgConfig() if e != nil { err = errors.New("GetMicroRoadShowDefaultImgConfig, Err: " + e.Error()) return } //Source string `description:"资源类型 报告 ：article 、图表 ：newchart、微路演 ：roadshow、活动 ：activity、活动视频：activityvideo、活动音频：activityvoice、专项调研活动：activityspecial"` for i := range listv { // 权限 au := new(models.UserPermissionAuthInfo) au.SellerName = authInfo.SellerName au.SellerMobile = authInfo.SellerMobile au.HasPermission = authInfo.HasPermission au.OperationMode = authInfo.OperationMode if au.HasPermission == 1 { // 非宏观权限进一步判断是否有权限 if listv[i].ChartPermissionId != utils.HONG_GUAN_ID && !utils.InArrayByStr(permissionArr, listv[i].ChartPermissionName) { au.HasPermission = 2 } } // 无权限的弹框提示 if au.HasPermission != 1 { if au.OperationMode == UserPermissionOperationModeCall { if listv[i].Type == 1 { au.PopupMsg = UserPermissionPopupMsgCallActivity } else { au.PopupMsg = UserPermissionPopupMsgCallMicroVideo } } else { if listv[i].Type == 1 { au.PopupMsg = UserPermissionPopupMsgApplyActivity } else { au.PopupMsg = UserPermissionPopupMsgApplyMicroVideo } } } listv[i].AuthInfo = au listv[i].PublishTime = utils.StrTimeToTime(listv[i].PublishTime).Format(utils.FormatDate) // 默认图 if listv[i].BackgroundImg == "" { if listv[i].Type == 1 { listv[i].BackgroundImg = audioMap[listv[i].ChartPermissionId] } else { listv[i].BackgroundImg = videoMap[listv[i].ChartPermissionId] } } // 分享图 if listv[i].ShareImg == "" { if listv[i].Type == 1 { listv[i].ShareImg = audioShareMap[listv[i].ChartPermissionId] } else { listv[i].ShareImg = videoShareMap[listv[i].ChartPermissionId] } } listv[i].LabelKeywordImgLink = utils.LABEL_ICO_10 } //Type int `description:"类型: 1-音频; 2-活动视频； 3-产业视频"` for _, item := range listv { if item.Type == 1 { mapItems[fmt.Sprint("activityvoice", item.Id)].Activityvoice = item } else if item.Type == 2 { mapItems[fmt.Sprint("activityvideo", item.Id)].Activityvideo = item } else if item.Type == 3 { mapItems[fmt.Sprint("roadshow", item.Id)].Roadshow = item } else if item.Type == 4 { mapItems[fmt.Sprint(utils.CYGX_OBJ_ASKSERIEVIDEO, item.Id)].AskserieVideo = item } } } for _, vList := range list { for _, v := range mapItems { //如果这些类型都为空，那么就不合并 if v.Article == nil && v.Newchart == nil && v.Roadshow == nil && v.Activity == nil && v.Activityvideo == nil && v.Activityvoice == nil && v.YanxuanSpecial == nil && v.AskserieVideo == nil { continue } if v.SourceId == vList.SourceId && v.Source == vList.Source { items = append(items, v) } } } return } func SqlComprehensiveSearch(user *models.WxUserItem, keyWord string, startSize, pageSize int) (result []*SearchComprehensiveItem, total int, err error) { yanxuanActivityIds := GetYanxuanActivityIds(user, "") // 获取所有的研选活动ID yanxuanArticleIds := GetYanxuanArticleIds() //获取所有研选文章ID yanxuanArticleIds = append(yanxuanArticleIds, 0) yanxuanActivityIds = append(yanxuanActivityIds, 0) var yanxuanArticleIdsStr []string var yanxuanActivityIdsStr []string for _, v := range yanxuanArticleIds { yanxuanArticleIdsStr = append(yanxuanArticleIdsStr, strconv.Itoa(v)) } for _, v := range yanxuanActivityIds { yanxuanActivityIdsStr = append(yanxuanActivityIdsStr, strconv.Itoa(v)) } //yanxuanspecialIds = append(yanxuanspecialIds, 0) condition := " AND source IN ('article','activity','yanxuanspecial','activityvoice','activityvideo') AND IF ( source IN('activityvoice','activityvideo') , chart_permission_id = 31 ,1=1 ) " // 只有研选的文章、研选的活动、研选的专栏这三种 condition += ` AND IF ( source = 'article' , source_id IN (` + strings.Join(yanxuanArticleIdsStr, ",") + `) ,1=1 ) ` //pars = append(pars, yanxuanArticleIds) condition += ` AND IF ( source = 'activity' , source_id IN (` + strings.Join(yanxuanActivityIdsStr, ",") + `) ,1=1 ) ` //pars = append(pars, yanxuanActivityIds) keyWord = "%" + keyWord + "%" var conditionTitle string var parsTitle []interface{} conditionTitle = " AND search_title LIKE ? " + condition parsTitle = append(parsTitle, keyWord) var conditionContentYxAuthor string // 研选专栏作者搜索条件 var parsContentYxAuthor []interface{} conditionContentYxAuthor = " AND (special_name LIKE ? OR nick_name LIKE ? ) " parsContentYxAuthor = append(parsContentYxAuthor, keyWord, keyWord) totalTitle, e := models.GetResourceDataAndYanxuanSpecialAuthorCount(conditionTitle, parsTitle, conditionContentYxAuthor, parsContentYxAuthor) //totalTitle, e := models.GetResourceDataCount(conditionTitle, parsTitle) if e != nil { err = errors.New("GetResourceDataAndYanxuanSpecialAuthorCount, Err: " + e.Error()) return } var conditionContent string var parsContent []interface{} conditionContent = " AND search_content LIKE ? AND search_title NOT LIKE ? " + condition parsContent = append(parsContent, keyWord, keyWord) totalContent, e := models.GetResourceDataCount(conditionContent, parsContent) if e != nil { err = errors.New("AddCygxArticleViewRecord, Err: " + e.Error()) return } var searchTotal int searchTotal = (startSize/pageSize + 1) * pageSize var list []*models.CygxResourceData if totalTitle >= searchTotal { //全部都是标题搜索 list, e = models.GetResourceDataAndYanxuanSpecialAuthorListCondition(conditionTitle, parsTitle, conditionContentYxAuthor, parsContentYxAuthor, startSize, pageSize) if e != nil && e.Error() != utils.ErrNoRow() { err = errors.New("GetResourceDataListCondition, Err: " + e.Error()) return } } else if totalTitle <= searchTotal-pageSize { //全部都是内容搜索 startSize = startSize - totalTitle list, e = models.GetResourceDataListCondition(conditionContent, parsContent, startSize, pageSize) if e != nil && e.Error() != utils.ErrNoRow() { err = errors.New("GetResourceDataListCondition, Err: " + e.Error()) return } } else { //一半标题搜索，一半内容搜索 //list, e = models.GetResourceDataListCondition(conditionTitle, parsTitle, startSize, pageSize) list, e = models.GetResourceDataAndYanxuanSpecialAuthorListCondition(conditionTitle, parsTitle, conditionContentYxAuthor, parsContentYxAuthor, startSize, pageSize) if e != nil && e.Error() != utils.ErrNoRow() { err = errors.New("GetResourceDataAndYanxuanSpecialAuthorListCondition, Err: " + e.Error()) return } listContent, e := models.GetResourceDataListCondition(conditionContent, parsContent, 0, pageSize-totalContent%pageSize) if e != nil && e.Error() != utils.ErrNoRow() { err = errors.New("GetResourceDataListCondition, Err: " + e.Error()) return } for _, v := range listContent { list = append(list, v) } } for _, v := range list { item := new(SearchComprehensiveItem) item.SourceId = v.SourceId item.Source = v.Source result = append(result, item) } total = totalTitle + totalContent return } // Es研选专栏 func EsAddYanxuanSpecial(sourceId int) { var err error defer func() { if err != nil { fmt.Println("err：", err) go utils.SendAlarmMsg(fmt.Sprint("更新研选专栏失败sourceId: ", sourceId), 2) } }() detail, e := models.GetYanxuanSpecialItemById(sourceId) if e != nil { err = errors.New("GetArticleInfoOtherByArticleId" + e.Error()) return } content := html.UnescapeString(detail.Content) doc, e := goquery.NewDocumentFromReader(strings.NewReader(content)) if e != nil { err = errors.New("goquery.NewDocumentFromReader" + e.Error()) return } bodyText := doc.Text() item := new(ElasticComprehensiveDetail) item.SourceId = detail.Id item.Source = utils.CYGX_OBJ_YANXUANSPECIAL item.Title = detail.Title item.PublishDate = detail.PublishTime item.BodyText = bodyText item.Abstract = bodyText item.IndustryName = detail.IndustryTags item.SubjectNames = detail.CompanyTags + detail.Tags if detail.Status == 3 { EsAddOrEditComprehensiveData(item) //如果发布了就新增 } else { EsDeleteComprehensiveData(item) // 没有发布就删除 } return } // 新增和修改数据 func EsAddOrEditComprehensiveData(item *ElasticComprehensiveDetail) (err error) { indexName := utils.IndexNameComprehensive //return defer func() { if err != nil { fmt.Println(err, item.SourceId) //go utils.SendAlarmMsg("更新综合页面数据Es失败"+err.Error()+fmt.Sprint(item), 2) } }() client := utils.Client mustMap := make([]interface{}, 0) mustMap = append(mustMap, map[string]interface{}{ "term": map[string]interface{}{ "SourceId": item.SourceId, }, }) mustMap = append(mustMap, map[string]interface{}{ "term": map[string]interface{}{ "Source": item.Source, }, }) queryMap := map[string]interface{}{ "query": map[string]interface{}{ "bool": map[string]interface{}{ "must": mustMap, }, }, } requestTotalHits := client.Count(indexName).BodyJson(queryMap) total, e := requestTotalHits.Do(context.Background()) if e != nil { err = errors.New("requestTotalHits.Do(context.Background()), Err: " + e.Error()) return } //return //根据来源以及ID ，判断内容是否存在，如果存在就新增，如果不存在就修改 if total == 0 { resp, e := client.Index().Index(indexName).BodyJson(item).Do(context.Background()) if e != nil { err = errors.New("client.Index().Index(indexName).BodyJson(item).Do(context.Background()), Err: " + e.Error()) return } if resp.Status == 0 && resp.Result == "created" { //fmt.Println("新增成功") //err = nil return } else { //err = errors.New(fmt.Sprint(resp)) err = errors.New(fmt.Sprint("articleId", item.SourceId)) return } } else { //拼接需要改动的前置条件 bool_query := elastic.NewBoolQuery() bool_query.Must(elastic.NewTermQuery("SourceId", item.SourceId)) bool_query.Must(elastic.NewTermQuery("Source", item.Source)) //设置需要改动的内容 var script string script += fmt.Sprint("ctx._source['SubjectNames'] = '", item.SubjectNames, "';") script += fmt.Sprint("ctx._source['PublishDate'] = '", item.PublishDate, "';") script += fmt.Sprint("ctx._source['IsSummary'] = ", item.IsSummary, ";") script += fmt.Sprint("ctx._source['Abstract'] = '", item.Abstract, "';") script += fmt.Sprint("ctx._source['Title'] = '", item.Title, "';") script += fmt.Sprint("ctx._source['BodyText'] = '", item.BodyText, "';") script += fmt.Sprint("ctx._source['Annotation'] = '", item.Annotation, "';") script += fmt.Sprint("ctx._source['IndustryName'] = '", item.IndustryName, "'") _, e = client.UpdateByQuery(indexName). Query(bool_query). Script(elastic.NewScriptInline(script)). Refresh("true"). Do(context.Background()) if e != nil && e.Error() != "elastic: Error 400 (Bad Request): compile error [type=script_exception]" { //文本内容过长的时候，修改会报 400 的错误，暂时先不处理 //fmt.Println("err", e.Error()) ////err = e //err = errors.New("client.UpdateByQuery(indexName), Err: " + e.Error()) return } } return } // 删除数据 func EsDeleteComprehensiveData(item *ElasticComprehensiveDetail) (err error) { defer func() { if err != nil { fmt.Println(err) go utils.SendAlarmMsg("删除数据综合页面数据Es失败"+err.Error()+fmt.Sprint(item), 2) } }() indexName := utils.IndexNameComprehensive client := utils.Client //拼接需要删除的前置条件 bool_query := elastic.NewBoolQuery() bool_query.Must(elastic.NewTermQuery("SourceId", item.SourceId)) bool_query.Must(elastic.NewTermQuery("Source", item.Source)) _, e := client.DeleteByQuery(indexName). Query(bool_query). Do(context.Background()) if e != nil { err = errors.New(" client.DeleteByQuery(indexName), Err: " + e.Error()) return } return }