hongze_cygx/services/elasticsearch.go

package services

import (
	"context"
	"encoding/json"
	"fmt"
	"hongze/hongze_cygx/models"
	"hongze/hongze_cygx/utils"
	"html"
	"strconv"
	"strings"
	"github.com/PuerkitoBio/goquery"
	"github.com/olivere/elastic/v7"
	"github.com/olivere/elastic/v7/config"
)

const (
	ES_URL      = "http://es-cn-nif227b580019rgw6.public.elasticsearch.aliyuncs.com:9200" //<1>
	ES_USERNAME = "elastic"                                                               //<2>
	ES_PASSWORD = "hongze@2021"                                                           //<3>
	//Grafana pwd-> 20521bb9
	//Grafana username-> emon
)

func SaveData() {
	//fmt.Println("start")
	var sniff = false //<4>
	cfg := &config.Config{
		URL:      ES_URL,
		Username: ES_USERNAME,
		Password: ES_PASSWORD,
	}

	cfg.Sniff = &sniff
	var client, err = elastic.NewClientFromConfig(cfg)
	if err != nil {
		fmt.Println("NewClientFromConfig Err:" + err.Error())
		return
	}
	var esIndex = "cygx_article"
	//var esType = "article"
	//
	exists, err := client.IndexExists(esIndex).Do(context.Background()) //<5>
	if err != nil {
		fmt.Println("IndexExists Err:" + err.Error())
		return
	}
	if !exists {
		_, err = client.CreateIndex(esIndex).Do(context.Background())
		if err != nil {
			fmt.Println("CreateIndex Err:" + err.Error())
			return
		}
	}

	/*
		3161,3190,3226,3244,3264,3285,3310,3334,3370,3397,3418,3446,3477,3497,3526,3554
	*/

	idStr := `3584,3644`
	idArr := strings.Split(idStr, ",")
	for _, v := range idArr {
		id, _ := strconv.Atoi(v)
		item, err := models.GetArticleDetailById(id)
		if err != nil {
			fmt.Println("GetArticleDetailById Err:" + err.Error())
			return
		}
		content := html.UnescapeString(item.Body)
		doc, err := goquery.NewDocumentFromReader(strings.NewReader(content))
		if err != nil {
			fmt.Println("create doc err:", err.Error())
			return
		}
		bodyText := doc.Text()
		item.BodyText = bodyText
		//新增
		resp, err := client.Index().Index(esIndex).Id(strconv.Itoa(item.ArticleId)).BodyJson(item).Do(context.Background())
		if err != nil {
			fmt.Println("insert es failed", err.Error())
			return
		}
		fmt.Println(resp.Status)
	}

	/*

		//根据id查询
		searchById, err := client.Get().Index(esIndex).Type(esType).Id("3138").Do(context.Background())
		if searchById.Found {
			body, err := searchById.Source.MarshalJSON()
			fmt.Println("body:",string(body))
			fmt.Println(err)
			//var resultType models.ArticleDetail
			//if err := json.Unmarshal(searchById.Source,&resultType); err != nil{
			//	log.Error(err.Error())
			//}
			//fmt.Printf("search by id: %#v \n",resultType)
		}
	*/

	////查询index中所有的数据
	//var resultType models.CygxArticle
	//searchAll,err := client.Search(esIndex).Type(esType).Do(context.Background())
	//for _,item := range searchAll.Each(reflect.TypeOf(resultType)) {
	//	language := item.(models.CygxArticle)
	//	fmt.Printf("search by index all: %#v \n",language)
	//}

	//根据检索条件查询

	// boolquery := elastic.NewBoolQuery()

	// boolquery.Should(elastic.NewMatchQuery("Body", "专家"))
	// highlight := elastic.NewHighlight()
	// highlight = highlight.Fields(elastic.NewHighlighterField("Body"))
	// highlight = highlight.PreTags("<font color='red'>").PostTags("</font>")

	// var pageSize int
	// pageSize = 20

	// searchByMatch, err := client.Search(esIndex).Highlight(highlight).Size(pageSize).Query(boolquery).Do(context.Background())

	// var result string
	// if searchByMatch.Hits != nil {

	// }
	// //fmt.Println(string(result))

	// utils.FileLog.Info("%s", string(result))

	//var resultType models.CygxArticle
	//for k,item := range searchByMatch.Each(reflect.TypeOf(resultType)) {
	//	language := item.(models.CygxArticle)
	//	fmt.Printf("search by match: %#v \n",language)
	//
	//	fmt.Println(k)
	//	result,err:=json.Marshal(language)
	//	fmt.Println(err)

	//	utils.FileLog.Info("%s",string(result))
	//}

	// Perform the search request.
	//searchByMatch, err := client.Search(esIndex).Type(esType).Query(query).From(1).Size(10).Do(context.Background())

	fmt.Println("end")
}

func SearchByKeyWord(keyWord string) (result []*models.SearchItem, err error) {
	fmt.Println("keyWord:",keyWord)
	pageSize := 20
	keyWordArr, err := GetIndustryMapNameSlice(keyWord)
	keyWordArr=RemoveDuplicatesAndEmpty(keyWordArr)

	fmt.Println(keyWordArr)
	fmt.Println(" keyWordArr ")

	if err != nil {
		go utils.SendEmail(utils.APPNAME+" "+utils.RunMode+"异常提醒:", "GetIndustryMapNameSlice："+err.Error(), utils.EmailSendToUsers)
	}
	var sniff = false //<4>
	cfg := &config.Config{
		URL:      ES_URL,
		Username: ES_USERNAME,
		Password: ES_PASSWORD,
	}

	cfg.Sniff = &sniff
	client, err := elastic.NewClientFromConfig(cfg)
	if err != nil {
		return
	}
	var esIndex = "cygx_article"

	searchMap := make(map[int]int)
	//boolquery := elastic.NewBoolQuery()
	//keyWordLen := len(keyWordArr)
	//n := keyWordLen
	//for _, v := range keyWordArr {
	//	keyWord = v
	//	boost := float64(n)
	//	fmt.Println("keyWord:", keyWord)
	//	fmt.Println("boost:", boost)
	//	//matchQueryList = append(matchQueryList, elastic.NewMatchQuery("Title", keyWord).Boost(boost+0.2))
	//	//matchQueryList = append(matchQueryList, elastic.NewMatchQuery("BodyText", keyWord).Boost(boost+0.1))
	//	q1 := elastic.NewMatchQuery("Title", keyWord)
	//	q1 = q1.Boost(boost + 0.2)
	//	q2 := elastic.NewMatchQuery("BodyText", keyWord)
	//	q2 = q2.Boost(boost + 0.1)
	//	boolquery.Should(q1, q2)
	//	n--
	//}
	keyWordStr := strings.Join(keyWordArr, ",")

	fmt.Println("keyWordStr ",keyWordStr)

	queryString := elastic.NewQueryStringQuery(`Title:(` + keyWordStr + `) BodyText:(`+keyWordStr+`)`).Analyzer("ik_smart")

	boolqueryJson, err := json.Marshal(queryString)
	fmt.Println("err:", err)
	fmt.Println("queryString ", string(boolqueryJson))

	//boolquery.Must(elastic.NewMatchQuery("Title", keyWord), elastic.NewMatchQuery("BodyText", keyWord))
	highlight := elastic.NewHighlight()
	highlight = highlight.Fields(elastic.NewHighlighterField("Title"), elastic.NewHighlighterField("BodyText"))
	highlight = highlight.PreTags("<font color='red'>").PostTags("</font>")
	request := client.Search(esIndex).Highlight(highlight).Size(pageSize).Query(queryString)

	requestJson, err := json.Marshal(request)
	fmt.Println("err:", err)
	fmt.Println("requestJson ", string(requestJson))

	searchByMatch, err := request.Do(context.Background())

	//searchByMatch, err := client.Search(esIndex).Highlight(highlight).Size(pageSize).Do(context.Background())
	if err != nil {
		return result, err
	}
	if searchByMatch.Hits != nil {
		for _, v := range searchByMatch.Hits.Hits {
			articleJson, err := v.Source.MarshalJSON()
			if err != nil {
				return nil, err
			}
			article := new(models.CygxArticle)
			err = json.Unmarshal(articleJson, &article)
			if err != nil {
				return nil, err
			}
			if _, ok := searchMap[article.ArticleId]; !ok {
				searchItem := new(models.SearchItem)
				searchItem.ArticleId, _ = strconv.Atoi(v.Id)
				searchItem.Body = v.Highlight["BodyText"]
				var title string
				if len(v.Highlight["Title"]) > 0 {
					title = v.Highlight["Title"][0]
				} else {
					title = article.Title
				}
				searchItem.Title = title
				searchItem.PublishDate = article.PublishDate
				result = append(result, searchItem)
				searchMap[article.ArticleId] = article.ArticleId
			}
		}
	}
	return
}

func SearchByKeyWordBack(keyWord string) (result []*models.SearchItem, err error) {
	pageSize := 20
	keyWordArr, err := GetIndustryMapNameSlice(keyWord)
	if err != nil {
		go utils.SendEmail(utils.APPNAME+" "+utils.RunMode+"异常提醒:", "GetIndustryMapNameSlice："+err.Error(), utils.EmailSendToUsers)
	}
	var sniff = false //<4>
	cfg := &config.Config{
		URL:      ES_URL,
		Username: ES_USERNAME,
		Password: ES_PASSWORD,
	}

	cfg.Sniff = &sniff
	client, err := elastic.NewClientFromConfig(cfg)
	if err != nil {
		return
	}
	var esIndex = "cygx_article"

	searchMap := make(map[int]int)
	for _, v := range keyWordArr {
		keyWord = v
		boolquery := elastic.NewBoolQuery()
		boolquery.Must(elastic.NewMatchQuery("Title", keyWord), elastic.NewMatchQuery("BodyText", keyWord))
		highlight := elastic.NewHighlight()
		highlight = highlight.Fields(elastic.NewHighlighterField("Title"), elastic.NewHighlighterField("BodyText"))
		highlight = highlight.PreTags("<font color='red'>").PostTags("</font>")
		searchByMatch, err := client.Search(esIndex).Highlight(highlight).Size(pageSize).Query(boolquery).Do(context.Background())
		if err != nil {
			return result, err
		}
		if searchByMatch.Hits != nil {
			for _, v := range searchByMatch.Hits.Hits {
				articleJson, err := v.Source.MarshalJSON()
				if err != nil {
					return nil, err
				}
				article := new(models.CygxArticle)
				err = json.Unmarshal(articleJson, &article)
				if err != nil {
					return nil, err
				}
				if _, ok := searchMap[article.ArticleId]; !ok {
					searchItem := new(models.SearchItem)
					searchItem.ArticleId, _ = strconv.Atoi(v.Id)
					searchItem.Body = v.Highlight["BodyText"]
					var title string
					if len(v.Highlight["Title"]) > 0 {
						title = v.Highlight["Title"][0]
					} else {
						title = article.Title
					}
					searchItem.Title = title
					searchItem.PublishDate = article.PublishDate
					result = append(result, searchItem)
					searchMap[article.ArticleId] = article.ArticleId
				}
			}
		}
	}
	return
}

func esSearch(keyWord, categoryName string) (result []*models.SearchItem, err error) {
	pageSize := 20
	var sniff = false //<4>
	cfg := &config.Config{
		URL:      ES_URL,
		Username: ES_USERNAME,
		Password: ES_PASSWORD,
	}

	cfg.Sniff = &sniff
	client, err := elastic.NewClientFromConfig(cfg)
	if err != nil {
		return
	}
	var esIndex = "cygx_article"

	termsQuery := elastic.NewTermsQuery("category_name", categoryName)

	boolquery := elastic.NewBoolQuery()
	boolquery.Must(elastic.NewMatchQuery("Title", keyWord), elastic.NewMatchQuery("BodyText", keyWord))
	highlight := elastic.NewHighlight()
	highlight = highlight.Fields(elastic.NewHighlighterField("Title"), elastic.NewHighlighterField("BodyText"))
	highlight = highlight.PreTags("<font color='red'>").PostTags("</font>")
	searchByMatch, err := client.Search(esIndex).Highlight(highlight).Size(pageSize).Query(termsQuery).Query(boolquery).Do(context.Background())

	if err != nil {
		return result, err
	}
	searchMap := make(map[int]int)
	if searchByMatch.Hits != nil {
		for _, v := range searchByMatch.Hits.Hits {
			articleJson, err := v.Source.MarshalJSON()
			if err != nil {
				return nil, err
			}
			article := new(models.CygxArticle)
			err = json.Unmarshal(articleJson, &article)
			if err != nil {
				return nil, err
			}
			if _, ok := searchMap[article.ArticleId]; !ok {
				searchItem := new(models.SearchItem)
				searchItem.ArticleId, _ = strconv.Atoi(v.Id)
				searchItem.Body = v.Highlight["BodyText"]
				var title string
				if len(v.Highlight["Title"]) > 0 {
					title = v.Highlight["Title"][0]
				} else {
					title = article.Title
				}
				searchItem.Title = title
				searchItem.PublishDate = article.PublishDate
				result = append(result, searchItem)
				searchMap[article.ArticleId] = article.ArticleId
			}
		}
	}
	return
}

func RemoveDuplicatesAndEmpty(a []string) (ret []string){
	a_len := len(a)
	for i:=0; i < a_len; i++{
		if (i > 0 && a[i-1] == a[i]) || len(a[i])==0{
			continue;
		}
		ret = append(ret, a[i])
	}
	return
}