hongze_cygx/services/elasticsearch.go

package services

import (
	"context"
	"encoding/json"
	"fmt"
	"github.com/PuerkitoBio/goquery"
	"github.com/olivere/elastic/v7"
	"github.com/olivere/elastic/v7/config"
	"hongze/hongze_cygx/models"
	"hongze/hongze_cygx/utils"
	"html"
	"log"
	"os"
	"strconv"
	"strings"
)

const (
	ES_URL      = "http://es-cn-nif227b580019rgw6.public.elasticsearch.aliyuncs.com:9200" //<1>
	ES_USERNAME = "elastic"                                                               //<2>
	ES_PASSWORD = "hongze@2021"                                                           //<3>
	//Grafana pwd-> 20521bb9
	//Grafana username-> emon
)

func SaveData() {
	//fmt.Println("start")
	var sniff = false //<4>
	cfg := &config.Config{
		URL:      ES_URL,
		Username: ES_USERNAME,
		Password: ES_PASSWORD,
	}

	cfg.Sniff = &sniff
	var client, err = elastic.NewClientFromConfig(cfg)
	if err != nil {
		fmt.Println("NewClientFromConfig Err:" + err.Error())
		return
	}
	var esIndex = "cygx_article"
	//var esType = "article"
	//
	exists, err := client.IndexExists(esIndex).Do(context.Background()) //<5>
	if err != nil {
		fmt.Println("IndexExists Err:" + err.Error())
		return
	}
	if !exists {
		_, err = client.CreateIndex(esIndex).Do(context.Background())
		if err != nil {
			fmt.Println("CreateIndex Err:" + err.Error())
			return
		}
	}

	/*
		3161,3190,3226,3244,3264,3285,3310,3334,3370,3397,3418,3446,3477,3497,3526,3554
	*/

	idStr := `3584,3644`
	idArr := strings.Split(idStr, ",")
	for _, v := range idArr {
		id, _ := strconv.Atoi(v)
		item, err := models.GetArticleDetailById(id)
		if err != nil {
			fmt.Println("GetArticleDetailById Err:" + err.Error())
			return
		}
		content := html.UnescapeString(item.Body)
		doc, err := goquery.NewDocumentFromReader(strings.NewReader(content))
		if err != nil {
			fmt.Println("create doc err:", err.Error())
			return
		}
		bodyText := doc.Text()
		item.BodyText = bodyText
		//新增
		resp, err := client.Index().Index(esIndex).Id(strconv.Itoa(item.ArticleId)).BodyJson(item).Do(context.Background())
		if err != nil {
			fmt.Println("insert es failed", err.Error())
			return
		}
		fmt.Println(resp.Status)
	}

	/*

		//根据id查询
		searchById, err := client.Get().Index(esIndex).Type(esType).Id("3138").Do(context.Background())
		if searchById.Found {
			body, err := searchById.Source.MarshalJSON()
			fmt.Println("body:",string(body))
			fmt.Println(err)
			//var resultType models.ArticleDetail
			//if err := json.Unmarshal(searchById.Source,&resultType); err != nil{
			//	log.Error(err.Error())
			//}
			//fmt.Printf("search by id: %#v \n",resultType)
		}
	*/

	////查询index中所有的数据
	//var resultType models.CygxArticle
	//searchAll,err := client.Search(esIndex).Type(esType).Do(context.Background())
	//for _,item := range searchAll.Each(reflect.TypeOf(resultType)) {
	//	language := item.(models.CygxArticle)
	//	fmt.Printf("search by index all: %#v \n",language)
	//}

	//根据检索条件查询

	// boolquery := elastic.NewBoolQuery()

	// boolquery.Should(elastic.NewMatchQuery("Body", "专家"))
	// highlight := elastic.NewHighlight()
	// highlight = highlight.Fields(elastic.NewHighlighterField("Body"))
	// highlight = highlight.PreTags("<font color='red'>").PostTags("</font>")

	// var pageSize int
	// pageSize = 20

	// searchByMatch, err := client.Search(esIndex).Highlight(highlight).Size(pageSize).Query(boolquery).Do(context.Background())

	// var result string
	// if searchByMatch.Hits != nil {

	// }
	// //fmt.Println(string(result))

	// utils.FileLog.Info("%s", string(result))

	//var resultType models.CygxArticle
	//for k,item := range searchByMatch.Each(reflect.TypeOf(resultType)) {
	//	language := item.(models.CygxArticle)
	//	fmt.Printf("search by match: %#v \n",language)
	//
	//	fmt.Println(k)
	//	result,err:=json.Marshal(language)
	//	fmt.Println(err)

	//	utils.FileLog.Info("%s",string(result))
	//}

	// Perform the search request.
	//searchByMatch, err := client.Search(esIndex).Type(esType).Query(query).From(1).Size(10).Do(context.Background())

	fmt.Println("end")
}

type tracelog struct{}

// 实现输出
func (tracelog) Printf(format string, v ...interface{}) {
	fmt.Printf(format, v...)
}

func SearchByKeyWordBack02(keyWord string) (result []*models.SearchItem, err error) {
	fmt.Println("keyWord:", keyWord)
	pageSize := 20
	keyWordArr, err := GetIndustryMapNameSlice(keyWord)
	keyWordArr = RemoveDuplicatesAndEmpty(keyWordArr)

	fmt.Println(keyWordArr)
	fmt.Println(" keyWordArr ")

	if err != nil {
		go utils.SendEmail(utils.APPNAME+" "+utils.RunMode+"异常提醒:", "GetIndustryMapNameSlice："+err.Error(), utils.EmailSendToUsers)
	}
	//var sniff = false //<4>
	//cfg := &config.Config{
	//	URL:      ES_URL,
	//	Username: ES_USERNAME,
	//	Password: ES_PASSWORD,
	//}
	//
	//cfg.Sniff = &sniff
	//client, err := elastic.NewClientFromConfig(cfg)
	errorlog := log.New(os.Stdout, "APP", log.LstdFlags)
	file := "./eslog.log"
	logFile, _ := os.OpenFile(file, os.O_RDWR|os.O_CREATE|os.O_APPEND, 0766)

	client, err := elastic.NewClient(
		elastic.SetURL(ES_URL),
		elastic.SetBasicAuth(ES_USERNAME, ES_PASSWORD),
		elastic.SetTraceLog(log.New(logFile, "ES-TRACE: ", 0)),
		elastic.SetSniff(false), elastic.SetErrorLog(errorlog))

	var esIndex = "cygx_article"
	searchMap := make(map[int]int)
	//boolquery := elastic.NewBoolQuery()
	//keyWordLen := len(keyWordArr)
	//n := keyWordLen
	//for _, v := range keyWordArr {
	//	keyWord = v
	//	boost := float64(n)
	//	fmt.Println("keyWord:", keyWord)
	//	fmt.Println("boost:", boost)
	//	//matchQueryList = append(matchQueryList, elastic.NewMatchQuery("Title", keyWord).Boost(boost+0.2))
	//	//matchQueryList = append(matchQueryList, elastic.NewMatchQuery("BodyText", keyWord).Boost(boost+0.1))
	//	q1 := elastic.NewMatchQuery("Title", keyWord)
	//	q1 = q1.Boost(boost + 0.2)
	//	q2 := elastic.NewMatchQuery("BodyText", keyWord)
	//	q2 = q2.Boost(boost + 0.1)
	//	boolquery.Should(q1, q2)
	//	n--
	//}
	keyWordStr := strings.Join(keyWordArr, ",")

	fmt.Println("keyWordStr ", keyWordStr)

	queryString := elastic.NewQueryStringQuery(`Title:(` + keyWordStr + `) BodyText:(` + keyWordStr + `)`).Analyzer("ik_smart")

	boolqueryJson, err := json.Marshal(queryString)
	fmt.Println("err:", err)
	fmt.Println("queryString ", string(boolqueryJson))

	//boolquery.Must(elastic.NewMatchQuery("Title", keyWord), elastic.NewMatchQuery("BodyText", keyWord))
	highlight := elastic.NewHighlight()
	highlight = highlight.Fields(elastic.NewHighlighterField("Title"), elastic.NewHighlighterField("BodyText"))
	highlight = highlight.PreTags("<font color='red'>").PostTags("</font>")
	request := client.Search(esIndex).Highlight(highlight).Size(pageSize).Query(queryString)

	requestJson, err := json.Marshal(request)
	fmt.Println("err:", err)
	fmt.Println("requestJson ", string(requestJson))

	searchByMatch, err := request.Do(context.Background())

	//searchByMatch, err := client.Search(esIndex).Highlight(highlight).Size(pageSize).Do(context.Background())
	if err != nil {
		return result, err
	}
	if searchByMatch.Hits != nil {
		for _, v := range searchByMatch.Hits.Hits {
			articleJson, err := v.Source.MarshalJSON()
			if err != nil {
				return nil, err
			}
			article := new(models.CygxArticle)
			err = json.Unmarshal(articleJson, &article)
			if err != nil {
				return nil, err
			}
			if _, ok := searchMap[article.ArticleId]; !ok {
				searchItem := new(models.SearchItem)
				searchItem.ArticleId, _ = strconv.Atoi(v.Id)
				searchItem.Body = v.Highlight["BodyText"]
				var title string
				if len(v.Highlight["Title"]) > 0 {
					title = v.Highlight["Title"][0]
				} else {
					title = article.Title
				}
				searchItem.Title = title
				searchItem.PublishDate = article.PublishDate
				result = append(result, searchItem)
				searchMap[article.ArticleId] = article.ArticleId
			}
		}
	}
	return
}

func SearchByKeyWordBack(indexName, keyWord string, startSize, pageSize, userId int) (result []*models.SearchItem, total int, err error) {
	keyWordArr, err := GetIndustryMapNameSlice(keyWord)
	if err != nil {
		go utils.SendEmail(utils.APPNAME+" "+utils.RunMode+"异常提醒:", "GetIndustryMapNameSlice："+err.Error(), utils.EmailSendToUsers)
	}
	var sniff = false //<4>
	cfg := &config.Config{
		URL:      ES_URL,
		Username: ES_USERNAME,
		Password: ES_PASSWORD,
	}

	cfg.Sniff = &sniff
	client, err := elastic.NewClientFromConfig(cfg)
	if err != nil {
		return
	}
	var esIndex = indexName

	searchMap := make(map[int]int)
	for _, v := range keyWordArr {
		keyWord = v
		boolquery := elastic.NewBoolQuery()
		boolquery.Must(elastic.NewMatchQuery("Title", keyWord), elastic.NewMatchQuery("BodyText", keyWord))
		highlight := elastic.NewHighlight()
		highlight = highlight.Fields(elastic.NewHighlighterField("Title"), elastic.NewHighlighterField("BodyText"))
		highlight = highlight.PreTags("<font color='red'>").PostTags("</font>")
		searchByMatch, err := client.Search(esIndex).Highlight(highlight).Size(pageSize).Query(boolquery).Do(context.Background())
		if err != nil {
			return result, 0, err
		}
		if searchByMatch.Hits != nil {
			for _, v := range searchByMatch.Hits.Hits {
				articleJson, err := v.Source.MarshalJSON()
				if err != nil {
					return nil, 0, err
				}
				article := new(models.CygxArticle)
				err = json.Unmarshal(articleJson, &article)
				if err != nil {
					return nil, 0, err
				}
				if _, ok := searchMap[article.ArticleId]; !ok {
					searchItem := new(models.SearchItem)
					searchItem.ArticleId, _ = strconv.Atoi(v.Id)
					searchItem.Body = v.Highlight["BodyText"]
					var title string
					if len(v.Highlight["Title"]) > 0 {
						title = v.Highlight["Title"][0]
					} else {
						title = article.Title
					}
					searchItem.Title = title
					searchItem.PublishDate = article.PublishDate
					result = append(result, searchItem)
					searchMap[article.ArticleId] = article.ArticleId
				}
			}
		}
	}
	return
}

func esSearch(keyWord, categoryName string) (result []*models.SearchItem, err error) {
	pageSize := 20
	var sniff = false //<4>
	cfg := &config.Config{
		URL:      ES_URL,
		Username: ES_USERNAME,
		Password: ES_PASSWORD,
	}

	cfg.Sniff = &sniff
	client, err := elastic.NewClientFromConfig(cfg)
	if err != nil {
		return
	}
	var esIndex = "cygx_article"

	termsQuery := elastic.NewTermsQuery("category_name", categoryName)

	boolquery := elastic.NewBoolQuery()
	boolquery.Must(elastic.NewMatchQuery("Title", keyWord), elastic.NewMatchQuery("BodyText", keyWord))
	highlight := elastic.NewHighlight()
	highlight = highlight.Fields(elastic.NewHighlighterField("Title"), elastic.NewHighlighterField("BodyText"))
	highlight = highlight.PreTags("<font color='red'>").PostTags("</font>")
	searchByMatch, err := client.Search(esIndex).Highlight(highlight).Size(pageSize).Query(termsQuery).Query(boolquery).Do(context.Background())

	if err != nil {
		return result, err
	}
	searchMap := make(map[int]int)
	if searchByMatch.Hits != nil {
		for _, v := range searchByMatch.Hits.Hits {
			articleJson, err := v.Source.MarshalJSON()
			if err != nil {
				return nil, err
			}
			article := new(models.CygxArticle)
			err = json.Unmarshal(articleJson, &article)
			if err != nil {
				return nil, err
			}
			if _, ok := searchMap[article.ArticleId]; !ok {
				searchItem := new(models.SearchItem)
				searchItem.ArticleId, _ = strconv.Atoi(v.Id)
				searchItem.Body = v.Highlight["BodyText"]
				var title string
				if len(v.Highlight["Title"]) > 0 {
					title = v.Highlight["Title"][0]
				} else {
					title = article.Title
				}
				searchItem.Title = title
				searchItem.PublishDate = article.PublishDate
				result = append(result, searchItem)
				searchMap[article.ArticleId] = article.ArticleId
			}
		}
	}
	return
}

func RemoveDuplicatesAndEmpty(a []string) (ret []string) {
	a_len := len(a)
	for i := 0; i < a_len; i++ {
		if (i > 0 && a[i-1] == a[i]) || len(a[i]) == 0 {
			continue
		}
		ret = append(ret, a[i])
	}
	return
}

// KeyWordArrSqlRegexp 预处理ik联想词的模糊查询语句
func KeyWordArrSqlRegexp(a []string) (ret string) {
	a_len := len(a)
	for i := 0; i < a_len; i++ {
		if i == 0 {
			continue
		}
		ret += a[i] + "|"
	}
	ret = strings.TrimRight(ret, "|")
	return
}

// KeyWordArrSqlRegexp 预处理ik联想词的模糊查询语句
func KeyWordArrSqlRegexpAll(a []string) (ret string) {
	a_len := len(a)
	for i := 0; i < a_len; i++ {
		ret += a[i] + "|"
	}
	ret = strings.TrimRight(ret, "|")
	//ret = "'" + ret + "'"
	return
}

func init21123() {
	fmt.Println("start")
	client, err := elastic.NewClient(elastic.SetURL(ES_URL), elastic.SetBasicAuth(ES_USERNAME, ES_PASSWORD), elastic.SetSniff(false))
	if err != nil {
		fmt.Println("err:", err)
	}
	fmt.Println(client)
	keyWordStr := "医疗器械"
	queryString := elastic.NewQueryStringQuery(`Title:(` + keyWordStr + `)`)

	boolqueryJson, err := json.Marshal(queryString)
	fmt.Println("err:", err)
	fmt.Println("queryString ", string(boolqueryJson))

	var esIndex = "cygx_article"
	//boolquery.Must(elastic.NewMatchQuery("Title", keyWord), elastic.NewMatchQuery("BodyText", keyWord))
	highlight := elastic.NewHighlight()
	highlight = highlight.Fields(elastic.NewHighlighterField("Title"), elastic.NewHighlighterField("BodyText"))
	highlight = highlight.PreTags("<font color='red'>").PostTags("</font>")
	request := client.Search(esIndex).Highlight(highlight).Query(queryString)

	requestJson, err := json.Marshal(request)
	fmt.Println("err:", err)
	fmt.Println("requestJson ", string(requestJson))

	searchByMatch, err := request.Do(context.Background())

	if searchByMatch.Hits != nil {

	}

	fmt.Println(searchByMatch)
	fmt.Println("end")
}

func SearchByKeyWordQuery(keyWord string) (result []*models.SearchItem, err error) {
	errorlog := log.New(os.Stdout, "APP", log.LstdFlags)
	file := "./rdlucklog/eslog.log"
	logFile, _ := os.OpenFile(file, os.O_RDWR|os.O_CREATE|os.O_APPEND, 0766)

	client, err := elastic.NewClient(
		elastic.SetURL(ES_URL),
		elastic.SetBasicAuth(ES_USERNAME, ES_PASSWORD),
		elastic.SetTraceLog(log.New(logFile, "ES-TRACE: ", 0)),
		elastic.SetSniff(false), elastic.SetErrorLog(errorlog))

	var esIndex = "cygx_article"

	//keyWordStr := "医疗"
	//字段相等
	queryString := elastic.NewQueryStringQuery("Title:医疗")
	//.Analyzer("ik_smart")
	highlight := elastic.NewHighlight()
	highlight = highlight.Fields(elastic.NewHighlighterField("Title"), elastic.NewHighlighterField("BodyText"))
	highlight = highlight.PreTags("<font color='red'>").PostTags("</font>")
	request := client.Search(esIndex).Highlight(highlight).Query(queryString)

	searchByMatch, err := request.Do(context.Background())
	if err != nil {
		fmt.Println("request.Do err:", err.Error())
		return result, err
	}
	if searchByMatch.Hits != nil {
		for _, v := range searchByMatch.Hits.Hits {
			articleJson, _ := v.Source.MarshalJSON()
			fmt.Println(string(articleJson))
		}
	}
	return
}

func SearchByKeyWord(keyWord string) (result []*models.SearchItem, err error) {
	fmt.Println("keyWord:", keyWord)
	keyWord = "医药"
	pageSize := 20
	keyWordArr, err := GetIndustryMapNameSlice(keyWord)
	keyWordArr = RemoveDuplicatesAndEmpty(keyWordArr)

	fmt.Println(keyWordArr)
	fmt.Println(" keyWordArr ")

	if err != nil {
		go utils.SendEmail(utils.APPNAME+" "+utils.RunMode+"异常提醒:", "GetIndustryMapNameSlice："+err.Error(), utils.EmailSendToUsers)
	}
	errorlog := log.New(os.Stdout, "APP", log.LstdFlags)
	file := "./rdlucklog/eslog.log"
	logFile, _ := os.OpenFile(file, os.O_RDWR|os.O_CREATE|os.O_APPEND, 0766)

	client, err := elastic.NewClient(
		elastic.SetURL(ES_URL),
		elastic.SetBasicAuth(ES_USERNAME, ES_PASSWORD),
		elastic.SetTraceLog(log.New(logFile, "ES-TRACE: ", 0)),
		elastic.SetSniff(false), elastic.SetErrorLog(errorlog))

	//var esIndex = "cygx_article"
	var esIndex = "article_two"
	searchMap := make(map[int]int)

	//queryString := elastic.NewQueryStringQuery(`Title:医疗 BodyText:医疗`).Analyzer("ik_smart")

	//queryTeerms:=elastic.NewTermsQuery("Title","医疗","费用")

	//queryMatchPhrase:=elastic.NewMatchPhraseQuery("Title","费用")
	//.Analyzer("ik_smart")
	//.Analyzer("ik_smart")
	//boolquery.Must(elastic.NewMatchQuery("Title", keyWord), elastic.NewMatchQuery("BodyText", keyWord))
	boolquery := elastic.NewBoolQuery()

	keyLen := len(keyWordArr)
	n := float64(keyLen)
	matchArr := make([]elastic.Query, 0)
	for _, v := range keyWordArr {
		if v != "" {
			matchq1 := elastic.NewMatchQuery("Title", v).Boost(n).Analyzer("ik_smart")
			matchq2 := elastic.NewMatchQuery("BodyText", v).Boost(n).Analyzer("ik_smart")
			matchArr = append(matchArr, matchq1)
			matchArr = append(matchArr, matchq2)
		}
		n--
	}
	//matchArr=append(matchArr,matchq1)
	//matchArr=append(matchArr,matchq2)

	boolquery.Should(matchArr...)
	//boolquery.Must(elastic.NewMatchQuery("BodyText", keyWord).Boost(1).Analyzer("ik_smart"))

	highlight := elastic.NewHighlight()
	highlight = highlight.Fields(elastic.NewHighlighterField("Title"), elastic.NewHighlighterField("BodyText"))
	highlight = highlight.PreTags("<font color='red'>").PostTags("</font>")
	//request := client.Search(esIndex).Highlight(highlight).Size(pageSize).Query(queryString)
	request := client.Search(esIndex).Highlight(highlight).Size(pageSize).Query(boolquery)
	searchByMatch, err := request.Do(context.Background())
	fmt.Println(searchByMatch, err)
	//searchByMatch, err := client.Search(esIndex).Highlight(highlight).Size(pageSize).Do(context.Background())
	if err != nil {
		return result, err
	}
	fmt.Println(searchByMatch.Status, searchByMatch.TotalHits())
	if searchByMatch.Hits != nil {
		for _, v := range searchByMatch.Hits.Hits {
			articleJson, err := v.Source.MarshalJSON()
			if err != nil {
				return nil, err
			}
			article := new(models.CygxArticle)
			err = json.Unmarshal(articleJson, &article)
			if err != nil {
				return nil, err
			}
			if _, ok := searchMap[article.ArticleId]; !ok {
				searchItem := new(models.SearchItem)
				searchItem.ArticleId, _ = strconv.Atoi(v.Id)
				searchItem.Body = v.Highlight["BodyText"]
				var title string
				if len(v.Highlight["Title"]) > 0 {
					title = v.Highlight["Title"][0]
				} else {
					title = article.Title
				}
				searchItem.Title = title
				searchItem.PublishDate = article.PublishDate
				result = append(result, searchItem)
				searchMap[article.ArticleId] = article.ArticleId
			}
		}
	}
	return
}

func AddMap() {
	errorlog := log.New(os.Stdout, "APP", log.LstdFlags)
	file := "./rdlucklog/eslog.log"
	logFile, _ := os.OpenFile(file, os.O_RDWR|os.O_CREATE|os.O_APPEND, 0766)

	client, err := elastic.NewClient(
		elastic.SetURL(ES_URL),
		elastic.SetBasicAuth(ES_USERNAME, ES_PASSWORD),
		elastic.SetTraceLog(log.New(logFile, "ES-TRACE: ", 0)),
		elastic.SetSniff(false), elastic.SetErrorLog(errorlog))
	if err != nil {
		fmt.Println("NewClient Err:", err.Error())
	}

	bodyJson := `{
  "mappings": {
    "properties": {
      "ArticleId": {
        "type": "integer"
      },
      "Title": {
        "type": "text",
        "analyzer": "ik_smart"
      },
      "BodyText": {
        "type": "text",
        "analyzer": "ik_smart"
      }
    }
  }
}`
	//var esMappingIndex = "article_mapping"
	var esIndex = "article_two"
	a := client.IndexAnalyze().Index(esIndex)

	analyzeBody, err := json.Marshal(a)
	fmt.Println(string(analyzeBody))

	exists, err := client.IndexExists(esIndex).Do(context.Background()) //<5>
	if err != nil {
		fmt.Println("IndexExists Err:" + err.Error())
		return
	}
	fmt.Println("exists:", exists)
	if !exists {
		_, err = client.CreateIndex(esIndex).BodyJson(bodyJson).Do(context.Background())
		//BodyJson(bodyJson).Do(context.Background())
		if err != nil {
			fmt.Println("CreateIndex Err:" + err.Error())
			return
		}
	}

	item, err := models.GetArticleDetailTestById(3546)
	if err != nil {
		fmt.Println("GetArticleDetailById Err:" + err.Error())
		return
	}
	content := html.UnescapeString(item.Body)
	doc, err := goquery.NewDocumentFromReader(strings.NewReader(content))
	if err != nil {
		fmt.Println("create doc err:", err.Error())
		return
	}
	bodyText := doc.Text()
	item.BodyText = bodyText
	//新增
	resp, err := client.Index().Index(esIndex).Id(strconv.Itoa(item.ArticleId)).BodyJson(item).Do(context.Background())
	if err != nil {
		fmt.Println("insert es failed", err.Error())
		return
	}
	fmt.Println(resp)
}