package services
import (
"context"
"encoding/json"
"fmt"
"hongze/hongze_cygx/models"
"hongze/hongze_cygx/utils"
"html"
"strconv"
"strings"
//"gopkg.in/olivere/elastic.v5"
//"gopkg.in/olivere/elastic.v5/config"
"github.com/PuerkitoBio/goquery"
"github.com/olivere/elastic/v7"
"github.com/olivere/elastic/v7/config"
)
const (
ES_URL = "http://es-cn-nif227b580019rgw6.public.elasticsearch.aliyuncs.com:9200" //<1>
ES_USERNAME = "elastic" //<2>
ES_PASSWORD = "hongze@2021" //<3>
//Grafana pwd-> 20521bb9
//Grafana username-> emon
)
func SaveData() {
//fmt.Println("start")
var sniff = false //<4>
cfg := &config.Config{
URL: ES_URL,
Username: ES_USERNAME,
Password: ES_PASSWORD,
}
cfg.Sniff = &sniff
var client, err = elastic.NewClientFromConfig(cfg)
if err != nil {
fmt.Println("NewClientFromConfig Err:" + err.Error())
return
}
var esIndex = "cygx_article"
//var esType = "article"
//
exists, err := client.IndexExists(esIndex).Do(context.Background()) //<5>
if err != nil {
fmt.Println("IndexExists Err:" + err.Error())
return
}
if !exists {
_, err = client.CreateIndex(esIndex).Do(context.Background())
if err != nil {
fmt.Println("CreateIndex Err:" + err.Error())
return
}
}
/*
3161,3190,3226,3244,3264,3285,3310,3334,3370,3397,3418,3446,3477,3497,3526,3554
*/
idStr := `3161,3190,3226,3244,3264,3285,3310,3334,3370,3397,3418,3446,3477,3497,3526,3554`
idArr := strings.Split(idStr, ",")
for _, v := range idArr {
id, _ := strconv.Atoi(v)
item, err := models.GetArticleDetailById(id)
if err != nil {
fmt.Println("GetArticleDetailById Err:" + err.Error())
return
}
content := html.UnescapeString(item.Body)
doc, err := goquery.NewDocumentFromReader(strings.NewReader(content))
if err != nil {
fmt.Println("create doc err:", err.Error())
return
}
bodyText := doc.Text()
item.BodyText = bodyText
//删除
resp, err := client.Index().Index(esIndex).Id(strconv.Itoa(item.ArticleId)).BodyJson(item).Do(context.Background())
if err != nil {
fmt.Println("insert es failed", err.Error())
return
}
fmt.Println(resp.Status)
}
/*
//根据id查询
searchById, err := client.Get().Index(esIndex).Type(esType).Id("3138").Do(context.Background())
if searchById.Found {
body, err := searchById.Source.MarshalJSON()
fmt.Println("body:",string(body))
fmt.Println(err)
//var resultType models.ArticleDetail
//if err := json.Unmarshal(searchById.Source,&resultType); err != nil{
// log.Error(err.Error())
//}
//fmt.Printf("search by id: %#v \n",resultType)
}
*/
////查询index中所有的数据
//var resultType models.CygxArticle
//searchAll,err := client.Search(esIndex).Type(esType).Do(context.Background())
//for _,item := range searchAll.Each(reflect.TypeOf(resultType)) {
// language := item.(models.CygxArticle)
// fmt.Printf("search by index all: %#v \n",language)
//}
//根据检索条件查询
// boolquery := elastic.NewBoolQuery()
// boolquery.Should(elastic.NewMatchQuery("Body", "专家"))
// highlight := elastic.NewHighlight()
// highlight = highlight.Fields(elastic.NewHighlighterField("Body"))
// highlight = highlight.PreTags("").PostTags("")
// var pageSize int
// pageSize = 20
// searchByMatch, err := client.Search(esIndex).Highlight(highlight).Size(pageSize).Query(boolquery).Do(context.Background())
// var result string
// if searchByMatch.Hits != nil {
// }
// //fmt.Println(string(result))
// utils.FileLog.Info("%s", string(result))
//var resultType models.CygxArticle
//for k,item := range searchByMatch.Each(reflect.TypeOf(resultType)) {
// language := item.(models.CygxArticle)
// fmt.Printf("search by match: %#v \n",language)
//
// fmt.Println(k)
// result,err:=json.Marshal(language)
// fmt.Println(err)
// utils.FileLog.Info("%s",string(result))
//}
// Perform the search request.
//searchByMatch, err := client.Search(esIndex).Type(esType).Query(query).From(1).Size(10).Do(context.Background())
fmt.Println("end")
}
func SearchByKeyWord(keyWord string, pageSize int) (result []*models.SearchItem, err error) {
if pageSize == 0 {
pageSize = 20
}
var sniff = false //<4>
cfg := &config.Config{
URL: ES_URL,
Username: ES_USERNAME,
Password: ES_PASSWORD,
}
cfg.Sniff = &sniff
client, err := elastic.NewClientFromConfig(cfg)
if err != nil {
return
}
var esIndex = "cygx_article"
fmt.Println("get start keyWordArr")
keyWordArr, err := GetIndustryMapNameSlice(keyWord)
if err != nil {
go utils.SendEmail(utils.APPNAME+" "+utils.RunMode+"异常提醒:", "GetIndustryMapNameSlice:"+err.Error(), utils.EmailSendToUsers)
}
keyWordArr = append(keyWordArr, keyWord)
fmt.Println(keyWord)
fmt.Println(keyWordArr)
var newKeyWordArr []string
if len(keyWordArr) > 2 {
newKeyWordArr = append(keyWordArr[2:], keyWordArr[0:2]...)
}
fmt.Println(newKeyWordArr)
fmt.Println("get end keyWordArr")
searchMap := make(map[int]int)
for k, v := range newKeyWordArr {
fmt.Println(k, v)
boolquery := elastic.NewBoolQuery()
boolquery.Must(elastic.NewMatchQuery("Title", keyWord), elastic.NewMatchQuery("BodyText", keyWord))
highlight := elastic.NewHighlight()
highlight = highlight.Fields(elastic.NewHighlighterField("Title"), elastic.NewHighlighterField("BodyText"))
highlight = highlight.PreTags("").PostTags("")
searchByMatch, err := client.Search(esIndex).Highlight(highlight).Size(pageSize).Query(boolquery).Do(context.Background())
if err != nil {
return result, err
}
if searchByMatch.Hits != nil {
for _, v := range searchByMatch.Hits.Hits {
articleJson, err := v.Source.MarshalJSON()
if err != nil {
return nil, err
}
article := new(models.CygxArticle)
err = json.Unmarshal(articleJson, &article)
if err != nil {
return nil, err
}
if _, ok := searchMap[article.ArticleId]; !ok {
searchItem := new(models.SearchItem)
searchItem.ArticleId, _ = strconv.Atoi(v.Id)
searchItem.Body = v.Highlight["BodyText"]
var title string
if len(v.Highlight["Title"]) > 0 {
title = v.Highlight["Title"][0]
} else {
title = article.Title
}
searchItem.Title = title
searchItem.PublishDate = article.PublishDate
result = append(result, searchItem)
searchMap[article.ArticleId] = article.ArticleId
}
}
}
}
return
}