|
|
@@ -1,654 +0,0 @@
|
|
|
-package services
|
|
|
-
|
|
|
-import (
|
|
|
- "context"
|
|
|
- "encoding/json"
|
|
|
- "fmt"
|
|
|
- "github.com/PuerkitoBio/goquery"
|
|
|
- "github.com/olivere/elastic/v7"
|
|
|
- "github.com/olivere/elastic/v7/config"
|
|
|
- "hongze/hongze_cygx/models"
|
|
|
- "hongze/hongze_cygx/utils"
|
|
|
- "html"
|
|
|
- "log"
|
|
|
- "os"
|
|
|
- "strconv"
|
|
|
- "strings"
|
|
|
-)
|
|
|
-
|
|
|
-const (
|
|
|
- ES_URL = "http://es-cn-nif227b580019rgw6.public.elasticsearch.aliyuncs.com:9200" //<1>
|
|
|
- ES_USERNAME = "elastic" //<2>
|
|
|
- ES_PASSWORD = "hongze@2021" //<3>
|
|
|
- //Grafana pwd-> 20521bb9
|
|
|
- //Grafana username-> emon
|
|
|
-)
|
|
|
-
|
|
|
-func SaveData() {
|
|
|
- //fmt.Println("start")
|
|
|
- var sniff = false //<4>
|
|
|
- cfg := &config.Config{
|
|
|
- URL: ES_URL,
|
|
|
- Username: ES_USERNAME,
|
|
|
- Password: ES_PASSWORD,
|
|
|
- }
|
|
|
-
|
|
|
- cfg.Sniff = &sniff
|
|
|
- var client, err = elastic.NewClientFromConfig(cfg)
|
|
|
- if err != nil {
|
|
|
- fmt.Println("NewClientFromConfig Err:" + err.Error())
|
|
|
- return
|
|
|
- }
|
|
|
- var esIndex = "cygx_article"
|
|
|
- //var esType = "article"
|
|
|
- //
|
|
|
- exists, err := client.IndexExists(esIndex).Do(context.Background()) //<5>
|
|
|
- if err != nil {
|
|
|
- fmt.Println("IndexExists Err:" + err.Error())
|
|
|
- return
|
|
|
- }
|
|
|
- if !exists {
|
|
|
- _, err = client.CreateIndex(esIndex).Do(context.Background())
|
|
|
- if err != nil {
|
|
|
- fmt.Println("CreateIndex Err:" + err.Error())
|
|
|
- return
|
|
|
- }
|
|
|
- }
|
|
|
-
|
|
|
- /*
|
|
|
- 3161,3190,3226,3244,3264,3285,3310,3334,3370,3397,3418,3446,3477,3497,3526,3554
|
|
|
- */
|
|
|
-
|
|
|
- idStr := `3584,3644`
|
|
|
- idArr := strings.Split(idStr, ",")
|
|
|
- for _, v := range idArr {
|
|
|
- id, _ := strconv.Atoi(v)
|
|
|
- item, err := models.GetArticleDetailById(id)
|
|
|
- if err != nil {
|
|
|
- fmt.Println("GetArticleDetailById Err:" + err.Error())
|
|
|
- return
|
|
|
- }
|
|
|
- content := html.UnescapeString(item.Body)
|
|
|
- doc, err := goquery.NewDocumentFromReader(strings.NewReader(content))
|
|
|
- if err != nil {
|
|
|
- fmt.Println("create doc err:", err.Error())
|
|
|
- return
|
|
|
- }
|
|
|
- bodyText := doc.Text()
|
|
|
- item.BodyText = bodyText
|
|
|
- //新增
|
|
|
- resp, err := client.Index().Index(esIndex).Id(strconv.Itoa(item.ArticleId)).BodyJson(item).Do(context.Background())
|
|
|
- if err != nil {
|
|
|
- fmt.Println("insert es failed", err.Error())
|
|
|
- return
|
|
|
- }
|
|
|
- fmt.Println(resp.Status)
|
|
|
- }
|
|
|
-
|
|
|
- /*
|
|
|
-
|
|
|
- //根据id查询
|
|
|
- searchById, err := client.Get().Index(esIndex).Type(esType).Id("3138").Do(context.Background())
|
|
|
- if searchById.Found {
|
|
|
- body, err := searchById.Source.MarshalJSON()
|
|
|
- fmt.Println("body:",string(body))
|
|
|
- fmt.Println(err)
|
|
|
- //var resultType models.ArticleDetail
|
|
|
- //if err := json.Unmarshal(searchById.Source,&resultType); err != nil{
|
|
|
- // log.Error(err.Error())
|
|
|
- //}
|
|
|
- //fmt.Printf("search by id: %#v \n",resultType)
|
|
|
- }
|
|
|
- */
|
|
|
-
|
|
|
- ////查询index中所有的数据
|
|
|
- //var resultType models.CygxArticle
|
|
|
- //searchAll,err := client.Search(esIndex).Type(esType).Do(context.Background())
|
|
|
- //for _,item := range searchAll.Each(reflect.TypeOf(resultType)) {
|
|
|
- // language := item.(models.CygxArticle)
|
|
|
- // fmt.Printf("search by index all: %#v \n",language)
|
|
|
- //}
|
|
|
-
|
|
|
- //根据检索条件查询
|
|
|
-
|
|
|
- // boolquery := elastic.NewBoolQuery()
|
|
|
-
|
|
|
- // boolquery.Should(elastic.NewMatchQuery("Body", "专家"))
|
|
|
- // highlight := elastic.NewHighlight()
|
|
|
- // highlight = highlight.Fields(elastic.NewHighlighterField("Body"))
|
|
|
- // highlight = highlight.PreTags("<font color='red'>").PostTags("</font>")
|
|
|
-
|
|
|
- // var pageSize int
|
|
|
- // pageSize = 20
|
|
|
-
|
|
|
- // searchByMatch, err := client.Search(esIndex).Highlight(highlight).Size(pageSize).Query(boolquery).Do(context.Background())
|
|
|
-
|
|
|
- // var result string
|
|
|
- // if searchByMatch.Hits != nil {
|
|
|
-
|
|
|
- // }
|
|
|
- // //fmt.Println(string(result))
|
|
|
-
|
|
|
- // utils.FileLog.Info("%s", string(result))
|
|
|
-
|
|
|
- //var resultType models.CygxArticle
|
|
|
- //for k,item := range searchByMatch.Each(reflect.TypeOf(resultType)) {
|
|
|
- // language := item.(models.CygxArticle)
|
|
|
- // fmt.Printf("search by match: %#v \n",language)
|
|
|
- //
|
|
|
- // fmt.Println(k)
|
|
|
- // result,err:=json.Marshal(language)
|
|
|
- // fmt.Println(err)
|
|
|
-
|
|
|
- // utils.FileLog.Info("%s",string(result))
|
|
|
- //}
|
|
|
-
|
|
|
- // Perform the search request.
|
|
|
- //searchByMatch, err := client.Search(esIndex).Type(esType).Query(query).From(1).Size(10).Do(context.Background())
|
|
|
-
|
|
|
- fmt.Println("end")
|
|
|
-}
|
|
|
-
|
|
|
-type tracelog struct{}
|
|
|
-
|
|
|
-//实现输出
|
|
|
-func (tracelog) Printf(format string, v ...interface{}) {
|
|
|
- fmt.Printf(format, v...)
|
|
|
-}
|
|
|
-
|
|
|
-func SearchByKeyWordBack02(keyWord string) (result []*models.SearchItem, err error) {
|
|
|
- fmt.Println("keyWord:", keyWord)
|
|
|
- pageSize := 20
|
|
|
- keyWordArr, err := GetIndustryMapNameSlice(keyWord)
|
|
|
- keyWordArr = RemoveDuplicatesAndEmpty(keyWordArr)
|
|
|
-
|
|
|
- fmt.Println(keyWordArr)
|
|
|
- fmt.Println(" keyWordArr ")
|
|
|
-
|
|
|
- if err != nil {
|
|
|
- go utils.SendEmail(utils.APPNAME+" "+utils.RunMode+"异常提醒:", "GetIndustryMapNameSlice："+err.Error(), utils.EmailSendToUsers)
|
|
|
- }
|
|
|
- //var sniff = false //<4>
|
|
|
- //cfg := &config.Config{
|
|
|
- // URL: ES_URL,
|
|
|
- // Username: ES_USERNAME,
|
|
|
- // Password: ES_PASSWORD,
|
|
|
- //}
|
|
|
- //
|
|
|
- //cfg.Sniff = &sniff
|
|
|
- //client, err := elastic.NewClientFromConfig(cfg)
|
|
|
- errorlog := log.New(os.Stdout, "APP", log.LstdFlags)
|
|
|
- file := "./eslog.log"
|
|
|
- logFile, _ := os.OpenFile(file, os.O_RDWR|os.O_CREATE|os.O_APPEND, 0766)
|
|
|
-
|
|
|
- client, err := elastic.NewClient(
|
|
|
- elastic.SetURL(ES_URL),
|
|
|
- elastic.SetBasicAuth(ES_USERNAME, ES_PASSWORD),
|
|
|
- elastic.SetTraceLog(log.New(logFile, "ES-TRACE: ", 0)),
|
|
|
- elastic.SetSniff(false), elastic.SetErrorLog(errorlog))
|
|
|
-
|
|
|
- var esIndex = "cygx_article"
|
|
|
- searchMap := make(map[int]int)
|
|
|
- //boolquery := elastic.NewBoolQuery()
|
|
|
- //keyWordLen := len(keyWordArr)
|
|
|
- //n := keyWordLen
|
|
|
- //for _, v := range keyWordArr {
|
|
|
- // keyWord = v
|
|
|
- // boost := float64(n)
|
|
|
- // fmt.Println("keyWord:", keyWord)
|
|
|
- // fmt.Println("boost:", boost)
|
|
|
- // //matchQueryList = append(matchQueryList, elastic.NewMatchQuery("Title", keyWord).Boost(boost+0.2))
|
|
|
- // //matchQueryList = append(matchQueryList, elastic.NewMatchQuery("BodyText", keyWord).Boost(boost+0.1))
|
|
|
- // q1 := elastic.NewMatchQuery("Title", keyWord)
|
|
|
- // q1 = q1.Boost(boost + 0.2)
|
|
|
- // q2 := elastic.NewMatchQuery("BodyText", keyWord)
|
|
|
- // q2 = q2.Boost(boost + 0.1)
|
|
|
- // boolquery.Should(q1, q2)
|
|
|
- // n--
|
|
|
- //}
|
|
|
- keyWordStr := strings.Join(keyWordArr, ",")
|
|
|
-
|
|
|
- fmt.Println("keyWordStr ", keyWordStr)
|
|
|
-
|
|
|
- queryString := elastic.NewQueryStringQuery(`Title:(` + keyWordStr + `) BodyText:(` + keyWordStr + `)`).Analyzer("ik_smart")
|
|
|
-
|
|
|
- boolqueryJson, err := json.Marshal(queryString)
|
|
|
- fmt.Println("err:", err)
|
|
|
- fmt.Println("queryString ", string(boolqueryJson))
|
|
|
-
|
|
|
- //boolquery.Must(elastic.NewMatchQuery("Title", keyWord), elastic.NewMatchQuery("BodyText", keyWord))
|
|
|
- highlight := elastic.NewHighlight()
|
|
|
- highlight = highlight.Fields(elastic.NewHighlighterField("Title"), elastic.NewHighlighterField("BodyText"))
|
|
|
- highlight = highlight.PreTags("<font color='red'>").PostTags("</font>")
|
|
|
- request := client.Search(esIndex).Highlight(highlight).Size(pageSize).Query(queryString)
|
|
|
-
|
|
|
- requestJson, err := json.Marshal(request)
|
|
|
- fmt.Println("err:", err)
|
|
|
- fmt.Println("requestJson ", string(requestJson))
|
|
|
-
|
|
|
- searchByMatch, err := request.Do(context.Background())
|
|
|
-
|
|
|
- //searchByMatch, err := client.Search(esIndex).Highlight(highlight).Size(pageSize).Do(context.Background())
|
|
|
- if err != nil {
|
|
|
- return result, err
|
|
|
- }
|
|
|
- if searchByMatch.Hits != nil {
|
|
|
- for _, v := range searchByMatch.Hits.Hits {
|
|
|
- articleJson, err := v.Source.MarshalJSON()
|
|
|
- if err != nil {
|
|
|
- return nil, err
|
|
|
- }
|
|
|
- article := new(models.CygxArticle)
|
|
|
- err = json.Unmarshal(articleJson, &article)
|
|
|
- if err != nil {
|
|
|
- return nil, err
|
|
|
- }
|
|
|
- if _, ok := searchMap[article.ArticleId]; !ok {
|
|
|
- searchItem := new(models.SearchItem)
|
|
|
- searchItem.ArticleId, _ = strconv.Atoi(v.Id)
|
|
|
- searchItem.Body = v.Highlight["BodyText"]
|
|
|
- var title string
|
|
|
- if len(v.Highlight["Title"]) > 0 {
|
|
|
- title = v.Highlight["Title"][0]
|
|
|
- } else {
|
|
|
- title = article.Title
|
|
|
- }
|
|
|
- searchItem.Title = title
|
|
|
- searchItem.PublishDate = article.PublishDate
|
|
|
- result = append(result, searchItem)
|
|
|
- searchMap[article.ArticleId] = article.ArticleId
|
|
|
- }
|
|
|
- }
|
|
|
- }
|
|
|
- return
|
|
|
-}
|
|
|
-
|
|
|
-func SearchByKeyWordBack(keyWord string) (result []*models.SearchItem, err error) {
|
|
|
- pageSize := 20
|
|
|
- keyWordArr, err := GetIndustryMapNameSlice(keyWord)
|
|
|
- if err != nil {
|
|
|
- go utils.SendEmail(utils.APPNAME+" "+utils.RunMode+"异常提醒:", "GetIndustryMapNameSlice："+err.Error(), utils.EmailSendToUsers)
|
|
|
- }
|
|
|
- var sniff = false //<4>
|
|
|
- cfg := &config.Config{
|
|
|
- URL: ES_URL,
|
|
|
- Username: ES_USERNAME,
|
|
|
- Password: ES_PASSWORD,
|
|
|
- }
|
|
|
-
|
|
|
- cfg.Sniff = &sniff
|
|
|
- client, err := elastic.NewClientFromConfig(cfg)
|
|
|
- if err != nil {
|
|
|
- return
|
|
|
- }
|
|
|
- var esIndex = "cygx_article"
|
|
|
-
|
|
|
- searchMap := make(map[int]int)
|
|
|
- for _, v := range keyWordArr {
|
|
|
- keyWord = v
|
|
|
- boolquery := elastic.NewBoolQuery()
|
|
|
- boolquery.Must(elastic.NewMatchQuery("Title", keyWord), elastic.NewMatchQuery("BodyText", keyWord))
|
|
|
- highlight := elastic.NewHighlight()
|
|
|
- highlight = highlight.Fields(elastic.NewHighlighterField("Title"), elastic.NewHighlighterField("BodyText"))
|
|
|
- highlight = highlight.PreTags("<font color='red'>").PostTags("</font>")
|
|
|
- searchByMatch, err := client.Search(esIndex).Highlight(highlight).Size(pageSize).Query(boolquery).Do(context.Background())
|
|
|
- if err != nil {
|
|
|
- return result, err
|
|
|
- }
|
|
|
- if searchByMatch.Hits != nil {
|
|
|
- for _, v := range searchByMatch.Hits.Hits {
|
|
|
- articleJson, err := v.Source.MarshalJSON()
|
|
|
- if err != nil {
|
|
|
- return nil, err
|
|
|
- }
|
|
|
- article := new(models.CygxArticle)
|
|
|
- err = json.Unmarshal(articleJson, &article)
|
|
|
- if err != nil {
|
|
|
- return nil, err
|
|
|
- }
|
|
|
- if _, ok := searchMap[article.ArticleId]; !ok {
|
|
|
- searchItem := new(models.SearchItem)
|
|
|
- searchItem.ArticleId, _ = strconv.Atoi(v.Id)
|
|
|
- searchItem.Body = v.Highlight["BodyText"]
|
|
|
- var title string
|
|
|
- if len(v.Highlight["Title"]) > 0 {
|
|
|
- title = v.Highlight["Title"][0]
|
|
|
- } else {
|
|
|
- title = article.Title
|
|
|
- }
|
|
|
- searchItem.Title = title
|
|
|
- searchItem.PublishDate = article.PublishDate
|
|
|
- result = append(result, searchItem)
|
|
|
- searchMap[article.ArticleId] = article.ArticleId
|
|
|
- }
|
|
|
- }
|
|
|
- }
|
|
|
- }
|
|
|
- return
|
|
|
-}
|
|
|
-
|
|
|
-func esSearch(keyWord, categoryName string) (result []*models.SearchItem, err error) {
|
|
|
- pageSize := 20
|
|
|
- var sniff = false //<4>
|
|
|
- cfg := &config.Config{
|
|
|
- URL: ES_URL,
|
|
|
- Username: ES_USERNAME,
|
|
|
- Password: ES_PASSWORD,
|
|
|
- }
|
|
|
-
|
|
|
- cfg.Sniff = &sniff
|
|
|
- client, err := elastic.NewClientFromConfig(cfg)
|
|
|
- if err != nil {
|
|
|
- return
|
|
|
- }
|
|
|
- var esIndex = "cygx_article"
|
|
|
-
|
|
|
- termsQuery := elastic.NewTermsQuery("category_name", categoryName)
|
|
|
-
|
|
|
- boolquery := elastic.NewBoolQuery()
|
|
|
- boolquery.Must(elastic.NewMatchQuery("Title", keyWord), elastic.NewMatchQuery("BodyText", keyWord))
|
|
|
- highlight := elastic.NewHighlight()
|
|
|
- highlight = highlight.Fields(elastic.NewHighlighterField("Title"), elastic.NewHighlighterField("BodyText"))
|
|
|
- highlight = highlight.PreTags("<font color='red'>").PostTags("</font>")
|
|
|
- searchByMatch, err := client.Search(esIndex).Highlight(highlight).Size(pageSize).Query(termsQuery).Query(boolquery).Do(context.Background())
|
|
|
-
|
|
|
- if err != nil {
|
|
|
- return result, err
|
|
|
- }
|
|
|
- searchMap := make(map[int]int)
|
|
|
- if searchByMatch.Hits != nil {
|
|
|
- for _, v := range searchByMatch.Hits.Hits {
|
|
|
- articleJson, err := v.Source.MarshalJSON()
|
|
|
- if err != nil {
|
|
|
- return nil, err
|
|
|
- }
|
|
|
- article := new(models.CygxArticle)
|
|
|
- err = json.Unmarshal(articleJson, &article)
|
|
|
- if err != nil {
|
|
|
- return nil, err
|
|
|
- }
|
|
|
- if _, ok := searchMap[article.ArticleId]; !ok {
|
|
|
- searchItem := new(models.SearchItem)
|
|
|
- searchItem.ArticleId, _ = strconv.Atoi(v.Id)
|
|
|
- searchItem.Body = v.Highlight["BodyText"]
|
|
|
- var title string
|
|
|
- if len(v.Highlight["Title"]) > 0 {
|
|
|
- title = v.Highlight["Title"][0]
|
|
|
- } else {
|
|
|
- title = article.Title
|
|
|
- }
|
|
|
- searchItem.Title = title
|
|
|
- searchItem.PublishDate = article.PublishDate
|
|
|
- result = append(result, searchItem)
|
|
|
- searchMap[article.ArticleId] = article.ArticleId
|
|
|
- }
|
|
|
- }
|
|
|
- }
|
|
|
- return
|
|
|
-}
|
|
|
-
|
|
|
-func RemoveDuplicatesAndEmpty(a []string) (ret []string) {
|
|
|
- a_len := len(a)
|
|
|
- for i := 0; i < a_len; i++ {
|
|
|
- if (i > 0 && a[i-1] == a[i]) || len(a[i]) == 0 {
|
|
|
- continue
|
|
|
- }
|
|
|
- ret = append(ret, a[i])
|
|
|
- }
|
|
|
- return
|
|
|
-}
|
|
|
-
|
|
|
-func init21123() {
|
|
|
- fmt.Println("start")
|
|
|
- client, err := elastic.NewClient(elastic.SetURL(ES_URL), elastic.SetBasicAuth(ES_USERNAME, ES_PASSWORD), elastic.SetSniff(false))
|
|
|
- if err != nil {
|
|
|
- fmt.Println("err:", err)
|
|
|
- }
|
|
|
- fmt.Println(client)
|
|
|
- keyWordStr := "医疗器械"
|
|
|
- queryString := elastic.NewQueryStringQuery(`Title:(` + keyWordStr + `)`)
|
|
|
-
|
|
|
- boolqueryJson, err := json.Marshal(queryString)
|
|
|
- fmt.Println("err:", err)
|
|
|
- fmt.Println("queryString ", string(boolqueryJson))
|
|
|
-
|
|
|
- var esIndex = "cygx_article"
|
|
|
- //boolquery.Must(elastic.NewMatchQuery("Title", keyWord), elastic.NewMatchQuery("BodyText", keyWord))
|
|
|
- highlight := elastic.NewHighlight()
|
|
|
- highlight = highlight.Fields(elastic.NewHighlighterField("Title"), elastic.NewHighlighterField("BodyText"))
|
|
|
- highlight = highlight.PreTags("<font color='red'>").PostTags("</font>")
|
|
|
- request := client.Search(esIndex).Highlight(highlight).Query(queryString)
|
|
|
-
|
|
|
- requestJson, err := json.Marshal(request)
|
|
|
- fmt.Println("err:", err)
|
|
|
- fmt.Println("requestJson ", string(requestJson))
|
|
|
-
|
|
|
- searchByMatch, err := request.Do(context.Background())
|
|
|
-
|
|
|
- if searchByMatch.Hits != nil {
|
|
|
-
|
|
|
- }
|
|
|
-
|
|
|
- fmt.Println(searchByMatch)
|
|
|
- fmt.Println("end")
|
|
|
-}
|
|
|
-
|
|
|
-func SearchByKeyWordQuery(keyWord string) (result []*models.SearchItem, err error) {
|
|
|
- errorlog := log.New(os.Stdout, "APP", log.LstdFlags)
|
|
|
- file := "./rdlucklog/eslog.log"
|
|
|
- logFile, _ := os.OpenFile(file, os.O_RDWR|os.O_CREATE|os.O_APPEND, 0766)
|
|
|
-
|
|
|
- client, err := elastic.NewClient(
|
|
|
- elastic.SetURL(ES_URL),
|
|
|
- elastic.SetBasicAuth(ES_USERNAME, ES_PASSWORD),
|
|
|
- elastic.SetTraceLog(log.New(logFile, "ES-TRACE: ", 0)),
|
|
|
- elastic.SetSniff(false), elastic.SetErrorLog(errorlog))
|
|
|
-
|
|
|
- var esIndex = "cygx_article"
|
|
|
-
|
|
|
- //keyWordStr := "医疗"
|
|
|
- //字段相等
|
|
|
- queryString := elastic.NewQueryStringQuery("Title:医疗")
|
|
|
- //.Analyzer("ik_smart")
|
|
|
- highlight := elastic.NewHighlight()
|
|
|
- highlight = highlight.Fields(elastic.NewHighlighterField("Title"), elastic.NewHighlighterField("BodyText"))
|
|
|
- highlight = highlight.PreTags("<font color='red'>").PostTags("</font>")
|
|
|
- request := client.Search(esIndex).Highlight(highlight).Query(queryString)
|
|
|
-
|
|
|
- searchByMatch, err := request.Do(context.Background())
|
|
|
- if err != nil {
|
|
|
- fmt.Println("request.Do err:", err.Error())
|
|
|
- return result, err
|
|
|
- }
|
|
|
- if searchByMatch.Hits != nil {
|
|
|
- for _, v := range searchByMatch.Hits.Hits {
|
|
|
- articleJson, _ := v.Source.MarshalJSON()
|
|
|
- utils.FileLog.Info("%s", string(articleJson))
|
|
|
- fmt.Println(string(articleJson))
|
|
|
- }
|
|
|
- }
|
|
|
- return
|
|
|
-}
|
|
|
-
|
|
|
-func SearchByKeyWord(keyWord string) (result []*models.SearchItem, err error) {
|
|
|
- fmt.Println("keyWord:", keyWord)
|
|
|
- keyWord = "医药"
|
|
|
- pageSize := 20
|
|
|
- keyWordArr, err := GetIndustryMapNameSlice(keyWord)
|
|
|
- keyWordArr = RemoveDuplicatesAndEmpty(keyWordArr)
|
|
|
-
|
|
|
- fmt.Println(keyWordArr)
|
|
|
- fmt.Println(" keyWordArr ")
|
|
|
-
|
|
|
- if err != nil {
|
|
|
- go utils.SendEmail(utils.APPNAME+" "+utils.RunMode+"异常提醒:", "GetIndustryMapNameSlice："+err.Error(), utils.EmailSendToUsers)
|
|
|
- }
|
|
|
- errorlog := log.New(os.Stdout, "APP", log.LstdFlags)
|
|
|
- file := "./rdlucklog/eslog.log"
|
|
|
- logFile, _ := os.OpenFile(file, os.O_RDWR|os.O_CREATE|os.O_APPEND, 0766)
|
|
|
-
|
|
|
- client, err := elastic.NewClient(
|
|
|
- elastic.SetURL(ES_URL),
|
|
|
- elastic.SetBasicAuth(ES_USERNAME, ES_PASSWORD),
|
|
|
- elastic.SetTraceLog(log.New(logFile, "ES-TRACE: ", 0)),
|
|
|
- elastic.SetSniff(false), elastic.SetErrorLog(errorlog))
|
|
|
-
|
|
|
- //var esIndex = "cygx_article"
|
|
|
- var esIndex = "article_two"
|
|
|
- searchMap := make(map[int]int)
|
|
|
-
|
|
|
- //queryString := elastic.NewQueryStringQuery(`Title:医疗 BodyText:医疗`).Analyzer("ik_smart")
|
|
|
-
|
|
|
- //queryTeerms:=elastic.NewTermsQuery("Title","医疗","费用")
|
|
|
-
|
|
|
- //queryMatchPhrase:=elastic.NewMatchPhraseQuery("Title","费用")
|
|
|
- //.Analyzer("ik_smart")
|
|
|
- //.Analyzer("ik_smart")
|
|
|
- //boolquery.Must(elastic.NewMatchQuery("Title", keyWord), elastic.NewMatchQuery("BodyText", keyWord))
|
|
|
- mapping := client.GetMapping()
|
|
|
- mapJson, err := json.Marshal(mapping)
|
|
|
- utils.FileLog.Info("%s", string(mapJson))
|
|
|
- boolquery := elastic.NewBoolQuery()
|
|
|
-
|
|
|
- keyLen := len(keyWordArr)
|
|
|
- n := float64(keyLen)
|
|
|
- matchArr := make([]elastic.Query, 0)
|
|
|
- for _, v := range keyWordArr {
|
|
|
- if v != "" {
|
|
|
- matchq1 := elastic.NewMatchQuery("Title", v).Boost(n).Analyzer("ik_smart")
|
|
|
- matchq2 := elastic.NewMatchQuery("BodyText", v).Boost(n).Analyzer("ik_smart")
|
|
|
- matchArr = append(matchArr, matchq1)
|
|
|
- matchArr = append(matchArr, matchq2)
|
|
|
- }
|
|
|
- n--
|
|
|
- }
|
|
|
- //matchArr=append(matchArr,matchq1)
|
|
|
- //matchArr=append(matchArr,matchq2)
|
|
|
-
|
|
|
- boolquery.Should(matchArr...)
|
|
|
- //boolquery.Must(elastic.NewMatchQuery("BodyText", keyWord).Boost(1).Analyzer("ik_smart"))
|
|
|
-
|
|
|
- highlight := elastic.NewHighlight()
|
|
|
- highlight = highlight.Fields(elastic.NewHighlighterField("Title"), elastic.NewHighlighterField("BodyText"))
|
|
|
- highlight = highlight.PreTags("<font color='red'>").PostTags("</font>")
|
|
|
- //request := client.Search(esIndex).Highlight(highlight).Size(pageSize).Query(queryString)
|
|
|
- request := client.Search(esIndex).Highlight(highlight).Size(pageSize).Query(boolquery)
|
|
|
- searchByMatch, err := request.Do(context.Background())
|
|
|
- fmt.Println(searchByMatch, err)
|
|
|
- //searchByMatch, err := client.Search(esIndex).Highlight(highlight).Size(pageSize).Do(context.Background())
|
|
|
- if err != nil {
|
|
|
- return result, err
|
|
|
- }
|
|
|
- fmt.Println(searchByMatch.Status, searchByMatch.TotalHits())
|
|
|
- if searchByMatch.Hits != nil {
|
|
|
- for _, v := range searchByMatch.Hits.Hits {
|
|
|
- articleJson, err := v.Source.MarshalJSON()
|
|
|
- if err != nil {
|
|
|
- return nil, err
|
|
|
- }
|
|
|
- article := new(models.CygxArticle)
|
|
|
- err = json.Unmarshal(articleJson, &article)
|
|
|
- if err != nil {
|
|
|
- return nil, err
|
|
|
- }
|
|
|
- if _, ok := searchMap[article.ArticleId]; !ok {
|
|
|
- searchItem := new(models.SearchItem)
|
|
|
- searchItem.ArticleId, _ = strconv.Atoi(v.Id)
|
|
|
- searchItem.Body = v.Highlight["BodyText"]
|
|
|
- var title string
|
|
|
- if len(v.Highlight["Title"]) > 0 {
|
|
|
- title = v.Highlight["Title"][0]
|
|
|
- } else {
|
|
|
- title = article.Title
|
|
|
- }
|
|
|
- searchItem.Title = title
|
|
|
- searchItem.PublishDate = article.PublishDate
|
|
|
- result = append(result, searchItem)
|
|
|
- searchMap[article.ArticleId] = article.ArticleId
|
|
|
- }
|
|
|
- }
|
|
|
- }
|
|
|
- return
|
|
|
-}
|
|
|
-
|
|
|
-func AddMap() {
|
|
|
- errorlog := log.New(os.Stdout, "APP", log.LstdFlags)
|
|
|
- file := "./rdlucklog/eslog.log"
|
|
|
- logFile, _ := os.OpenFile(file, os.O_RDWR|os.O_CREATE|os.O_APPEND, 0766)
|
|
|
-
|
|
|
- client, err := elastic.NewClient(
|
|
|
- elastic.SetURL(ES_URL),
|
|
|
- elastic.SetBasicAuth(ES_USERNAME, ES_PASSWORD),
|
|
|
- elastic.SetTraceLog(log.New(logFile, "ES-TRACE: ", 0)),
|
|
|
- elastic.SetSniff(false), elastic.SetErrorLog(errorlog))
|
|
|
- if err != nil {
|
|
|
- fmt.Println("NewClient Err:", err.Error())
|
|
|
- }
|
|
|
-
|
|
|
- bodyJson := `{
|
|
|
- "mappings": {
|
|
|
- "properties": {
|
|
|
- "ArticleId": {
|
|
|
- "type": "integer"
|
|
|
- },
|
|
|
- "Title": {
|
|
|
- "type": "text",
|
|
|
- "analyzer": "ik_smart"
|
|
|
- },
|
|
|
- "BodyText": {
|
|
|
- "type": "text",
|
|
|
- "analyzer": "ik_smart"
|
|
|
- }
|
|
|
- }
|
|
|
- }
|
|
|
-}`
|
|
|
- //var esMappingIndex = "article_mapping"
|
|
|
- var esIndex = "article_two"
|
|
|
-
|
|
|
- exists, err := client.IndexExists(esIndex).Do(context.Background()) //<5>
|
|
|
- if err != nil {
|
|
|
- fmt.Println("IndexExists Err:" + err.Error())
|
|
|
- return
|
|
|
- }
|
|
|
- fmt.Println("exists:", exists)
|
|
|
- if !exists {
|
|
|
- _, err = client.CreateIndex(esIndex).BodyJson(bodyJson).Do(context.Background())
|
|
|
- //BodyJson(bodyJson).Do(context.Background())
|
|
|
- if err != nil {
|
|
|
- fmt.Println("CreateIndex Err:" + err.Error())
|
|
|
- return
|
|
|
- }
|
|
|
- }
|
|
|
-
|
|
|
- item, err := models.GetArticleDetailTestById(3584)
|
|
|
- if err != nil {
|
|
|
- fmt.Println("GetArticleDetailById Err:" + err.Error())
|
|
|
- return
|
|
|
- }
|
|
|
- content := html.UnescapeString(item.Body)
|
|
|
- doc, err := goquery.NewDocumentFromReader(strings.NewReader(content))
|
|
|
- if err != nil {
|
|
|
- fmt.Println("create doc err:", err.Error())
|
|
|
- return
|
|
|
- }
|
|
|
- bodyText := doc.Text()
|
|
|
- item.BodyText = bodyText
|
|
|
- //新增
|
|
|
- resp, err := client.Index().Index(esIndex).Id(strconv.Itoa(item.ArticleId)).BodyJson(item).Do(context.Background())
|
|
|
- if err != nil {
|
|
|
- fmt.Println("insert es failed", err.Error())
|
|
|
- return
|
|
|
- }
|
|
|
- fmt.Println(resp)
|
|
|
- //mappping := make(map[string]interface{})
|
|
|
- //mappping[esMappingIndex] = bodyJson
|
|
|
- //putResult,err:=client.PutMapping().Index(esIndex).BodyJson(mappping).Do(context.Background())
|
|
|
- //mapping,err := client.GetMapping().Index(esIndex).Do(context.Background())
|
|
|
- //fmt.Println(mapping)
|
|
|
- //mapJson, err := json.Marshal(mapping)
|
|
|
- //utils.FileLog.Info("%s", string(mapJson))
|
|
|
- //fmt.Println(esMappingIndex)
|
|
|
- //
|
|
|
- //fmt.Println(err)
|
|
|
- //fmt.Println(putResult)
|
|
|
-}
|
