package services import ( "context" "encoding/json" "fmt" "hongze/hongze_cygx/models" "hongze/hongze_cygx/utils" "html" "strconv" "strings" //"gopkg.in/olivere/elastic.v5" //"gopkg.in/olivere/elastic.v5/config" "github.com/PuerkitoBio/goquery" "github.com/olivere/elastic/v7" "github.com/olivere/elastic/v7/config" ) const ( ES_URL = "http://es-cn-nif227b580019rgw6.public.elasticsearch.aliyuncs.com:9200" //<1> ES_USERNAME = "elastic" //<2> ES_PASSWORD = "hongze@2021" //<3> //Grafana pwd-> 20521bb9 //Grafana username-> emon ) func SaveData() { //fmt.Println("start") var sniff = false //<4> cfg := &config.Config{ URL: ES_URL, Username: ES_USERNAME, Password: ES_PASSWORD, } cfg.Sniff = &sniff var client, err = elastic.NewClientFromConfig(cfg) if err != nil { fmt.Println("NewClientFromConfig Err:" + err.Error()) return } var esIndex = "cygx_article" //var esType = "article" // exists, err := client.IndexExists(esIndex).Do(context.Background()) //<5> if err != nil { fmt.Println("IndexExists Err:" + err.Error()) return } if !exists { _, err = client.CreateIndex(esIndex).Do(context.Background()) if err != nil { fmt.Println("CreateIndex Err:" + err.Error()) return } } /* 3161,3190,3226,3244,3264,3285,3310,3334,3370,3397,3418,3446,3477,3497,3526,3554 */ idStr := `3161,3190,3226,3244,3264,3285,3310,3334,3370,3397,3418,3446,3477,3497,3526,3554` idArr := strings.Split(idStr, ",") for _, v := range idArr { id, _ := strconv.Atoi(v) item, err := models.GetArticleDetailById(id) if err != nil { fmt.Println("GetArticleDetailById Err:" + err.Error()) return } content := html.UnescapeString(item.Body) doc, err := goquery.NewDocumentFromReader(strings.NewReader(content)) if err != nil { fmt.Println("create doc err:", err.Error()) return } bodyText := doc.Text() item.BodyText = bodyText //删除 resp, err := client.Index().Index(esIndex).Id(strconv.Itoa(item.ArticleId)).BodyJson(item).Do(context.Background()) if err != nil { fmt.Println("insert es failed", err.Error()) return } fmt.Println(resp.Status) } /* //根据id查询 searchById, err := client.Get().Index(esIndex).Type(esType).Id("3138").Do(context.Background()) if searchById.Found { body, err := searchById.Source.MarshalJSON() fmt.Println("body:",string(body)) fmt.Println(err) //var resultType models.ArticleDetail //if err := json.Unmarshal(searchById.Source,&resultType); err != nil{ // log.Error(err.Error()) //} //fmt.Printf("search by id: %#v \n",resultType) } */ ////查询index中所有的数据 //var resultType models.CygxArticle //searchAll,err := client.Search(esIndex).Type(esType).Do(context.Background()) //for _,item := range searchAll.Each(reflect.TypeOf(resultType)) { // language := item.(models.CygxArticle) // fmt.Printf("search by index all: %#v \n",language) //} //根据检索条件查询 // boolquery := elastic.NewBoolQuery() // boolquery.Should(elastic.NewMatchQuery("Body", "专家")) // highlight := elastic.NewHighlight() // highlight = highlight.Fields(elastic.NewHighlighterField("Body")) // highlight = highlight.PreTags("").PostTags("") // var pageSize int // pageSize = 20 // searchByMatch, err := client.Search(esIndex).Highlight(highlight).Size(pageSize).Query(boolquery).Do(context.Background()) // var result string // if searchByMatch.Hits != nil { // } // //fmt.Println(string(result)) // utils.FileLog.Info("%s", string(result)) //var resultType models.CygxArticle //for k,item := range searchByMatch.Each(reflect.TypeOf(resultType)) { // language := item.(models.CygxArticle) // fmt.Printf("search by match: %#v \n",language) // // fmt.Println(k) // result,err:=json.Marshal(language) // fmt.Println(err) // utils.FileLog.Info("%s",string(result)) //} // Perform the search request. //searchByMatch, err := client.Search(esIndex).Type(esType).Query(query).From(1).Size(10).Do(context.Background()) fmt.Println("end") } func SearchByKeyWord(keyWord string, pageSize int) (result []*models.SearchItem, err error) { if pageSize == 0 { pageSize = 20 } var sniff = false //<4> cfg := &config.Config{ URL: ES_URL, Username: ES_USERNAME, Password: ES_PASSWORD, } cfg.Sniff = &sniff client, err := elastic.NewClientFromConfig(cfg) if err != nil { return } var esIndex = "cygx_article" fmt.Println("get start keyWordArr") keyWordArr, err := GetIndustryMapNameSlice(keyWord) if err != nil { go utils.SendEmail(utils.APPNAME+" "+utils.RunMode+"异常提醒:", "GetIndustryMapNameSlice:"+err.Error(), utils.EmailSendToUsers) } keyWordArr = append(keyWordArr, keyWord) fmt.Println(keyWord) fmt.Println(keyWordArr) var newKeyWordArr []string if len(keyWordArr) > 2 { newKeyWordArr = append(keyWordArr[2:], keyWordArr[0:2]...) } fmt.Println(newKeyWordArr) fmt.Println("get end keyWordArr") searchMap := make(map[int]int) for k, v := range newKeyWordArr { fmt.Println(k, v) boolquery := elastic.NewBoolQuery() boolquery.Must(elastic.NewMatchQuery("Title", keyWord), elastic.NewMatchQuery("BodyText", keyWord)) highlight := elastic.NewHighlight() highlight = highlight.Fields(elastic.NewHighlighterField("Title"), elastic.NewHighlighterField("BodyText")) highlight = highlight.PreTags("").PostTags("") searchByMatch, err := client.Search(esIndex).Highlight(highlight).Size(pageSize).Query(boolquery).Do(context.Background()) if err != nil { return result, err } if searchByMatch.Hits != nil { for _, v := range searchByMatch.Hits.Hits { articleJson, err := v.Source.MarshalJSON() if err != nil { return nil, err } article := new(models.CygxArticle) err = json.Unmarshal(articleJson, &article) if err != nil { return nil, err } if _, ok := searchMap[article.ArticleId]; !ok { searchItem := new(models.SearchItem) searchItem.ArticleId, _ = strconv.Atoi(v.Id) searchItem.Body = v.Highlight["BodyText"] var title string if len(v.Highlight["Title"]) > 0 { title = v.Highlight["Title"][0] } else { title = article.Title } searchItem.Title = title searchItem.PublishDate = article.PublishDate result = append(result, searchItem) searchMap[article.ArticleId] = article.ArticleId } } } } return }