package elastic import ( "context" "encoding/json" "errors" "eta/eta_mobile/models/data_manage" "eta/eta_mobile/utils" "fmt" "github.com/olivere/elastic/v7" "strconv" ) // indexName:索引名称 // mappingJson:表结构 func EsCreateIndex(indexName, mappingJson string) (err error) { client := utils.EsClient //定义表结构 exists, err := client.IndexExists(indexName).Do(context.Background()) //<5> if err != nil { return } if !exists { resp, err := client.CreateIndex(indexName).BodyJson(mappingJson).Do(context.Background()) //BodyJson(bodyJson).Do(context.Background()) if err != nil { fmt.Println("CreateIndex Err:" + err.Error()) return err } fmt.Println(resp.Index, resp.ShardsAcknowledged, resp.Acknowledged) } else { fmt.Println(indexName + " 已存在") } return } // 新增和修改数据 //func EsAddOrEditData(item *cygx.CygxArticle) (err error) { // defer func() { // if err != nil { // fmt.Println("EsAddOrEditData Err:", err.Error()) // } // }() // indexName := utils.IndexName // content := html.UnescapeString(item.Body) // doc, errDoc := goquery.NewDocumentFromReader(strings.NewReader(content)) // if errDoc != nil { // return // } // doc.Find("a").Each(func(i int, a *goquery.Selection) { // a.Remove() // }) // bodyText := doc.Text() // client, err := NewClient() // if err != nil { // fmt.Println(err, "err1") // return // } // docId := strconv.Itoa(item.ArticleId) // searchById, err := client.Get().Index(indexName).Id(docId).Do(context.Background()) // if err != nil && !strings.Contains(err.Error(), "404") { // fmt.Println("Get Err" + err.Error()) // return // } // if err != nil && strings.Contains(err.Error(), "404") { // err = nil // } // if searchById != nil && searchById.Found { // resp, err := client.Update().Index(indexName).Id(docId).Doc(map[string]interface{}{ // "BodyText": bodyText, // "Title": item.Title, // "PublishDate": item.PublishDate.Format(utils.FormatDateTime), // "CategoryId": item.CategoryId, // "ExpertBackground": item.ExpertBackground, // }).Do(context.Background()) // if err != nil { // fmt.Println(err, "err") // return err // } // if resp.Status == 0 { // fmt.Println("修改成功") // } else { // fmt.Println("EditData", resp.Status, resp.Result) // } // } else { // itemEs := new(cygx.ElasticTestArticleDetail) // itemEs.ArticleId = item.ArticleId // itemEs.Title = item.Title // itemEs.PublishDate = item.PublishDate.Format(utils.FormatDateTime) // itemEs.BodyText = bodyText // itemEs.CategoryId = strconv.Itoa(item.CategoryId) // itemEs.ExpertBackground = item.ExpertBackground // resp, err := client.Index().Index(indexName).Id(docId).BodyJson(itemEs).Do(context.Background()) // if err != nil { // fmt.Println("新增失败:", err.Error()) // return err // } // if resp.Status == 0 && resp.Result == "created" { // fmt.Println("新增成功") // err = nil // } else { // fmt.Println("AddData", resp.Status, resp.Result) // } // } // return //} // 删除数据 func EsDeleteData(indexName, docId string) (err error) { client := utils.EsClient resp, err := client.Delete().Index(indexName).Id(docId).Do(context.Background()) fmt.Println(resp) if err != nil { return } if resp.Status == 0 { fmt.Println("删除成功") } else { fmt.Println("AddData", resp.Status, resp.Result) } return } func MappingModify(indexName, mappingJson string) { client := utils.EsClient result, err := client.PutMapping().Index(indexName).BodyString(mappingJson).Do(context.Background()) fmt.Println(err) fmt.Println(result) return } // EsAddOrEditEdbInfoData 新增/修改es中的指标数据 func EsAddOrEditEdbInfoData(indexName, docId string, item *data_manage.EdbInfoList) (err error) { defer func() { if err != nil { fmt.Println("EsAddOrEditData Err:", err.Error()) } }() client := utils.EsClient resp, err := client.Index().Index(indexName).Id(docId).BodyJson(item).Do(context.Background()) if err != nil { fmt.Println("新增失败:", err.Error()) return err } fmt.Println(resp) if resp.Status == 0 { fmt.Println("新增成功", resp.Result) err = nil } else { fmt.Println("AddData", resp.Status, resp.Result) } return } // SearchEdbInfoData 查询es中的指标数据 func SearchEdbInfoData(indexName, keywordStr string, from, size, filterSource, source int, edbInfoType int8, frequency string, noPermissionEdbInfoIdList []int) (total int64, list []*data_manage.EdbInfoList, err error) { list = make([]*data_manage.EdbInfoList, 0) defer func() { if err != nil { fmt.Println("EsAddOrEditData Err:", err.Error()) } }() highlight := elastic.NewHighlight() highlight = highlight.Fields(elastic.NewHighlighterField("EdbCode"), elastic.NewHighlighterField("EdbName")) highlight = highlight.PreTags("").PostTags("") //var source map[string]interface{} //source := map[string]interface{}{ // "query": map[string]interface{}{ // "match_all": map[string]interface{}{}, // }, //} mustMap := make([]interface{}, 0) mustNotMap := make([]interface{}, 0) //source := map[string]interface{}{ // "query": map[string]interface{}{ // "bool": map[string]interface{}{ // "must": map[string]interface{}{ // "query_string": map[string]interface{}{ // "query": keywordStr, // "fields": []string{"EdbCode", "EdbName"}, // }, // }, // }, // }, //} switch filterSource { case 2: //source = map[string]interface{}{ // "query": map[string]interface{}{ // "bool": map[string]interface{}{ // "must": map[string]interface{}{ // "query_string": map[string]interface{}{ // "query": keywordStr, // }, // }, // "filter": []interface{}{ // map[string]interface{}{ // "term": map[string]interface{}{ // "Frequency.keyword": "月度", // }, // }}, // }, // }, //} mustMap = []interface{}{ map[string]interface{}{ "term": map[string]interface{}{ "Frequency.keyword": "月度", //"Frequency.keyword": "月度", }, }, } case 3: //source = map[string]interface{}{ // "query": map[string]interface{}{ // "bool": map[string]interface{}{ // "must": map[string]interface{}{ // "query_string": map[string]interface{}{ // "query": keywordStr, // }, // }, // "must_not": []interface{}{ // map[string]interface{}{ // "match": map[string]interface{}{ // "Frequency.keyword": "日度", // }, // }}, // }, // }, //} ////注释掉,所有频度都可以变频 2022-08-31 14:31:28 //mustNotMap = []interface{}{ // map[string]interface{}{ // "match": map[string]interface{}{ // "Frequency.keyword": "日度", // //"Frequency.keyword": "月度", // }, // }, //} case 4: //source = map[string]interface{}{ // "query": map[string]interface{}{ // "bool": map[string]interface{}{ // "must": map[string]interface{}{ // "query_string": map[string]interface{}{ // "query": keywordStr, // }, // }, // "filter": []interface{}{ // map[string]interface{}{ // "term": map[string]interface{}{ // "EdbType": 1, // }, // }}, // }, // }, //} mustMap = []interface{}{ map[string]interface{}{ "term": map[string]interface{}{ "EdbType": 1, }, }, } case 5: mustMap = []interface{}{ map[string]interface{}{ "term": map[string]interface{}{ "Source": 6, }, }, } case 6: mustNotMap = []interface{}{ map[string]interface{}{ "match": map[string]interface{}{ "Frequency.keyword": "年度", }, }, } } //指标来源 if source > 0 { mustMap = append(mustMap, map[string]interface{}{ "term": map[string]interface{}{ "Source": source, //"Frequency.keyword": "月度", }, }) } if frequency != "" { mustMap = append(mustMap, map[string]interface{}{ "term": map[string]interface{}{ "Frequency.keyword": frequency, //"Frequency.keyword": "月度", }, }) } // noPermissionEdbInfoIdList 无权限指标id if len(noPermissionEdbInfoIdList) > 0 { mustNotMap = append(mustNotMap, map[string]interface{}{ "terms": map[string]interface{}{ "EdbInfoId": noPermissionEdbInfoIdList, //"Frequency.keyword": "月度", }, }) } // 指标类型:普通指标、预测指标(小于0 代表不区分指标是普通还是预测) if edbInfoType >= 0 { mustMap = append(mustMap, map[string]interface{}{ "term": map[string]interface{}{ "EdbInfoType": edbInfoType, }, }) } //普通指标 //mustMap = append(mustMap, map[string]interface{}{ // "term": map[string]interface{}{ // "EdbInfoType": 0, // //"Frequency.keyword": "月度", // }, //}) //关键字匹配 shouldMap := map[string]interface{}{ "should": []interface{}{ map[string]interface{}{ "match": map[string]interface{}{ "EdbCode": keywordStr, //"Frequency.keyword": "月度", }, }, map[string]interface{}{ "match": map[string]interface{}{ "EdbName": keywordStr, //"Frequency.keyword": "月度", }, }, map[string]interface{}{ "match": map[string]interface{}{ "EdbNameEn": keywordStr, //"Frequency.keyword": "月度", }, }, }, } mustMap = append(mustMap, map[string]interface{}{ "bool": shouldMap, }) return searchEdbInfoData(indexName, mustMap, mustNotMap, shouldMap, from, size) } func SearchEdbInfoDataBak(indexName, keywordStr string, from, size, filterSource, source int, frequency string) (total int64, list []*data_manage.EdbInfoList, err error) { list = make([]*data_manage.EdbInfoList, 0) defer func() { if err != nil { fmt.Println("EsAddOrEditData Err:", err.Error()) } }() client := utils.EsClient //queryString := elastic.NewQueryStringQuery(keywordStr) //boolQueryJson, err := json.Marshal(queryString) //if err != nil { // fmt.Println("boolQueryJson err:", err) //} else { // fmt.Println("boolQueryJson ", string(boolQueryJson)) //} highlight := elastic.NewHighlight() highlight = highlight.Fields(elastic.NewHighlighterField("EdbCode"), elastic.NewHighlighterField("EdbName")) highlight = highlight.PreTags("").PostTags("") //query := elastic.RawStringQuery(`{"match_all":{}}`) //var source map[string]interface{} //source := map[string]interface{}{ // "query": map[string]interface{}{ // "match_all": map[string]interface{}{}, // }, //} mustMap := make([]interface{}, 0) mustNotMap := make([]interface{}, 0) //source := map[string]interface{}{ // "query": map[string]interface{}{ // "bool": map[string]interface{}{ // "must": map[string]interface{}{ // "query_string": map[string]interface{}{ // "query": keywordStr, // "fields": []string{"EdbCode", "EdbName"}, // }, // }, // }, // }, //} switch filterSource { case 2: //source = map[string]interface{}{ // "query": map[string]interface{}{ // "bool": map[string]interface{}{ // "must": map[string]interface{}{ // "query_string": map[string]interface{}{ // "query": keywordStr, // }, // }, // "filter": []interface{}{ // map[string]interface{}{ // "term": map[string]interface{}{ // "Frequency.keyword": "月度", // }, // }}, // }, // }, //} mustMap = []interface{}{ map[string]interface{}{ "term": map[string]interface{}{ "Frequency.keyword": "月度", //"Frequency.keyword": "月度", }, }, } case 3: //source = map[string]interface{}{ // "query": map[string]interface{}{ // "bool": map[string]interface{}{ // "must": map[string]interface{}{ // "query_string": map[string]interface{}{ // "query": keywordStr, // }, // }, // "must_not": []interface{}{ // map[string]interface{}{ // "match": map[string]interface{}{ // "Frequency.keyword": "日度", // }, // }}, // }, // }, //} ////注释掉,所有频度都可以变频 2022-08-31 14:31:28 //mustNotMap = []interface{}{ // map[string]interface{}{ // "match": map[string]interface{}{ // "Frequency.keyword": "日度", // //"Frequency.keyword": "月度", // }, // }, //} case 4: //source = map[string]interface{}{ // "query": map[string]interface{}{ // "bool": map[string]interface{}{ // "must": map[string]interface{}{ // "query_string": map[string]interface{}{ // "query": keywordStr, // }, // }, // "filter": []interface{}{ // map[string]interface{}{ // "term": map[string]interface{}{ // "EdbType": 1, // }, // }}, // }, // }, //} mustMap = []interface{}{ map[string]interface{}{ "term": map[string]interface{}{ "EdbType": 1, }, }, } case 5: mustMap = []interface{}{ map[string]interface{}{ "term": map[string]interface{}{ "Source": 6, }, }, } } //指标来源 if source > 0 { mustMap = append(mustMap, map[string]interface{}{ "term": map[string]interface{}{ "Source": source, //"Frequency.keyword": "月度", }, }) } if frequency != "" { mustMap = append(mustMap, map[string]interface{}{ "term": map[string]interface{}{ "Frequency.keyword": frequency, //"Frequency.keyword": "月度", }, }) } //普通指标 //mustMap = append(mustMap, map[string]interface{}{ // "term": map[string]interface{}{ // "EdbInfoType": 0, // //"Frequency.keyword": "月度", // }, //}) //关键字匹配 shouldMap := map[string]interface{}{ "should": []interface{}{ map[string]interface{}{ "match": map[string]interface{}{ "EdbCode": keywordStr, //"Frequency.keyword": "月度", }, }, map[string]interface{}{ "match": map[string]interface{}{ "EdbName": keywordStr, //"Frequency.keyword": "月度", }, }, // 因为关键词被分了,所以需要用下面的语句来让他 整个词 查询,从而加重整词的权重 map[string]interface{}{ "match": map[string]interface{}{ "EdbCode": map[string]interface{}{ "query": keywordStr, "operator": "and", }, //"Frequency.keyword": "月度", }, }, // 因为关键词被分了,所以需要用下面的语句来让他 整个词 查询,从而加重整词的权重 map[string]interface{}{ "match": map[string]interface{}{ "EdbName": map[string]interface{}{ "query": keywordStr, "operator": "and", }, //"Frequency.keyword": "月度", }, }, }, } mustMap = append(mustMap, map[string]interface{}{ "bool": shouldMap, }) queryMap := map[string]interface{}{ "query": map[string]interface{}{ "bool": map[string]interface{}{ "must": mustMap, "must_not": mustNotMap, //"should": shouldMap, }, }, } //根据条件数量统计 requestTotalHits := client.Count(indexName).BodyJson(queryMap) total, err = requestTotalHits.Do(context.Background()) if err != nil { return } queryMap["from"] = from queryMap["size"] = size jsonBytes, _ := json.Marshal(queryMap) fmt.Println(string(jsonBytes)) //queryStr := fmt.Sprintf(`{"query":{"bool":{"must":{"query_string":{"query":"%s","fields":["EdbCode","EdbName"]}}}}}`, keywordStr) //switch filterSource { //case 2: // queryStr = fmt.Sprintf(`{"query":{"bool":{"must":{"query_string":{"query":"%s","fields":["EdbCode","EdbName"]}},"filter":{"term":{"Frequency.keyword":"%s"}}}}}`, keywordStr, "月度") //case 3: // queryStr = fmt.Sprintf(`{"query":{"bool":{"must":{"query_string":{"query":"%s","fields":["EdbCode","EdbName"]}},"must_not":[{"match":{"Frequency.keyword":"%s"}}]}}}`, keywordStr, "日度") //case 4: // queryStr = fmt.Sprintf(`{"query":{"bool":{"must":{"query_string":{"query":"%s","fields":["EdbCode","EdbName"]}},"must_not":[{"match":{"EdbType":1}}]}}}`, keywordStr) //} //queryString := elastic.RawStringQuery(queryStr) //fmt.Println("queryString:", queryString) //queryString := elastic.NewMatchQuery("EdbCode", keywordStr) //request := client.Search(indexName).Highlight(highlight).From(from).Size(size).Query(queryString) request := client.Search(indexName).Highlight(highlight).Source(queryMap) // sets the JSON request //requestJson, err := json.Marshal(request) //if err != nil { // fmt.Println("requestJson err:", err) //} //fmt.Println("requestJson ", string(requestJson)) searchMap := make(map[string]string) searchResp, err := request.Do(context.Background()) if err != nil { return } fmt.Println(searchResp) fmt.Println(searchResp.Status) if searchResp.Status != 0 { return } if searchResp.Hits != nil { for _, v := range searchResp.Hits.Hits { if _, ok := searchMap[v.Id]; !ok { itemJson, tmpErr := v.Source.MarshalJSON() if tmpErr != nil { err = tmpErr fmt.Println("movieJson err:", err) return } edbInfoItem := new(data_manage.EdbInfoList) tmpErr = json.Unmarshal(itemJson, &edbInfoItem) if err != nil { fmt.Println("json.Unmarshal movieJson err:", err) err = tmpErr return } if len(v.Highlight["EdbCode"]) > 0 { edbInfoItem.EdbCode = v.Highlight["EdbCode"][0] } if len(v.Highlight["EdbName"]) > 0 { edbInfoItem.EdbCode = v.Highlight["EdbName"][0] } list = append(list, edbInfoItem) searchMap[v.Id] = v.Id } } } //for _, v := range result { // fmt.Println(v) //} return } // SearchAddPredictEdbInfoData 查询允许添加预测指标的数据 func SearchAddPredictEdbInfoData(indexName, keywordStr string, noPermissionEdbInfoIdList []int, from, size int) (total int64, list []*data_manage.EdbInfoList, err error) { list = make([]*data_manage.EdbInfoList, 0) defer func() { if err != nil { fmt.Println("EsAddOrEditData Err:", err.Error()) } }() highlight := elastic.NewHighlight() highlight = highlight.Fields(elastic.NewHighlighterField("EdbCode"), elastic.NewHighlighterField("EdbName")) highlight = highlight.PreTags("").PostTags("") mustMap := make([]interface{}, 0) mustNotMap := make([]interface{}, 0) mustNotMap = []interface{}{ //map[string]interface{}{ // "terms": map[string]interface{}{ // "Frequency.keyword": []string{"日度", "周度", "月度"}, // }, // //"match": map[string]interface{}{ // // "Frequency": []string{"日度", "周度", "月度"}, // // //"Frequency.keyword": []string{"日度", "周度", "月度"}, // //}, //}, } // 指标类型:普通指标、预算指标 mustMap = append(mustMap, map[string]interface{}{ "term": map[string]interface{}{ "EdbInfoType": 0, }, }) mustMap = append(mustMap, map[string]interface{}{ "terms": map[string]interface{}{ "Frequency.keyword": []string{"日度", "周度", "月度"}, }, }) //关键字匹配 shouldMap := map[string]interface{}{ "should": []interface{}{ map[string]interface{}{ "match": map[string]interface{}{ "EdbCode": keywordStr, //"Frequency.keyword": "月度", }, }, map[string]interface{}{ "match": map[string]interface{}{ "EdbName": keywordStr, //"Frequency.keyword": "月度", }, }, }, } // noPermissionEdbInfoIdList 无权限指标id if len(noPermissionEdbInfoIdList) > 0 { mustNotMap = append(mustNotMap, map[string]interface{}{ "terms": map[string]interface{}{ "EdbInfoId": noPermissionEdbInfoIdList, //"Frequency.keyword": "月度", }, }) } return searchEdbInfoData(indexName, mustMap, mustNotMap, shouldMap, from, size) } // searchEdbInfoData 查询es中的指标数据 func searchEdbInfoData(indexName string, mustMap, mustNotMap []interface{}, shouldMap map[string]interface{}, from, size int) (total int64, list []*data_manage.EdbInfoList, err error) { list = make([]*data_manage.EdbInfoList, 0) defer func() { if err != nil { fmt.Println("EsAddOrEditData Err:", err.Error()) } }() client := utils.EsClient //queryString := elastic.NewQueryStringQuery(keywordStr) //boolQueryJson, err := json.Marshal(queryString) //if err != nil { // fmt.Println("boolQueryJson err:", err) //} else { // fmt.Println("boolQueryJson ", string(boolQueryJson)) //} highlight := elastic.NewHighlight() highlight = highlight.Fields(elastic.NewHighlighterField("EdbCode"), elastic.NewHighlighterField("EdbName")) highlight = highlight.PreTags("").PostTags("") //query := elastic.RawStringQuery(`{"match_all":{}}`) //关键字匹配 mustMap = append(mustMap, map[string]interface{}{ "bool": shouldMap, }) queryMap := map[string]interface{}{ "query": map[string]interface{}{ "bool": map[string]interface{}{ "must": mustMap, "must_not": mustNotMap, //"should": shouldMap, }, }, } //根据条件数量统计 requestTotalHits := client.Count(indexName).BodyJson(queryMap) total, err = requestTotalHits.Do(context.Background()) if err != nil { return } queryMap["from"] = from queryMap["size"] = size jsonBytes, _ := json.Marshal(queryMap) fmt.Println(string(jsonBytes)) //queryStr := fmt.Sprintf(`{"query":{"bool":{"must":{"query_string":{"query":"%s","fields":["EdbCode","EdbName"]}}}}}`, keywordStr) //switch filterSource { //case 2: // queryStr = fmt.Sprintf(`{"query":{"bool":{"must":{"query_string":{"query":"%s","fields":["EdbCode","EdbName"]}},"filter":{"term":{"Frequency.keyword":"%s"}}}}}`, keywordStr, "月度") //case 3: // queryStr = fmt.Sprintf(`{"query":{"bool":{"must":{"query_string":{"query":"%s","fields":["EdbCode","EdbName"]}},"must_not":[{"match":{"Frequency.keyword":"%s"}}]}}}`, keywordStr, "日度") //case 4: // queryStr = fmt.Sprintf(`{"query":{"bool":{"must":{"query_string":{"query":"%s","fields":["EdbCode","EdbName"]}},"must_not":[{"match":{"EdbType":1}}]}}}`, keywordStr) //} //queryString := elastic.RawStringQuery(queryStr) //fmt.Println("queryString:", queryString) //queryString := elastic.NewMatchQuery("EdbCode", keywordStr) //request := client.Search(indexName).Highlight(highlight).From(from).Size(size).Query(queryString) request := client.Search(indexName).Highlight(highlight).Source(queryMap) // sets the JSON request //requestJson, err := json.Marshal(request) //if err != nil { // fmt.Println("requestJson err:", err) //} //fmt.Println("requestJson ", string(requestJson)) searchMap := make(map[string]string) searchResp, err := request.Do(context.Background()) if err != nil { return } fmt.Println(searchResp) fmt.Println(searchResp.Status) if searchResp.Status != 0 { return } //total = searchResp.TotalHits() if searchResp.Hits != nil { for _, v := range searchResp.Hits.Hits { if _, ok := searchMap[v.Id]; !ok { itemJson, tmpErr := v.Source.MarshalJSON() if tmpErr != nil { err = tmpErr fmt.Println("movieJson err:", err) return } edbInfoItem := new(data_manage.EdbInfoList) tmpErr = json.Unmarshal(itemJson, &edbInfoItem) if err != nil { fmt.Println("json.Unmarshal movieJson err:", err) err = tmpErr return } if len(v.Highlight["EdbCode"]) > 0 { edbInfoItem.EdbCode = v.Highlight["EdbCode"][0] } if len(v.Highlight["EdbName"]) > 0 { edbInfoItem.EdbCode = v.Highlight["EdbName"][0] } list = append(list, edbInfoItem) searchMap[v.Id] = v.Id } } } return } // EsDeleteEdbInfoData 删除es中的指标数据 func EsDeleteEdbInfoData(indexName, docId string) (err error) { defer func() { if err != nil { fmt.Println("EsDeleteEdbInfoData Err:", err.Error()) } }() client := utils.EsClient resp, err := client.Delete().Index(indexName).Id(docId).Do(context.Background()) fmt.Println(resp) if err != nil { return } if resp.Status == 0 { fmt.Println("删除成功") } else { fmt.Println("AddData", resp.Status, resp.Result) } return } // EsAddOrEditReport 新增编辑es报告 //func EsAddOrEditReport(indexName, docId string, item *models.ElasticReportDetail) (err error) { // defer func() { // if err != nil { // fmt.Println("EsAddOrEditReport Err:", err.Error()) // } // }() // client, err := NewClient() // if err != nil { // return // } // // docId为报告ID+章节ID // searchById, err := client.Get().Index(indexName).Id(docId).Do(context.Background()) // if err != nil && !strings.Contains(err.Error(), "404") { // fmt.Println("Get Err" + err.Error()) // return // } // if searchById != nil && searchById.Found { // resp, err := client.Update().Index(indexName).Id(docId).Doc(map[string]interface{}{ // "ReportId": item.ReportId, // "ReportChapterId": item.ReportChapterId, // "Title": item.Title, // "Abstract": item.Abstract, // "BodyContent": item.BodyContent, // "PublishTime": item.PublishTime, // "PublishState": item.PublishState, // "Author": item.Author, // "ClassifyIdFirst": item.ClassifyIdFirst, // "ClassifyNameFirst": item.ClassifyNameFirst, // "ClassifyIdSecond": item.ClassifyIdSecond, // "ClassifyNameSecond": item.ClassifyNameSecond, // "Categories": item.Categories, // "StageStr": item.StageStr, // }).Do(context.Background()) // if err != nil { // return err // } // //fmt.Println(resp.Status, resp.Result) // if resp.Status == 0 { // fmt.Println("修改成功" + docId) // err = nil // } else { // fmt.Println("EditData", resp.Status, resp.Result) // } // } else { // resp, err := client.Index().Index(indexName).Id(docId).BodyJson(item).Do(context.Background()) // if err != nil { // fmt.Println("新增失败:", err.Error()) // return err // } // if resp.Status == 0 && resp.Result == "created" { // fmt.Println("新增成功" + docId) // return nil // } else { // fmt.Println("AddData", resp.Status, resp.Result) // } // } // return //} // AnalyzeResp 分词接口返回结构体 type AnalyzeResp struct { Tokens []struct { EndOffset int64 `json:"end_offset"` Position int64 `json:"position"` StartOffset int64 `json:"start_offset"` Token string `json:"token"` Type string `json:"type"` } `json:"tokens"` } // Analyze 根据输入的文字获取分词后的文字 func Analyze(content string) (contentList []string, err error) { defer func() { if err != nil { fmt.Println("Analyze Err:", err.Error()) } }() client := utils.EsClient queryMap := map[string]string{ "text": content, "analyzer": "ik_max_word", } res, err := client.PerformRequest( context.Background(), elastic.PerformRequestOptions{ Method: "GET", Path: "/_analyze", Body: queryMap, Stream: false, }, ) if res.StatusCode == 200 { var analyzeResp AnalyzeResp tmpErr := json.Unmarshal(res.Body, &analyzeResp) if tmpErr != nil { err = errors.New("返回数据转结构体失败:" + tmpErr.Error()) return } for _, v := range analyzeResp.Tokens { contentList = append(contentList, v.Token) } } else { err = errors.New("分词失败,返回code异常:" + strconv.Itoa(res.StatusCode)) } return } // SearcCygxArticleHistoryData 查询查研观向的文章阅读记录 //func SearcCygxArticleHistoryData(indexName, keyword, startDate, endDate, companyIds string, from, size int) (total int64, list []*cygx.EsUserInteraction, err error) { // list = make([]*cygx.EsUserInteraction, 0) // defer func() { // if err != nil { // fmt.Println("EsUserInteraction Err:", err.Error()) // } // }() // client, err := NewClient() // if err != nil { // return // } // // mustMap := make([]interface{}, 0) // seliceid := []int{} // //管理员权限||对应销售 // if companyIds != "" { // silceCompanyId := strings.Split(companyIds, ",") // for _, v := range silceCompanyId { // companyId, _ := strconv.Atoi(v) // seliceid = append(seliceid, companyId) // } // mustMap = append(mustMap, map[string]interface{}{ // "terms": map[string]interface{}{ // "CompanyId": seliceid, // }, // }) // } // // //时间 // if startDate != "" && endDate != "" { // mustMap = append(mustMap, map[string]interface{}{ // "range": map[string]interface{}{ // "CreateTime": map[string]interface{}{ // "gte": startDate, // "lte": endDate, // }, // }, // }) // } // shouldMap := make(map[string]interface{}, 0) // //关键字匹配 // if keyword != "" { // shouldMap = map[string]interface{}{ // "should": []interface{}{ // map[string]interface{}{ // "wildcard": map[string]interface{}{ // "RealName": "*" + keyword + "*", // }, // }, // map[string]interface{}{ // "wildcard": map[string]interface{}{ // "Email": "*" + keyword + "*", // }, // }, // map[string]interface{}{ // "wildcard": map[string]interface{}{ // "Mobile": "*" + keyword + "*", // }, // }, // map[string]interface{}{ // "wildcard": map[string]interface{}{ // "CompanyName": "*" + keyword + "*", // }, // }, // }, // } // } // mustMap = append(mustMap, map[string]interface{}{ // "bool": shouldMap, // }) // // queryMap := map[string]interface{}{ // "query": map[string]interface{}{ // "bool": map[string]interface{}{ // "must": mustMap, // }, // }, // } // //排序 // //机构阅读数量 // sortMap := make([]interface{}, 0) // //如果是一家公司就不按照这个字段排序 // if len(seliceid) > 1 { // sortMap = append(sortMap, map[string]interface{}{ // "CompanyArticleHistoryNum": map[string]interface{}{ // "order": "desc", // }, // }) // } // // //用户阅读数量 // sortMap = append(sortMap, map[string]interface{}{ // "UserArticleHistoryNum": map[string]interface{}{ // "order": "desc", // }, // }) // //时间 // sortMap = append(sortMap, map[string]interface{}{ // "CreateTime": map[string]interface{}{ // "order": "desc", // }, // }) // //根据条件数量统计 // requestTotalHits := client.Count(indexName).BodyJson(queryMap) // total, err = requestTotalHits.Do(context.Background()) // if err != nil { // return // } // // queryMap["sort"] = sortMap // queryMap["from"] = from // queryMap["size"] = size // jsonBytes, _ := json.Marshal(queryMap) // fmt.Println(string(jsonBytes)) // // request := client.Search(indexName).Source(queryMap) // sets the JSON request // searchMap := make(map[string]string) // // searchResp, err := request.Do(context.Background()) // if err != nil { // return // } // //fmt.Println(searchResp) // //fmt.Println(searchResp.Status) // if searchResp.Status != 0 { // return // } // if searchResp.Hits != nil { // for _, v := range searchResp.Hits.Hits { // if _, ok := searchMap[v.Id]; !ok { // itemJson, tmpErr := v.Source.MarshalJSON() // if tmpErr != nil { // err = tmpErr // fmt.Println("movieJson err:", err) // return // } // edbInfoItem := new(cygx.EsUserInteraction) // tmpErr = json.Unmarshal(itemJson, &edbInfoItem) // if tmpErr != nil { // fmt.Println("json.Unmarshal movieJson err:", err) // err = tmpErr // return // } // list = append(list, edbInfoItem) // searchMap[v.Id] = v.Id // } // } // } // return //} // EsAddOrEditChartInfoData 新增/修改es中的图表数据 func EsAddOrEditChartInfoData(indexName, docId string, item *data_manage.ChartInfo) (err error) { defer func() { if err != nil { fmt.Println("EsAddOrEditData Err:", err.Error()) } }() client := utils.EsClient resp, err := client.Index().Index(indexName).Id(docId).BodyJson(item).Do(context.Background()) if err != nil { fmt.Println("新增失败:", err.Error()) return err } fmt.Println(resp) if resp.Status == 0 { fmt.Println("新增成功", resp.Result) err = nil } else { fmt.Println("AddData", resp.Status, resp.Result) } return } // EsDeleteDataV2 删除es中的数据 func EsDeleteDataV2(indexName, docId string) (err error) { defer func() { if err != nil { fmt.Println("EsDeleteEdbInfoData Err:", err.Error()) } }() client := utils.EsClient resp, err := client.Delete().Index(indexName).Id(docId).Do(context.Background()) fmt.Println(resp) if err != nil { return } if resp.Status == 0 { fmt.Println("删除成功") } else { fmt.Println("AddData", resp.Status, resp.Result) } return } // SearchChartInfoData 查询es中的图表数据 func SearchChartInfoData(indexName, keywordStr string, showSysId int, sourceList []int, noPermissionChartIdList []int, from, size int) (list []*data_manage.ChartInfo, total int64, err error) { list = make([]*data_manage.ChartInfo, 0) defer func() { if err != nil { fmt.Println("EsAddOrEditData Err:", err.Error()) } }() client := utils.EsClient //queryString := elastic.NewQueryStringQuery(keywordStr) //boolQueryJson, err := json.Marshal(queryString) //if err != nil { // fmt.Println("boolQueryJson err:", err) //} else { // fmt.Println("boolQueryJson ", string(boolQueryJson)) //} highlight := elastic.NewHighlight() highlight = highlight.Fields(elastic.NewHighlighterField("ChartName")) highlight = highlight.PreTags("").PostTags("") mustMap := make([]interface{}, 0) mustNotMap := make([]interface{}, 0) //指标来源 if showSysId > 0 { mustMap = append(mustMap, map[string]interface{}{ "term": map[string]interface{}{ "SysUserId": showSysId, //"Frequency.keyword": "月度", }, }) } mustMap = append(mustMap, map[string]interface{}{ "terms": map[string]interface{}{ "Source": sourceList, }, }) //关键字匹配 shouldMap := map[string]interface{}{ "should": []interface{}{ map[string]interface{}{ "match": map[string]interface{}{ "ChartName": keywordStr, //"Frequency.keyword": "月度", }, }, // 因为关键词被分了,所以需要用下面的语句来让他 整个词 查询,从而加重整词的权重 map[string]interface{}{ "match": map[string]interface{}{ "ChartName": map[string]interface{}{ "query": keywordStr, "operator": "and", }, //"Frequency.keyword": "月度", }, }, map[string]interface{}{ "match": map[string]interface{}{ "ChartNameEn": keywordStr, //"Frequency.keyword": "月度", }, }, // 因为关键词被分了,所以需要用下面的语句来让他 整个词 查询,从而加重整词的权重 map[string]interface{}{ "match": map[string]interface{}{ "ChartNameEn": map[string]interface{}{ "query": keywordStr, "operator": "and", }, //"Frequency.keyword": "月度", }, }, }, } mustMap = append(mustMap, map[string]interface{}{ "bool": shouldMap, }) // noPermissionEdbInfoIdList 无权限指标id if len(noPermissionChartIdList) > 0 { mustNotMap = append(mustNotMap, map[string]interface{}{ "terms": map[string]interface{}{ "ChartInfoId": noPermissionChartIdList, //"Frequency.keyword": "月度", }, }) } queryMap := map[string]interface{}{ "query": map[string]interface{}{ "bool": map[string]interface{}{ "must": mustMap, "must_not": mustNotMap, //"should": shouldMap, }, }, } //根据条件数量统计 requestTotalHits := client.Count(indexName).BodyJson(queryMap) total, err = requestTotalHits.Do(context.Background()) if err != nil { return } // 分页查询 queryMap["from"] = from queryMap["size"] = size jsonBytes, _ := json.Marshal(queryMap) fmt.Println(string(jsonBytes)) request := client.Search(indexName).Highlight(highlight).Source(queryMap) // sets the JSON request //requestJson, err := json.Marshal(request) //if err != nil { // fmt.Println("requestJson err:", err) //} //fmt.Println("requestJson ", string(requestJson)) searchMap := make(map[string]string) searchResp, err := request.Do(context.Background()) if err != nil { return } fmt.Println(searchResp) fmt.Println(searchResp.Status) if searchResp.Status != 0 { return } if searchResp.Hits != nil { for _, v := range searchResp.Hits.Hits { if _, ok := searchMap[v.Id]; !ok { itemJson, tmpErr := v.Source.MarshalJSON() if tmpErr != nil { err = tmpErr fmt.Println("movieJson err:", err) return } chartInfoItem := new(data_manage.ChartInfo) tmpErr = json.Unmarshal(itemJson, &chartInfoItem) if err != nil { fmt.Println("json.Unmarshal chartInfoJson err:", err) err = tmpErr return } if len(v.Highlight["ChartName"]) > 0 { chartInfoItem.ChartName = v.Highlight["ChartName"][0] } list = append(list, chartInfoItem) searchMap[v.Id] = v.Id } } } //for _, v := range result { // fmt.Println(v) //} return } // EsAddOrEditDataInterface 新增/修改es中的数据 func EsAddOrEditDataInterface(indexName, docId string, item interface{}) (err error) { defer func() { if err != nil { fmt.Println("EsAddOrEditData Err:", err.Error()) } }() client := utils.EsClient resp, err := client.Index().Index(indexName).Id(docId).BodyJson(item).Do(context.Background()) if err != nil { fmt.Println("新增失败:", err.Error()) return err } fmt.Println(resp) if resp.Status == 0 { fmt.Println("新增成功", resp.Result) err = nil } else { fmt.Println("AddData", resp.Status, resp.Result) } return } // SearchMyChartInfoData 查询es中的我的图表数据 func SearchMyChartInfoData(indexName, keywordStr string, adminId int, noPermissionChartIdList []int, from, size int) (list []*data_manage.MyChartList, total int64, err error) { list = make([]*data_manage.MyChartList, 0) defer func() { if err != nil { fmt.Println("EsAddOrEditData Err:", err.Error()) } }() client := utils.EsClient //queryString := elastic.NewQueryStringQuery(keywordStr) //boolQueryJson, err := json.Marshal(queryString) //if err != nil { // fmt.Println("boolQueryJson err:", err) //} else { // fmt.Println("boolQueryJson ", string(boolQueryJson)) //} highlight := elastic.NewHighlight() highlight = highlight.Fields(elastic.NewHighlighterField("ChartName")) highlight = highlight.PreTags("").PostTags("") mustMap := make([]interface{}, 0) mustNotMap := make([]interface{}, 0) //指标来源 if adminId > 0 { mustMap = append(mustMap, map[string]interface{}{ "term": map[string]interface{}{ "AdminId": adminId, //"Frequency.keyword": "月度", }, }) } //关键字匹配 shouldMap := map[string]interface{}{ "should": []interface{}{ map[string]interface{}{ "match": map[string]interface{}{ "ChartName": keywordStr, //"Frequency.keyword": "月度", }, }, // 因为关键词被分了,所以需要用下面的语句来让他 整个词 查询,从而加重整词的权重 map[string]interface{}{ "match": map[string]interface{}{ "ChartName": map[string]interface{}{ "query": keywordStr, "operator": "and", }, //"Frequency.keyword": "月度", }, }, }, } mustMap = append(mustMap, map[string]interface{}{ "bool": shouldMap, }) // noPermissionEdbInfoIdList 无权限指标id if len(noPermissionChartIdList) > 0 { mustNotMap = append(mustNotMap, map[string]interface{}{ "terms": map[string]interface{}{ "ChartInfoId": noPermissionChartIdList, //"Frequency.keyword": "月度", }, }) } queryMap := map[string]interface{}{ "query": map[string]interface{}{ "bool": map[string]interface{}{ "must": mustMap, "must_not": mustNotMap, //"should": shouldMap, }, }, } //根据条件数量统计 requestTotalHits := client.Count(indexName).BodyJson(queryMap) total, err = requestTotalHits.Do(context.Background()) if err != nil { return } // 分页查询 queryMap["from"] = from queryMap["size"] = size jsonBytes, _ := json.Marshal(queryMap) fmt.Println(string(jsonBytes)) request := client.Search(indexName).Highlight(highlight).Source(queryMap) // sets the JSON request //requestJson, err := json.Marshal(request) //if err != nil { // fmt.Println("requestJson err:", err) //} //fmt.Println("requestJson ", string(requestJson)) searchMap := make(map[string]string) searchResp, err := request.Do(context.Background()) if err != nil { return } fmt.Println(searchResp) fmt.Println(searchResp.Status) if searchResp.Status != 0 { return } if searchResp.Hits != nil { for _, v := range searchResp.Hits.Hits { if _, ok := searchMap[v.Id]; !ok { itemJson, tmpErr := v.Source.MarshalJSON() if tmpErr != nil { err = tmpErr fmt.Println("movieJson err:", err) return } chartInfoItem := new(data_manage.MyChartList) tmpErr = json.Unmarshal(itemJson, &chartInfoItem) if err != nil { fmt.Println("json.Unmarshal chartInfoJson err:", err) err = tmpErr return } if len(v.Highlight["ChartName"]) > 0 { chartInfoItem.ChartName = v.Highlight["ChartName"][0] } list = append(list, chartInfoItem) searchMap[v.Id] = v.Id } } } //for _, v := range result { // fmt.Println(v) //} return } // SearchEdbInfoDataByAdminId 查询es中的指标数据 func SearchEdbInfoDataByAdminId(indexName, keywordStr string, from, size, filterSource, source int, edbInfoType uint8, frequency string, adminId int) (total int64, list []*data_manage.EdbInfoList, err error) { list = make([]*data_manage.EdbInfoList, 0) defer func() { if err != nil { fmt.Println("EsAddOrEditData Err:", err.Error()) } }() highlight := elastic.NewHighlight() highlight = highlight.Fields(elastic.NewHighlighterField("EdbCode"), elastic.NewHighlighterField("EdbName")) highlight = highlight.PreTags("").PostTags("") //var source map[string]interface{} //source := map[string]interface{}{ // "query": map[string]interface{}{ // "match_all": map[string]interface{}{}, // }, //} mustMap := make([]interface{}, 0) mustNotMap := make([]interface{}, 0) //source := map[string]interface{}{ // "query": map[string]interface{}{ // "bool": map[string]interface{}{ // "must": map[string]interface{}{ // "query_string": map[string]interface{}{ // "query": keywordStr, // "fields": []string{"EdbCode", "EdbName"}, // }, // }, // }, // }, //} switch filterSource { case 2: //source = map[string]interface{}{ // "query": map[string]interface{}{ // "bool": map[string]interface{}{ // "must": map[string]interface{}{ // "query_string": map[string]interface{}{ // "query": keywordStr, // }, // }, // "filter": []interface{}{ // map[string]interface{}{ // "term": map[string]interface{}{ // "Frequency.keyword": "月度", // }, // }}, // }, // }, //} mustMap = []interface{}{ map[string]interface{}{ "term": map[string]interface{}{ "Frequency.keyword": "月度", //"Frequency.keyword": "月度", }, }, } case 3: //source = map[string]interface{}{ // "query": map[string]interface{}{ // "bool": map[string]interface{}{ // "must": map[string]interface{}{ // "query_string": map[string]interface{}{ // "query": keywordStr, // }, // }, // "must_not": []interface{}{ // map[string]interface{}{ // "match": map[string]interface{}{ // "Frequency.keyword": "日度", // }, // }}, // }, // }, //} ////注释掉,所有频度都可以变频 2022-08-31 14:31:28 //mustNotMap = []interface{}{ // map[string]interface{}{ // "match": map[string]interface{}{ // "Frequency.keyword": "日度", // //"Frequency.keyword": "月度", // }, // }, //} case 4: //source = map[string]interface{}{ // "query": map[string]interface{}{ // "bool": map[string]interface{}{ // "must": map[string]interface{}{ // "query_string": map[string]interface{}{ // "query": keywordStr, // }, // }, // "filter": []interface{}{ // map[string]interface{}{ // "term": map[string]interface{}{ // "EdbType": 1, // }, // }}, // }, // }, //} mustMap = []interface{}{ map[string]interface{}{ "term": map[string]interface{}{ "EdbType": 1, }, }, } case 5: mustMap = []interface{}{ map[string]interface{}{ "term": map[string]interface{}{ "Source": 6, }, }, } } //指标来源 if source > 0 { mustMap = append(mustMap, map[string]interface{}{ "term": map[string]interface{}{ "Source": source, //"Frequency.keyword": "月度", }, }) } if frequency != "" { mustMap = append(mustMap, map[string]interface{}{ "term": map[string]interface{}{ "Frequency.keyword": frequency, //"Frequency.keyword": "月度", }, }) } // 指标类型:普通指标、预算指标 mustMap = append(mustMap, map[string]interface{}{ "term": map[string]interface{}{ "EdbInfoType": edbInfoType, }, }) //普通指标 //mustMap = append(mustMap, map[string]interface{}{ // "term": map[string]interface{}{ // "EdbInfoType": 0, // //"Frequency.keyword": "月度", // }, //}) //关键字匹配 shouldMap := map[string]interface{}{ "should": []interface{}{ map[string]interface{}{ "match": map[string]interface{}{ "EdbCode": keywordStr, //"Frequency.keyword": "月度", }, }, map[string]interface{}{ "match": map[string]interface{}{ "EdbName": keywordStr, //"Frequency.keyword": "月度", }, }, }, } mustMap = append(mustMap, map[string]interface{}{ "bool": shouldMap, }) //创建人 if adminId > 0 { mustMap = append(mustMap, map[string]interface{}{ "term": map[string]interface{}{ "SysUserId": adminId, }, }) } return searchEdbInfoData(indexName, mustMap, mustNotMap, shouldMap, from, size) }