package elastic
import (
"context"
"encoding/json"
"errors"
"fmt"
"github.com/PuerkitoBio/goquery"
"github.com/olivere/elastic/v7"
"hongze/hz_crm_api/models/cygx"
"hongze/hz_crm_api/models/data_manage"
"hongze/hz_crm_api/utils"
"html"
"strconv"
"strings"
)
func NewClient() (client *elastic.Client, err error) {
client, err = elastic.NewClient(
elastic.SetURL(ES_URL),
elastic.SetBasicAuth(ES_USERNAME, ES_PASSWORD),
elastic.SetSniff(false))
return
}
// indexName:索引名称
// mappingJson:表结构
func EsCreateIndex(indexName, mappingJson string) (err error) {
client, err := NewClient()
if err != nil {
return
}
//定义表结构
exists, err := client.IndexExists(indexName).Do(context.Background()) //<5>
if err != nil {
return
}
if !exists {
resp, err := client.CreateIndex(indexName).BodyJson(mappingJson).Do(context.Background())
//BodyJson(bodyJson).Do(context.Background())
if err != nil {
fmt.Println("CreateIndex Err:" + err.Error())
return err
}
fmt.Println(resp.Index, resp.ShardsAcknowledged, resp.Acknowledged)
} else {
fmt.Println(indexName + " 已存在")
}
return
}
// 新增和修改数据
func EsAddOrEditData(item *cygx.CygxArticle) (err error) {
defer func() {
if err != nil {
fmt.Println("EsAddOrEditData Err:", err.Error())
}
}()
indexName := utils.IndexName
content := html.UnescapeString(item.Body)
doc, errDoc := goquery.NewDocumentFromReader(strings.NewReader(content))
if errDoc != nil {
return
}
doc.Find("a").Each(func(i int, a *goquery.Selection) {
a.Remove()
})
bodyText := doc.Text()
client, err := NewClient()
if err != nil {
fmt.Println(err, "err1")
return
}
docId := strconv.Itoa(item.ArticleId)
searchById, err := client.Get().Index(indexName).Id(docId).Do(context.Background())
if err != nil && !strings.Contains(err.Error(), "404") {
fmt.Println("Get Err" + err.Error())
return
}
if err != nil && strings.Contains(err.Error(), "404") {
err = nil
}
if searchById != nil && searchById.Found {
resp, err := client.Update().Index(indexName).Id(docId).Doc(map[string]interface{}{
"BodyText": bodyText,
"Title": item.Title,
"PublishDate": item.PublishDate.Format(utils.FormatDateTime),
"CategoryId": item.CategoryId,
"ExpertBackground": item.ExpertBackground,
}).Do(context.Background())
if err != nil {
fmt.Println(err, "err")
return err
}
if resp.Status == 0 {
fmt.Println("修改成功")
} else {
fmt.Println("EditData", resp.Status, resp.Result)
}
} else {
itemEs := new(cygx.ElasticTestArticleDetail)
itemEs.ArticleId = item.ArticleId
itemEs.Title = item.Title
itemEs.PublishDate = item.PublishDate.Format(utils.FormatDateTime)
itemEs.BodyText = bodyText
itemEs.CategoryId = strconv.Itoa(item.CategoryId)
itemEs.ExpertBackground = item.ExpertBackground
resp, err := client.Index().Index(indexName).Id(docId).BodyJson(itemEs).Do(context.Background())
if err != nil {
fmt.Println("新增失败:", err.Error())
return err
}
if resp.Status == 0 && resp.Result == "created" {
fmt.Println("新增成功")
err = nil
} else {
fmt.Println("AddData", resp.Status, resp.Result)
}
}
return
}
// 删除数据
func EsDeleteData(indexName, docId string) (err error) {
client, err := NewClient()
if err != nil {
return
}
resp, err := client.Delete().Index(indexName).Id(docId).Do(context.Background())
fmt.Println(resp)
if err != nil {
return
}
if resp.Status == 0 {
fmt.Println("删除成功")
} else {
fmt.Println("AddData", resp.Status, resp.Result)
}
return
}
func MappingModify(indexName, mappingJson string) {
client, err := NewClient()
if err != nil {
return
}
result, err := client.PutMapping().Index(indexName).BodyString(mappingJson).Do(context.Background())
fmt.Println(err)
fmt.Println(result)
return
}
// EsAddOrEditEdbInfoData 新增/修改es中的指标数据
func EsAddOrEditEdbInfoData(indexName, docId string, item *data_manage.EdbInfoList) (err error) {
defer func() {
if err != nil {
fmt.Println("EsAddOrEditData Err:", err.Error())
}
}()
client, err := NewClient()
if err != nil {
return
}
resp, err := client.Index().Index(indexName).Id(docId).BodyJson(item).Do(context.Background())
if err != nil {
fmt.Println("新增失败:", err.Error())
return err
}
fmt.Println(resp)
if resp.Status == 0 {
fmt.Println("新增成功", resp.Result)
err = nil
} else {
fmt.Println("AddData", resp.Status, resp.Result)
}
return
}
// SearchEdbInfoData 查询es中的指标数据
func SearchEdbInfoData(indexName, keywordStr string, from, size, filterSource, source int, edbInfoType int8, frequency string, noPermissionEdbInfoIdList []int) (total int64, list []*data_manage.EdbInfoList, err error) {
list = make([]*data_manage.EdbInfoList, 0)
defer func() {
if err != nil {
fmt.Println("EsAddOrEditData Err:", err.Error())
}
}()
highlight := elastic.NewHighlight()
highlight = highlight.Fields(elastic.NewHighlighterField("EdbCode"), elastic.NewHighlighterField("EdbName"))
highlight = highlight.PreTags("").PostTags("")
//var source map[string]interface{}
//source := map[string]interface{}{
// "query": map[string]interface{}{
// "match_all": map[string]interface{}{},
// },
//}
mustMap := make([]interface{}, 0)
mustNotMap := make([]interface{}, 0)
//source := map[string]interface{}{
// "query": map[string]interface{}{
// "bool": map[string]interface{}{
// "must": map[string]interface{}{
// "query_string": map[string]interface{}{
// "query": keywordStr,
// "fields": []string{"EdbCode", "EdbName"},
// },
// },
// },
// },
//}
switch filterSource {
case 2:
//source = map[string]interface{}{
// "query": map[string]interface{}{
// "bool": map[string]interface{}{
// "must": map[string]interface{}{
// "query_string": map[string]interface{}{
// "query": keywordStr,
// },
// },
// "filter": []interface{}{
// map[string]interface{}{
// "term": map[string]interface{}{
// "Frequency.keyword": "月度",
// },
// }},
// },
// },
//}
mustMap = []interface{}{
map[string]interface{}{
"term": map[string]interface{}{
"Frequency.keyword": "月度",
//"Frequency.keyword": "月度",
},
},
}
case 3:
//source = map[string]interface{}{
// "query": map[string]interface{}{
// "bool": map[string]interface{}{
// "must": map[string]interface{}{
// "query_string": map[string]interface{}{
// "query": keywordStr,
// },
// },
// "must_not": []interface{}{
// map[string]interface{}{
// "match": map[string]interface{}{
// "Frequency.keyword": "日度",
// },
// }},
// },
// },
//}
////注释掉,所有频度都可以变频 2022-08-31 14:31:28
//mustNotMap = []interface{}{
// map[string]interface{}{
// "match": map[string]interface{}{
// "Frequency.keyword": "日度",
// //"Frequency.keyword": "月度",
// },
// },
//}
case 4:
//source = map[string]interface{}{
// "query": map[string]interface{}{
// "bool": map[string]interface{}{
// "must": map[string]interface{}{
// "query_string": map[string]interface{}{
// "query": keywordStr,
// },
// },
// "filter": []interface{}{
// map[string]interface{}{
// "term": map[string]interface{}{
// "EdbType": 1,
// },
// }},
// },
// },
//}
mustMap = []interface{}{
map[string]interface{}{
"term": map[string]interface{}{
"EdbType": 1,
},
},
}
case 5:
mustMap = []interface{}{
map[string]interface{}{
"term": map[string]interface{}{
"Source": 6,
},
},
}
case 6:
mustNotMap = []interface{}{
map[string]interface{}{
"match": map[string]interface{}{
"Frequency.keyword": "年度",
},
},
}
}
//指标来源
if source > 0 {
mustMap = append(mustMap, map[string]interface{}{
"term": map[string]interface{}{
"Source": source,
//"Frequency.keyword": "月度",
},
})
}
if frequency != "" {
mustMap = append(mustMap, map[string]interface{}{
"term": map[string]interface{}{
"Frequency.keyword": frequency,
//"Frequency.keyword": "月度",
},
})
}
// noPermissionEdbInfoIdList 无权限指标id
if len(noPermissionEdbInfoIdList) > 0 {
mustNotMap = append(mustNotMap, map[string]interface{}{
"terms": map[string]interface{}{
"EdbInfoId": noPermissionEdbInfoIdList,
//"Frequency.keyword": "月度",
},
})
}
// 指标类型:普通指标、预测指标(小于0 代表不区分指标是普通还是预测)
if edbInfoType >= 0 {
mustMap = append(mustMap, map[string]interface{}{
"term": map[string]interface{}{
"EdbInfoType": edbInfoType,
},
})
}
//普通指标
//mustMap = append(mustMap, map[string]interface{}{
// "term": map[string]interface{}{
// "EdbInfoType": 0,
// //"Frequency.keyword": "月度",
// },
//})
//关键字匹配
shouldMap := map[string]interface{}{
"should": []interface{}{
map[string]interface{}{
"match": map[string]interface{}{
"EdbCode": keywordStr,
//"Frequency.keyword": "月度",
},
},
map[string]interface{}{
"match": map[string]interface{}{
"EdbName": keywordStr,
//"Frequency.keyword": "月度",
},
},
map[string]interface{}{
"match": map[string]interface{}{
"EdbNameEn": keywordStr,
//"Frequency.keyword": "月度",
},
},
},
}
mustMap = append(mustMap, map[string]interface{}{
"bool": shouldMap,
})
return searchEdbInfoData(indexName, mustMap, mustNotMap, shouldMap, from, size)
}
func SearchEdbInfoDataBak(indexName, keywordStr string, from, size, filterSource, source int, frequency string) (total int64, list []*data_manage.EdbInfoList, err error) {
list = make([]*data_manage.EdbInfoList, 0)
defer func() {
if err != nil {
fmt.Println("EsAddOrEditData Err:", err.Error())
}
}()
client, err := NewClient()
if err != nil {
return
}
//queryString := elastic.NewQueryStringQuery(keywordStr)
//boolQueryJson, err := json.Marshal(queryString)
//if err != nil {
// fmt.Println("boolQueryJson err:", err)
//} else {
// fmt.Println("boolQueryJson ", string(boolQueryJson))
//}
highlight := elastic.NewHighlight()
highlight = highlight.Fields(elastic.NewHighlighterField("EdbCode"), elastic.NewHighlighterField("EdbName"))
highlight = highlight.PreTags("").PostTags("")
//query := elastic.RawStringQuery(`{"match_all":{}}`)
//var source map[string]interface{}
//source := map[string]interface{}{
// "query": map[string]interface{}{
// "match_all": map[string]interface{}{},
// },
//}
mustMap := make([]interface{}, 0)
mustNotMap := make([]interface{}, 0)
//source := map[string]interface{}{
// "query": map[string]interface{}{
// "bool": map[string]interface{}{
// "must": map[string]interface{}{
// "query_string": map[string]interface{}{
// "query": keywordStr,
// "fields": []string{"EdbCode", "EdbName"},
// },
// },
// },
// },
//}
switch filterSource {
case 2:
//source = map[string]interface{}{
// "query": map[string]interface{}{
// "bool": map[string]interface{}{
// "must": map[string]interface{}{
// "query_string": map[string]interface{}{
// "query": keywordStr,
// },
// },
// "filter": []interface{}{
// map[string]interface{}{
// "term": map[string]interface{}{
// "Frequency.keyword": "月度",
// },
// }},
// },
// },
//}
mustMap = []interface{}{
map[string]interface{}{
"term": map[string]interface{}{
"Frequency.keyword": "月度",
//"Frequency.keyword": "月度",
},
},
}
case 3:
//source = map[string]interface{}{
// "query": map[string]interface{}{
// "bool": map[string]interface{}{
// "must": map[string]interface{}{
// "query_string": map[string]interface{}{
// "query": keywordStr,
// },
// },
// "must_not": []interface{}{
// map[string]interface{}{
// "match": map[string]interface{}{
// "Frequency.keyword": "日度",
// },
// }},
// },
// },
//}
////注释掉,所有频度都可以变频 2022-08-31 14:31:28
//mustNotMap = []interface{}{
// map[string]interface{}{
// "match": map[string]interface{}{
// "Frequency.keyword": "日度",
// //"Frequency.keyword": "月度",
// },
// },
//}
case 4:
//source = map[string]interface{}{
// "query": map[string]interface{}{
// "bool": map[string]interface{}{
// "must": map[string]interface{}{
// "query_string": map[string]interface{}{
// "query": keywordStr,
// },
// },
// "filter": []interface{}{
// map[string]interface{}{
// "term": map[string]interface{}{
// "EdbType": 1,
// },
// }},
// },
// },
//}
mustMap = []interface{}{
map[string]interface{}{
"term": map[string]interface{}{
"EdbType": 1,
},
},
}
case 5:
mustMap = []interface{}{
map[string]interface{}{
"term": map[string]interface{}{
"Source": 6,
},
},
}
case 6:
mustNotMap = []interface{}{
map[string]interface{}{
"match": map[string]interface{}{
"Frequency.keyword": "年度",
},
},
}
}
//指标来源
if source > 0 {
mustMap = append(mustMap, map[string]interface{}{
"term": map[string]interface{}{
"Source": source,
//"Frequency.keyword": "月度",
},
})
}
if frequency != "" {
mustMap = append(mustMap, map[string]interface{}{
"term": map[string]interface{}{
"Frequency.keyword": frequency,
//"Frequency.keyword": "月度",
},
})
}
//普通指标
//mustMap = append(mustMap, map[string]interface{}{
// "term": map[string]interface{}{
// "EdbInfoType": 0,
// //"Frequency.keyword": "月度",
// },
//})
//关键字匹配
shouldMap := map[string]interface{}{
"should": []interface{}{
map[string]interface{}{
"match": map[string]interface{}{
"EdbCode": keywordStr,
//"Frequency.keyword": "月度",
},
},
map[string]interface{}{
"match": map[string]interface{}{
"EdbName": keywordStr,
//"Frequency.keyword": "月度",
},
},
// 因为关键词被分了,所以需要用下面的语句来让他 整个词 查询,从而加重整词的权重
map[string]interface{}{
"match": map[string]interface{}{
"EdbCode": map[string]interface{}{
"query": keywordStr,
"operator": "and",
},
//"Frequency.keyword": "月度",
},
},
// 因为关键词被分了,所以需要用下面的语句来让他 整个词 查询,从而加重整词的权重
map[string]interface{}{
"match": map[string]interface{}{
"EdbName": map[string]interface{}{
"query": keywordStr,
"operator": "and",
},
//"Frequency.keyword": "月度",
},
},
},
}
mustMap = append(mustMap, map[string]interface{}{
"bool": shouldMap,
})
queryMap := map[string]interface{}{
"query": map[string]interface{}{
"bool": map[string]interface{}{
"must": mustMap,
"must_not": mustNotMap,
//"should": shouldMap,
},
},
}
//根据条件数量统计
requestTotalHits := client.Count(indexName).BodyJson(queryMap)
total, err = requestTotalHits.Do(context.Background())
if err != nil {
return
}
queryMap["from"] = from
queryMap["size"] = size
jsonBytes, _ := json.Marshal(queryMap)
fmt.Println(string(jsonBytes))
//queryStr := fmt.Sprintf(`{"query":{"bool":{"must":{"query_string":{"query":"%s","fields":["EdbCode","EdbName"]}}}}}`, keywordStr)
//switch filterSource {
//case 2:
// queryStr = fmt.Sprintf(`{"query":{"bool":{"must":{"query_string":{"query":"%s","fields":["EdbCode","EdbName"]}},"filter":{"term":{"Frequency.keyword":"%s"}}}}}`, keywordStr, "月度")
//case 3:
// queryStr = fmt.Sprintf(`{"query":{"bool":{"must":{"query_string":{"query":"%s","fields":["EdbCode","EdbName"]}},"must_not":[{"match":{"Frequency.keyword":"%s"}}]}}}`, keywordStr, "日度")
//case 4:
// queryStr = fmt.Sprintf(`{"query":{"bool":{"must":{"query_string":{"query":"%s","fields":["EdbCode","EdbName"]}},"must_not":[{"match":{"EdbType":1}}]}}}`, keywordStr)
//}
//queryString := elastic.RawStringQuery(queryStr)
//fmt.Println("queryString:", queryString)
//queryString := elastic.NewMatchQuery("EdbCode", keywordStr)
//request := client.Search(indexName).Highlight(highlight).From(from).Size(size).Query(queryString)
request := client.Search(indexName).Highlight(highlight).Source(queryMap) // sets the JSON request
//requestJson, err := json.Marshal(request)
//if err != nil {
// fmt.Println("requestJson err:", err)
//}
//fmt.Println("requestJson ", string(requestJson))
searchMap := make(map[string]string)
searchResp, err := request.Do(context.Background())
if err != nil {
return
}
fmt.Println(searchResp)
fmt.Println(searchResp.Status)
if searchResp.Status != 0 {
return
}
if searchResp.Hits != nil {
for _, v := range searchResp.Hits.Hits {
if _, ok := searchMap[v.Id]; !ok {
itemJson, tmpErr := v.Source.MarshalJSON()
if tmpErr != nil {
err = tmpErr
fmt.Println("movieJson err:", err)
return
}
edbInfoItem := new(data_manage.EdbInfoList)
tmpErr = json.Unmarshal(itemJson, &edbInfoItem)
if err != nil {
fmt.Println("json.Unmarshal movieJson err:", err)
err = tmpErr
return
}
if len(v.Highlight["EdbCode"]) > 0 {
edbInfoItem.EdbCode = v.Highlight["EdbCode"][0]
}
if len(v.Highlight["EdbName"]) > 0 {
edbInfoItem.EdbCode = v.Highlight["EdbName"][0]
}
list = append(list, edbInfoItem)
searchMap[v.Id] = v.Id
}
}
}
//for _, v := range result {
// fmt.Println(v)
//}
return
}
// SearchAddPredictEdbInfoData 查询允许添加预测指标的数据
func SearchAddPredictEdbInfoData(indexName, keywordStr string, noPermissionEdbInfoIdList []int, from, size int) (total int64, list []*data_manage.EdbInfoList, err error) {
list = make([]*data_manage.EdbInfoList, 0)
defer func() {
if err != nil {
fmt.Println("EsAddOrEditData Err:", err.Error())
}
}()
highlight := elastic.NewHighlight()
highlight = highlight.Fields(elastic.NewHighlighterField("EdbCode"), elastic.NewHighlighterField("EdbName"))
highlight = highlight.PreTags("").PostTags("")
mustMap := make([]interface{}, 0)
mustNotMap := make([]interface{}, 0)
mustNotMap = []interface{}{
//map[string]interface{}{
// "terms": map[string]interface{}{
// "Frequency.keyword": []string{"日度", "周度", "月度"},
// },
// //"match": map[string]interface{}{
// // "Frequency": []string{"日度", "周度", "月度"},
// // //"Frequency.keyword": []string{"日度", "周度", "月度"},
// //},
//},
}
// 指标类型:普通指标、预算指标
mustMap = append(mustMap, map[string]interface{}{
"term": map[string]interface{}{
"EdbInfoType": 0,
},
})
mustMap = append(mustMap, map[string]interface{}{
"terms": map[string]interface{}{
"Frequency.keyword": []string{"日度", "周度", "月度"},
},
})
//关键字匹配
shouldMap := map[string]interface{}{
"should": []interface{}{
map[string]interface{}{
"match": map[string]interface{}{
"EdbCode": keywordStr,
//"Frequency.keyword": "月度",
},
},
map[string]interface{}{
"match": map[string]interface{}{
"EdbName": keywordStr,
//"Frequency.keyword": "月度",
},
},
},
}
// noPermissionEdbInfoIdList 无权限指标id
if len(noPermissionEdbInfoIdList) > 0 {
mustNotMap = append(mustNotMap, map[string]interface{}{
"terms": map[string]interface{}{
"EdbInfoId": noPermissionEdbInfoIdList,
//"Frequency.keyword": "月度",
},
})
}
return searchEdbInfoData(indexName, mustMap, mustNotMap, shouldMap, from, size)
}
// searchEdbInfoData 查询es中的指标数据
func searchEdbInfoData(indexName string, mustMap, mustNotMap []interface{}, shouldMap map[string]interface{}, from, size int) (total int64, list []*data_manage.EdbInfoList, err error) {
list = make([]*data_manage.EdbInfoList, 0)
defer func() {
if err != nil {
fmt.Println("EsAddOrEditData Err:", err.Error())
}
}()
client, err := NewClient()
if err != nil {
return
}
//queryString := elastic.NewQueryStringQuery(keywordStr)
//boolQueryJson, err := json.Marshal(queryString)
//if err != nil {
// fmt.Println("boolQueryJson err:", err)
//} else {
// fmt.Println("boolQueryJson ", string(boolQueryJson))
//}
highlight := elastic.NewHighlight()
highlight = highlight.Fields(elastic.NewHighlighterField("EdbCode"), elastic.NewHighlighterField("EdbName"))
highlight = highlight.PreTags("").PostTags("")
//query := elastic.RawStringQuery(`{"match_all":{}}`)
//关键字匹配
mustMap = append(mustMap, map[string]interface{}{
"bool": shouldMap,
})
queryMap := map[string]interface{}{
"query": map[string]interface{}{
"bool": map[string]interface{}{
"must": mustMap,
"must_not": mustNotMap,
//"should": shouldMap,
},
},
}
//根据条件数量统计
requestTotalHits := client.Count(indexName).BodyJson(queryMap)
total, err = requestTotalHits.Do(context.Background())
if err != nil {
return
}
queryMap["from"] = from
queryMap["size"] = size
jsonBytes, _ := json.Marshal(queryMap)
fmt.Println(string(jsonBytes))
//queryStr := fmt.Sprintf(`{"query":{"bool":{"must":{"query_string":{"query":"%s","fields":["EdbCode","EdbName"]}}}}}`, keywordStr)
//switch filterSource {
//case 2:
// queryStr = fmt.Sprintf(`{"query":{"bool":{"must":{"query_string":{"query":"%s","fields":["EdbCode","EdbName"]}},"filter":{"term":{"Frequency.keyword":"%s"}}}}}`, keywordStr, "月度")
//case 3:
// queryStr = fmt.Sprintf(`{"query":{"bool":{"must":{"query_string":{"query":"%s","fields":["EdbCode","EdbName"]}},"must_not":[{"match":{"Frequency.keyword":"%s"}}]}}}`, keywordStr, "日度")
//case 4:
// queryStr = fmt.Sprintf(`{"query":{"bool":{"must":{"query_string":{"query":"%s","fields":["EdbCode","EdbName"]}},"must_not":[{"match":{"EdbType":1}}]}}}`, keywordStr)
//}
//queryString := elastic.RawStringQuery(queryStr)
//fmt.Println("queryString:", queryString)
//queryString := elastic.NewMatchQuery("EdbCode", keywordStr)
//request := client.Search(indexName).Highlight(highlight).From(from).Size(size).Query(queryString)
request := client.Search(indexName).Highlight(highlight).Source(queryMap) // sets the JSON request
//requestJson, err := json.Marshal(request)
//if err != nil {
// fmt.Println("requestJson err:", err)
//}
//fmt.Println("requestJson ", string(requestJson))
searchMap := make(map[string]string)
searchResp, err := request.Do(context.Background())
if err != nil {
return
}
fmt.Println(searchResp)
fmt.Println(searchResp.Status)
if searchResp.Status != 0 {
return
}
//total = searchResp.TotalHits()
if searchResp.Hits != nil {
for _, v := range searchResp.Hits.Hits {
if _, ok := searchMap[v.Id]; !ok {
itemJson, tmpErr := v.Source.MarshalJSON()
if tmpErr != nil {
err = tmpErr
fmt.Println("movieJson err:", err)
return
}
edbInfoItem := new(data_manage.EdbInfoList)
tmpErr = json.Unmarshal(itemJson, &edbInfoItem)
if err != nil {
fmt.Println("json.Unmarshal movieJson err:", err)
err = tmpErr
return
}
if len(v.Highlight["EdbCode"]) > 0 {
edbInfoItem.EdbCode = v.Highlight["EdbCode"][0]
}
if len(v.Highlight["EdbName"]) > 0 {
edbInfoItem.EdbCode = v.Highlight["EdbName"][0]
}
list = append(list, edbInfoItem)
searchMap[v.Id] = v.Id
}
}
}
return
}
// EsDeleteEdbInfoData 删除es中的指标数据
func EsDeleteEdbInfoData(indexName, docId string) (err error) {
defer func() {
if err != nil {
fmt.Println("EsDeleteEdbInfoData Err:", err.Error())
}
}()
client, err := NewClient()
if err != nil {
return
}
resp, err := client.Delete().Index(indexName).Id(docId).Do(context.Background())
fmt.Println(resp)
if err != nil {
return
}
if resp.Status == 0 {
fmt.Println("删除成功")
} else {
fmt.Println("AddData", resp.Status, resp.Result)
}
return
}
// EsAddOrEditReport 新增编辑es报告
//func EsAddOrEditReport(indexName, docId string, item *models.ElasticReportDetail) (err error) {
// defer func() {
// if err != nil {
// fmt.Println("EsAddOrEditReport Err:", err.Error())
// }
// }()
// client, err := NewClient()
// if err != nil {
// return
// }
// // docId为报告ID+章节ID
// searchById, err := client.Get().Index(indexName).Id(docId).Do(context.Background())
// if err != nil && !strings.Contains(err.Error(), "404") {
// fmt.Println("Get Err" + err.Error())
// return
// }
// if searchById != nil && searchById.Found {
// resp, err := client.Update().Index(indexName).Id(docId).Doc(map[string]interface{}{
// "ReportId": item.ReportId,
// "ReportChapterId": item.ReportChapterId,
// "Title": item.Title,
// "Abstract": item.Abstract,
// "BodyContent": item.BodyContent,
// "PublishTime": item.PublishTime,
// "PublishState": item.PublishState,
// "Author": item.Author,
// "ClassifyIdFirst": item.ClassifyIdFirst,
// "ClassifyNameFirst": item.ClassifyNameFirst,
// "ClassifyIdSecond": item.ClassifyIdSecond,
// "ClassifyNameSecond": item.ClassifyNameSecond,
// "Categories": item.Categories,
// "StageStr": item.StageStr,
// }).Do(context.Background())
// if err != nil {
// return err
// }
// //fmt.Println(resp.Status, resp.Result)
// if resp.Status == 0 {
// fmt.Println("修改成功" + docId)
// err = nil
// } else {
// fmt.Println("EditData", resp.Status, resp.Result)
// }
// } else {
// resp, err := client.Index().Index(indexName).Id(docId).BodyJson(item).Do(context.Background())
// if err != nil {
// fmt.Println("新增失败:", err.Error())
// return err
// }
// if resp.Status == 0 && resp.Result == "created" {
// fmt.Println("新增成功" + docId)
// return nil
// } else {
// fmt.Println("AddData", resp.Status, resp.Result)
// }
// }
// return
//}
// AnalyzeResp 分词接口返回结构体
type AnalyzeResp struct {
Tokens []struct {
EndOffset int64 `json:"end_offset"`
Position int64 `json:"position"`
StartOffset int64 `json:"start_offset"`
Token string `json:"token"`
Type string `json:"type"`
} `json:"tokens"`
}
// Analyze 根据输入的文字获取分词后的文字
func Analyze(content string) (contentList []string, err error) {
defer func() {
if err != nil {
fmt.Println("Analyze Err:", err.Error())
}
}()
client, err := NewClient()
if err != nil {
return
}
queryMap := map[string]string{
"text": content,
"analyzer": "ik_max_word",
}
res, err := client.PerformRequest(
context.Background(),
elastic.PerformRequestOptions{
Method: "GET",
Path: "/_analyze",
Body: queryMap,
Stream: false,
},
)
if res.StatusCode == 200 {
var analyzeResp AnalyzeResp
tmpErr := json.Unmarshal(res.Body, &analyzeResp)
if tmpErr != nil {
err = errors.New("返回数据转结构体失败:" + tmpErr.Error())
return
}
for _, v := range analyzeResp.Tokens {
contentList = append(contentList, v.Token)
}
} else {
err = errors.New("分词失败,返回code异常:" + strconv.Itoa(res.StatusCode))
}
return
}
// SearcCygxArticleHistoryData 查询查研观向的文章阅读记录
func SearcCygxArticleHistoryData(indexName, keyword, startDate, endDate, companyIds string, from, size int) (total int64, list []*cygx.EsUserInteraction, err error) {
list = make([]*cygx.EsUserInteraction, 0)
defer func() {
if err != nil {
fmt.Println("EsUserInteraction Err:", err.Error())
}
}()
client, err := NewClient()
if err != nil {
return
}
mustMap := make([]interface{}, 0)
seliceid := []int{}
//管理员权限||对应销售
if companyIds != "" {
silceCompanyId := strings.Split(companyIds, ",")
for _, v := range silceCompanyId {
companyId, _ := strconv.Atoi(v)
seliceid = append(seliceid, companyId)
}
mustMap = append(mustMap, map[string]interface{}{
"terms": map[string]interface{}{
"CompanyId": seliceid,
},
})
}
//时间
if startDate != "" && endDate != "" {
mustMap = append(mustMap, map[string]interface{}{
"range": map[string]interface{}{
"CreateTime": map[string]interface{}{
"gte": startDate,
"lte": endDate,
},
},
})
}
shouldMap := make(map[string]interface{}, 0)
//关键字匹配
if keyword != "" {
shouldMap = map[string]interface{}{
"should": []interface{}{
map[string]interface{}{
"wildcard": map[string]interface{}{
"RealName": "*" + keyword + "*",
},
},
map[string]interface{}{
"wildcard": map[string]interface{}{
"Email": "*" + keyword + "*",
},
},
map[string]interface{}{
"wildcard": map[string]interface{}{
"Mobile": "*" + keyword + "*",
},
},
map[string]interface{}{
"wildcard": map[string]interface{}{
"CompanyName": "*" + keyword + "*",
},
},
},
}
}
mustMap = append(mustMap, map[string]interface{}{
"bool": shouldMap,
})
queryMap := map[string]interface{}{
"query": map[string]interface{}{
"bool": map[string]interface{}{
"must": mustMap,
},
},
}
//排序
//机构阅读数量
sortMap := make([]interface{}, 0)
//如果是一家公司就不按照这个字段排序
if len(seliceid) > 1 {
sortMap = append(sortMap, map[string]interface{}{
"CompanyArticleHistoryNum": map[string]interface{}{
"order": "desc",
},
})
}
//用户阅读数量
sortMap = append(sortMap, map[string]interface{}{
"UserArticleHistoryNum": map[string]interface{}{
"order": "desc",
},
})
//时间
sortMap = append(sortMap, map[string]interface{}{
"CreateTime": map[string]interface{}{
"order": "desc",
},
})
//根据条件数量统计
requestTotalHits := client.Count(indexName).BodyJson(queryMap)
total, err = requestTotalHits.Do(context.Background())
if err != nil {
return
}
queryMap["sort"] = sortMap
queryMap["from"] = from
queryMap["size"] = size
jsonBytes, _ := json.Marshal(queryMap)
fmt.Println(string(jsonBytes))
request := client.Search(indexName).Source(queryMap) // sets the JSON request
searchMap := make(map[string]string)
searchResp, err := request.Do(context.Background())
if err != nil {
return
}
//fmt.Println(searchResp)
//fmt.Println(searchResp.Status)
if searchResp.Status != 0 {
return
}
if searchResp.Hits != nil {
for _, v := range searchResp.Hits.Hits {
if _, ok := searchMap[v.Id]; !ok {
itemJson, tmpErr := v.Source.MarshalJSON()
if tmpErr != nil {
err = tmpErr
fmt.Println("movieJson err:", err)
return
}
edbInfoItem := new(cygx.EsUserInteraction)
tmpErr = json.Unmarshal(itemJson, &edbInfoItem)
if tmpErr != nil {
fmt.Println("json.Unmarshal movieJson err:", err)
err = tmpErr
return
}
list = append(list, edbInfoItem)
searchMap[v.Id] = v.Id
}
}
}
return
}
// EsAddOrEditChartInfoData 新增/修改es中的图表数据
func EsAddOrEditChartInfoData(indexName, docId string, item *data_manage.ChartInfo) (err error) {
defer func() {
if err != nil {
fmt.Println("EsAddOrEditData Err:", err.Error())
}
}()
client, err := NewClient()
if err != nil {
return
}
resp, err := client.Index().Index(indexName).Id(docId).BodyJson(item).Do(context.Background())
if err != nil {
fmt.Println("新增失败:", err.Error())
return err
}
fmt.Println(resp)
if resp.Status == 0 {
fmt.Println("新增成功", resp.Result)
err = nil
} else {
fmt.Println("AddData", resp.Status, resp.Result)
}
return
}
// EsDeleteDataV2 删除es中的数据
func EsDeleteDataV2(indexName, docId string) (err error) {
defer func() {
if err != nil {
fmt.Println("EsDeleteEdbInfoData Err:", err.Error())
}
}()
client, err := NewClient()
if err != nil {
return
}
resp, err := client.Delete().Index(indexName).Id(docId).Do(context.Background())
fmt.Println(resp)
if err != nil {
return
}
if resp.Status == 0 {
fmt.Println("删除成功")
} else {
fmt.Println("AddData", resp.Status, resp.Result)
}
return
}
// SearchChartInfoData 查询es中的图表数据
func SearchChartInfoData(indexName, keywordStr string, showSysId int, sourceList []int, noPermissionChartIdList []int, from, size int) (list []*data_manage.ChartInfo, total int64, err error) {
list = make([]*data_manage.ChartInfo, 0)
defer func() {
if err != nil {
fmt.Println("EsAddOrEditData Err:", err.Error())
}
}()
client, err := NewClient()
if err != nil {
return
}
//queryString := elastic.NewQueryStringQuery(keywordStr)
//boolQueryJson, err := json.Marshal(queryString)
//if err != nil {
// fmt.Println("boolQueryJson err:", err)
//} else {
// fmt.Println("boolQueryJson ", string(boolQueryJson))
//}
highlight := elastic.NewHighlight()
highlight = highlight.Fields(elastic.NewHighlighterField("ChartName"))
highlight = highlight.PreTags("").PostTags("")
mustMap := make([]interface{}, 0)
mustNotMap := make([]interface{}, 0)
//指标来源
if showSysId > 0 {
mustMap = append(mustMap, map[string]interface{}{
"term": map[string]interface{}{
"SysUserId": showSysId,
//"Frequency.keyword": "月度",
},
})
}
mustMap = append(mustMap, map[string]interface{}{
"terms": map[string]interface{}{
"Source": sourceList,
},
})
//关键字匹配
shouldMap := map[string]interface{}{
"should": []interface{}{
map[string]interface{}{
"match": map[string]interface{}{
"ChartName": keywordStr,
//"Frequency.keyword": "月度",
},
},
// 因为关键词被分了,所以需要用下面的语句来让他 整个词 查询,从而加重整词的权重
map[string]interface{}{
"match": map[string]interface{}{
"ChartName": map[string]interface{}{
"query": keywordStr,
"operator": "and",
},
//"Frequency.keyword": "月度",
},
},
map[string]interface{}{
"match": map[string]interface{}{
"ChartNameEn": keywordStr,
//"Frequency.keyword": "月度",
},
},
// 因为关键词被分了,所以需要用下面的语句来让他 整个词 查询,从而加重整词的权重
map[string]interface{}{
"match": map[string]interface{}{
"ChartNameEn": map[string]interface{}{
"query": keywordStr,
"operator": "and",
},
//"Frequency.keyword": "月度",
},
},
},
}
mustMap = append(mustMap, map[string]interface{}{
"bool": shouldMap,
})
// noPermissionEdbInfoIdList 无权限指标id
if len(noPermissionChartIdList) > 0 {
mustNotMap = append(mustNotMap, map[string]interface{}{
"terms": map[string]interface{}{
"ChartInfoId": noPermissionChartIdList,
//"Frequency.keyword": "月度",
},
})
}
queryMap := map[string]interface{}{
"query": map[string]interface{}{
"bool": map[string]interface{}{
"must": mustMap,
"must_not": mustNotMap,
//"should": shouldMap,
},
},
}
//根据条件数量统计
requestTotalHits := client.Count(indexName).BodyJson(queryMap)
total, err = requestTotalHits.Do(context.Background())
if err != nil {
return
}
// 分页查询
queryMap["from"] = from
queryMap["size"] = size
jsonBytes, _ := json.Marshal(queryMap)
fmt.Println(string(jsonBytes))
request := client.Search(indexName).Highlight(highlight).Source(queryMap) // sets the JSON request
//requestJson, err := json.Marshal(request)
//if err != nil {
// fmt.Println("requestJson err:", err)
//}
//fmt.Println("requestJson ", string(requestJson))
searchMap := make(map[string]string)
searchResp, err := request.Do(context.Background())
if err != nil {
return
}
fmt.Println(searchResp)
fmt.Println(searchResp.Status)
if searchResp.Status != 0 {
return
}
if searchResp.Hits != nil {
for _, v := range searchResp.Hits.Hits {
if _, ok := searchMap[v.Id]; !ok {
itemJson, tmpErr := v.Source.MarshalJSON()
if tmpErr != nil {
err = tmpErr
fmt.Println("movieJson err:", err)
return
}
chartInfoItem := new(data_manage.ChartInfo)
tmpErr = json.Unmarshal(itemJson, &chartInfoItem)
if err != nil {
fmt.Println("json.Unmarshal chartInfoJson err:", err)
err = tmpErr
return
}
if len(v.Highlight["ChartName"]) > 0 {
chartInfoItem.ChartName = v.Highlight["ChartName"][0]
}
list = append(list, chartInfoItem)
searchMap[v.Id] = v.Id
}
}
}
//for _, v := range result {
// fmt.Println(v)
//}
return
}
// EsAddOrEditDataInterface 新增/修改es中的数据
func EsAddOrEditDataInterface(indexName, docId string, item interface{}) (err error) {
defer func() {
if err != nil {
fmt.Println("EsAddOrEditData Err:", err.Error())
}
}()
client, err := NewClient()
if err != nil {
return
}
resp, err := client.Index().Index(indexName).Id(docId).BodyJson(item).Do(context.Background())
if err != nil {
fmt.Println("新增失败:", err.Error())
return err
}
fmt.Println(resp)
if resp.Status == 0 {
fmt.Println("新增成功", resp.Result)
err = nil
} else {
fmt.Println("AddData", resp.Status, resp.Result)
}
return
}
// SearchMyChartInfoData 查询es中的我的图表数据
func SearchMyChartInfoData(indexName, keywordStr string, adminId int, noPermissionChartIdList []int, from, size int) (list []*data_manage.MyChartList, total int64, err error) {
list = make([]*data_manage.MyChartList, 0)
defer func() {
if err != nil {
fmt.Println("EsAddOrEditData Err:", err.Error())
}
}()
client, err := NewClient()
if err != nil {
return
}
//queryString := elastic.NewQueryStringQuery(keywordStr)
//boolQueryJson, err := json.Marshal(queryString)
//if err != nil {
// fmt.Println("boolQueryJson err:", err)
//} else {
// fmt.Println("boolQueryJson ", string(boolQueryJson))
//}
highlight := elastic.NewHighlight()
highlight = highlight.Fields(elastic.NewHighlighterField("ChartName"))
highlight = highlight.PreTags("").PostTags("")
mustMap := make([]interface{}, 0)
mustNotMap := make([]interface{}, 0)
//指标来源
if adminId > 0 {
mustMap = append(mustMap, map[string]interface{}{
"term": map[string]interface{}{
"AdminId": adminId,
//"Frequency.keyword": "月度",
},
})
}
//关键字匹配
shouldMap := map[string]interface{}{
"should": []interface{}{
map[string]interface{}{
"match": map[string]interface{}{
"ChartName": keywordStr,
//"Frequency.keyword": "月度",
},
},
// 因为关键词被分了,所以需要用下面的语句来让他 整个词 查询,从而加重整词的权重
map[string]interface{}{
"match": map[string]interface{}{
"ChartName": map[string]interface{}{
"query": keywordStr,
"operator": "and",
},
//"Frequency.keyword": "月度",
},
},
map[string]interface{}{
"match": map[string]interface{}{
"ChartNameEn": keywordStr,
//"Frequency.keyword": "月度",
},
},
// 因为关键词被分了,所以需要用下面的语句来让他 整个词 查询,从而加重整词的权重
map[string]interface{}{
"match": map[string]interface{}{
"ChartNameEn": map[string]interface{}{
"query": keywordStr,
"operator": "and",
},
//"Frequency.keyword": "月度",
},
},
},
}
mustMap = append(mustMap, map[string]interface{}{
"bool": shouldMap,
})
// noPermissionEdbInfoIdList 无权限指标id
if len(noPermissionChartIdList) > 0 {
mustNotMap = append(mustNotMap, map[string]interface{}{
"terms": map[string]interface{}{
"ChartInfoId": noPermissionChartIdList,
//"Frequency.keyword": "月度",
},
})
}
queryMap := map[string]interface{}{
"query": map[string]interface{}{
"bool": map[string]interface{}{
"must": mustMap,
"must_not": mustNotMap,
//"should": shouldMap,
},
},
}
//根据条件数量统计
requestTotalHits := client.Count(indexName).BodyJson(queryMap)
total, err = requestTotalHits.Do(context.Background())
if err != nil {
return
}
// 分页查询
queryMap["from"] = from
queryMap["size"] = size
jsonBytes, _ := json.Marshal(queryMap)
fmt.Println(string(jsonBytes))
request := client.Search(indexName).Highlight(highlight).Source(queryMap) // sets the JSON request
//requestJson, err := json.Marshal(request)
//if err != nil {
// fmt.Println("requestJson err:", err)
//}
//fmt.Println("requestJson ", string(requestJson))
searchMap := make(map[string]string)
searchResp, err := request.Do(context.Background())
if err != nil {
return
}
fmt.Println(searchResp)
fmt.Println(searchResp.Status)
if searchResp.Status != 0 {
return
}
if searchResp.Hits != nil {
for _, v := range searchResp.Hits.Hits {
if _, ok := searchMap[v.Id]; !ok {
itemJson, tmpErr := v.Source.MarshalJSON()
if tmpErr != nil {
err = tmpErr
fmt.Println("movieJson err:", err)
return
}
chartInfoItem := new(data_manage.MyChartList)
tmpErr = json.Unmarshal(itemJson, &chartInfoItem)
if err != nil {
fmt.Println("json.Unmarshal chartInfoJson err:", err)
err = tmpErr
return
}
if len(v.Highlight["ChartName"]) > 0 {
chartInfoItem.ChartName = v.Highlight["ChartName"][0]
}
list = append(list, chartInfoItem)
searchMap[v.Id] = v.Id
}
}
}
//for _, v := range result {
// fmt.Println(v)
//}
return
}
// SearchEdbInfoDataByAdminId 查询es中的指标数据
func SearchEdbInfoDataByAdminId(indexName, keywordStr string, from, size, filterSource, source int, edbInfoType uint8, frequency string, adminId int) (total int64, list []*data_manage.EdbInfoList, err error) {
list = make([]*data_manage.EdbInfoList, 0)
defer func() {
if err != nil {
fmt.Println("EsAddOrEditData Err:", err.Error())
}
}()
highlight := elastic.NewHighlight()
highlight = highlight.Fields(elastic.NewHighlighterField("EdbCode"), elastic.NewHighlighterField("EdbName"))
highlight = highlight.PreTags("").PostTags("")
//var source map[string]interface{}
//source := map[string]interface{}{
// "query": map[string]interface{}{
// "match_all": map[string]interface{}{},
// },
//}
mustMap := make([]interface{}, 0)
mustNotMap := make([]interface{}, 0)
//source := map[string]interface{}{
// "query": map[string]interface{}{
// "bool": map[string]interface{}{
// "must": map[string]interface{}{
// "query_string": map[string]interface{}{
// "query": keywordStr,
// "fields": []string{"EdbCode", "EdbName"},
// },
// },
// },
// },
//}
switch filterSource {
case 2:
//source = map[string]interface{}{
// "query": map[string]interface{}{
// "bool": map[string]interface{}{
// "must": map[string]interface{}{
// "query_string": map[string]interface{}{
// "query": keywordStr,
// },
// },
// "filter": []interface{}{
// map[string]interface{}{
// "term": map[string]interface{}{
// "Frequency.keyword": "月度",
// },
// }},
// },
// },
//}
mustMap = []interface{}{
map[string]interface{}{
"term": map[string]interface{}{
"Frequency.keyword": "月度",
//"Frequency.keyword": "月度",
},
},
}
case 3:
//source = map[string]interface{}{
// "query": map[string]interface{}{
// "bool": map[string]interface{}{
// "must": map[string]interface{}{
// "query_string": map[string]interface{}{
// "query": keywordStr,
// },
// },
// "must_not": []interface{}{
// map[string]interface{}{
// "match": map[string]interface{}{
// "Frequency.keyword": "日度",
// },
// }},
// },
// },
//}
////注释掉,所有频度都可以变频 2022-08-31 14:31:28
//mustNotMap = []interface{}{
// map[string]interface{}{
// "match": map[string]interface{}{
// "Frequency.keyword": "日度",
// //"Frequency.keyword": "月度",
// },
// },
//}
case 4:
//source = map[string]interface{}{
// "query": map[string]interface{}{
// "bool": map[string]interface{}{
// "must": map[string]interface{}{
// "query_string": map[string]interface{}{
// "query": keywordStr,
// },
// },
// "filter": []interface{}{
// map[string]interface{}{
// "term": map[string]interface{}{
// "EdbType": 1,
// },
// }},
// },
// },
//}
mustMap = []interface{}{
map[string]interface{}{
"term": map[string]interface{}{
"EdbType": 1,
},
},
}
case 5:
mustMap = []interface{}{
map[string]interface{}{
"term": map[string]interface{}{
"Source": 6,
},
},
}
case 6:
mustNotMap = []interface{}{
map[string]interface{}{
"match": map[string]interface{}{
"Frequency.keyword": "年度",
},
},
}
}
//指标来源
if source > 0 {
mustMap = append(mustMap, map[string]interface{}{
"term": map[string]interface{}{
"Source": source,
//"Frequency.keyword": "月度",
},
})
}
if frequency != "" {
mustMap = append(mustMap, map[string]interface{}{
"term": map[string]interface{}{
"Frequency.keyword": frequency,
//"Frequency.keyword": "月度",
},
})
}
// 指标类型:普通指标、预算指标
mustMap = append(mustMap, map[string]interface{}{
"term": map[string]interface{}{
"EdbInfoType": edbInfoType,
},
})
//普通指标
//mustMap = append(mustMap, map[string]interface{}{
// "term": map[string]interface{}{
// "EdbInfoType": 0,
// //"Frequency.keyword": "月度",
// },
//})
//关键字匹配
shouldMap := map[string]interface{}{
"should": []interface{}{
map[string]interface{}{
"match": map[string]interface{}{
"EdbCode": keywordStr,
//"Frequency.keyword": "月度",
},
},
map[string]interface{}{
"match": map[string]interface{}{
"EdbName": keywordStr,
//"Frequency.keyword": "月度",
},
},
},
}
mustMap = append(mustMap, map[string]interface{}{
"bool": shouldMap,
})
//创建人
if adminId > 0 {
mustMap = append(mustMap, map[string]interface{}{
"term": map[string]interface{}{
"SysUserId": adminId,
},
})
}
return searchEdbInfoData(indexName, mustMap, mustNotMap, shouldMap, from, size)
}