eta_api/services/knowledge/es.go

package knowledge

import (
	"context"
	"encoding/json"
	"eta_gn/eta_api/models/knowledge"
	"eta_gn/eta_api/utils"
	"fmt"
	"html"
	"strconv"
	"strings"

	"github.com/PuerkitoBio/goquery"
	"github.com/olivere/elastic/v7"
)

func EsBatchAddOrEditKnowledgeResource(list []*knowledge.KnowledgeResource) (err error) {
	defer func() {
		if err != nil {
			fmt.Println("EsBatchAddOrEditData Err:", err.Error())
			utils.FileLog.Info("EsBatchAddOrEditKnowledgeResource err:", err)
		}
	}()
	indexName := utils.EsKnowledgeResourceIndexName
	client := utils.EsClient

	actions := make([]elastic.BulkableRequest, len(list))
	for i, item := range list {
		request := elastic.NewBulkIndexRequest().Index(indexName).Id(strconv.Itoa(item.KnowledgeResourceId)).Doc(item)
		actions[i] = request
	}
	response, err := client.Bulk().Add(actions...).Do(context.Background())
	if err != nil {
		jsonBytes, _ := json.Marshal(list)
		fmt.Println("add json:", string(jsonBytes))
		fmt.Println("EsBatchAddOrEditKnowledgeResource err:", err)
		return
	}
	if response.Errors {
		for i, item := range response.Items {
			for _, op := range item {
				if op.Error != nil {
					utils.FileLog.Warning("EsBatchAddOrEditKnowledgeResource Insert failed: %v", list[i])
				}
			}
		}
		fmt.Println("EsBatchAddOrEditKnowledgeResource:", response.Errors)
	} else {
		fmt.Printf("EsBatchAddOrEditKnowledgeResource:%+v\n", response)
	}
	return
}

// SearchChartInfoData 查询es中的图表数据
func SearchKnowledgeResourceByEs(resourceType int, keywordStr string, showSysIds []int, myId int, classifyIds []int, sourceList []string, tagIds []int, isShowMe, isIncludeFile, isQueryRef bool, from, size int) (list []*knowledge.KnowledgeResource, total int64, err error) {
	indexName := utils.EsKnowledgeResourceIndexName
	list = make([]*knowledge.KnowledgeResource, 0)
	defer func() {
		if err != nil {
			fmt.Println("SearchKnowledgeResource Err:", err.Error())
			utils.FileLog.Info("SearchKnowledgeResource Err:", err.Error())
		}
	}()
	client := utils.EsClient

	mustMap := make([]interface{}, 0)

	mustMap = append(mustMap, map[string]interface{}{
		"term": map[string]interface{}{
			"ResourceType": resourceType,
		},
	})

	//指标来源
	if len(showSysIds) > 0 {
		mustMap = append(mustMap, map[string]interface{}{
			"terms": map[string]interface{}{
				"AdminId": showSysIds,
			},
		})
	}
	if myId > 0 && isShowMe {
		mustMap = append(mustMap, map[string]interface{}{
			"term": map[string]interface{}{
				"AdminId": myId,
			},
		})
	}
	if len(tagIds) > 0 {
		mustMap = append(mustMap, map[string]interface{}{
			"terms": map[string]interface{}{
				"TagId": tagIds,
			},
		})
	}
	if len(sourceList) > 0 {
		mustMap = append(mustMap, map[string]interface{}{
			"terms": map[string]interface{}{
				"SourceFrom": sourceList,
			},
		})
	}
	if len(classifyIds) > 0 {
		mustMap = append(mustMap, map[string]interface{}{
			"terms": map[string]interface{}{
				"ClassifyId": classifyIds,
			},
		})
	}

	mustMap = append(mustMap, map[string]interface{}{
		"term": map[string]interface{}{
			"IsDelete": 0,
		},
	})
	if !isIncludeFile {
		mustMap = append(mustMap, map[string]interface{}{
			"term": map[string]interface{}{
				"IsFile": 0,
			},
		})
	}

	if isQueryRef && resourceType == knowledge.KnowledgeResourceTypeOpinion {
		shouldMap := map[string]interface{}{
			"should": []interface{}{
				map[string]interface{}{
					"term": map[string]interface{}{
						"State": knowledge.KnowledgeResourceStatePassed,
					},
				},
				map[string]interface{}{
					"term": map[string]interface{}{
						"AdminId": myId,
					},
				},
			},
			"minimum_should_match": 1,
		}
		mustMap = append(mustMap, map[string]interface{}{
			"bool": shouldMap,
		})
	}

	if keywordStr != "" {
		shouldMap := map[string]interface{}{
			"should": []interface{}{
				map[string]interface{}{
					"match": map[string]interface{}{
						"Title": keywordStr,
					},
				},
				// 因为关键词被分了，所以需要用下面的语句来让他 整个词 查询，从而加重整词的权重
				map[string]interface{}{
					"match": map[string]interface{}{
						"Title": map[string]interface{}{
							"query":    keywordStr,
							"operator": "and",
						},
					},
				},
			},
		}
		mustMap = append(mustMap, map[string]interface{}{
			"bool": shouldMap,
		})
	}

	queryMap := map[string]interface{}{
		"query": map[string]interface{}{
			"bool": map[string]interface{}{
				"must": mustMap,
			},
		},
	}

	//根据条件数量统计
	requestTotalHits := client.Count(indexName).BodyJson(queryMap)
	total, err = requestTotalHits.Do(context.Background())
	if err != nil {
		return
	}

	// 分页查询
	queryMap["from"] = from
	queryMap["size"] = size

	if keywordStr == "" {
		sortMap := []map[string]interface{}{
			{
				"StartTime.keyword": map[string]interface{}{
					"order": "desc", // 按照开始时间倒序排列
				},
			},
		}
		queryMap["sort"] = sortMap

	}

	request := client.Search(indexName).Source(queryMap) // sets the JSON request
	searchMap := make(map[string]string)

	searchResp, err := request.Do(context.Background())
	if err != nil {
		return
	}
	if searchResp.Status != 0 {
		jsonBytes, _ := json.Marshal(queryMap)
		utils.FileLog.Info("search json:%s,SearchKnowledgeResourceByEs Status:%d", string(jsonBytes), searchResp.Status)
		return
	}

	if searchResp.Hits != nil {
		for _, v := range searchResp.Hits.Hits {
			if _, ok := searchMap[v.Id]; !ok {
				itemJson, tmpErr := v.Source.MarshalJSON()
				if tmpErr != nil {
					err = tmpErr
					utils.FileLog.Info("search json:%s,SearchKnowledgeResourceByEs Hits Source err:%s", string(itemJson), tmpErr.Error())
					return
				}
				knowledgeItem := new(knowledge.KnowledgeResource)
				tmpErr = json.Unmarshal(itemJson, &knowledgeItem)
				if err != nil {
					utils.FileLog.Info("json.Unmarshal KnowledgeResource err:%s", err.Error())
					err = tmpErr
					return
				}
				list = append(list, knowledgeItem)
				searchMap[v.Id] = v.Id
			}
		}
	}
	return
}

func ExtractTextFromResourceContent(content string) (text string) {
	content = html.UnescapeString(content)
	doc, err := goquery.NewDocumentFromReader(strings.NewReader(content))
	if err != nil {
		return
	}
	text = doc.Text()
	text = strings.ReplaceAll(text, "\n", "")
	return
}

// EsAddOrEditKnowledgeResource 新增/修改es中的知识资源数据
func EsAddOrEditKnowledgeResource(item *knowledge.KnowledgeResource) (err error) {
	defer func() {
		if err != nil {
			fmt.Println("EsAddOrEditData Err:", err.Error())
			utils.FileLog.Info("EsAddOrEditKnowledgeResource err:", err)
		}
	}()
	indexName := utils.EsKnowledgeResourceIndexName
	client := utils.EsClient
	if item.IsFile == 0 {
		content := ExtractTextFromResourceContent(item.Content)
		contentRunes := []rune(content)
		if len(contentRunes) > 60 {
			item.Content = string(contentRunes[:60])
		} else {
			item.Content = content
		}
	}
	request := client.Index().Index(indexName).Id(strconv.Itoa(item.KnowledgeResourceId)).BodyJson(item)
	response, err := request.Do(context.Background())
	if err != nil {
		jsonBytes, _ := json.Marshal(item)
		utils.FileLog.Info("add json:%s,EsAddOrEditKnowledgeResource err:%s", string(jsonBytes), err.Error())
		return
	}
	if response.Status == 0 {
		err = nil
	} else {
		fmt.Println("EsAddOrEditKnowledgeResource:", response.Status, response.Result)
	}
	return
}