Merge remote-tracking branch 'origin/feature/deepseek_rag_1.0' into debug

controllers/llm/wechat_platform.go

@@ -10,6 +10,7 @@ import (
 	"eta/eta_api/models/rag/response"
 	"eta/eta_api/models/system"
 	"eta/eta_api/services"
+	"eta/eta_api/services/elastic"
 	"eta/eta_api/utils"
 	"fmt"
 	"github.com/rdlucklib/rdluck_tools/paging"
@@ -507,34 +508,71 @@ func (c *WechatPlatformController) ArticleList() {
 	}
 	startSize = utils.StartIndex(currentIndex, pageSize)
 
-	var condition string
-	var pars []interface{}
+	//var condition string
+	//var pars []interface{}
+	//
+	//if keyWord != "" {
+	//	condition += fmt.Sprintf(` AND (b.%s like ? or a.%s like ? ) `, rag.WechatPlatformColumns.Nickname, rag.WechatArticleColumns.Title)
+	//	pars = append(pars, `%`+keyWord+`%`, `%`+keyWord+`%`)
+	//}
+	//
+	//if wechatPlatformId > 0 {
+	//	condition += fmt.Sprintf(` AND a.%s = ?`, rag.WechatArticleColumns.WechatPlatformID)
+	//	pars = append(pars, wechatPlatformId)
+	//}
+	//
+	//condition += fmt.Sprintf(` AND b.%s = ? `, rag.WechatPlatformColumns.Enabled)
+	//pars = append(pars, 1)
+
+	var total int
+	viewList := make([]rag.WechatArticleView, 0)
 
-	if keyWord != "" {
-		condition += fmt.Sprintf(` AND (b.%s like ? or a.%s like ? ) `, rag.WechatPlatformColumns.Nickname, rag.WechatArticleColumns.Title)
-		pars = append(pars, `%`+keyWord+`%`, `%`+keyWord+`%`)
-	}
+	if keyWord == `` {
+		var condition string
+		var pars []interface{}
 
-	if wechatPlatformId > 0 {
-		condition += fmt.Sprintf(` AND a.%s = ?`, rag.WechatArticleColumns.WechatPlatformID)
-		pars = append(pars, wechatPlatformId)
-	}
+		if keyWord != "" {
+			condition += fmt.Sprintf(` AND (b.%s like ? or a.%s like ? ) `, rag.WechatPlatformColumns.Nickname, rag.WechatArticleColumns.Title)
+			pars = append(pars, `%`+keyWord+`%`, `%`+keyWord+`%`)
+		}
 
-	condition += fmt.Sprintf(` AND b.%s = ? `, rag.WechatPlatformColumns.Enabled)
-	pars = append(pars, 1)
+		if wechatPlatformId > 0 {
+			condition += fmt.Sprintf(` AND a.%s = ?`, rag.WechatArticleColumns.WechatPlatformID)
+			pars = append(pars, wechatPlatformId)
+		}
 
-	obj := new(rag.WechatArticle)
-	total, list, err := obj.GetPageListByPlatformCondition(condition, pars, startSize, pageSize)
-	if err != nil {
-		br.Msg = "获取失败"
-		br.ErrMsg = "获取失败，Err:" + err.Error()
-		return
-	}
+		condition += fmt.Sprintf(` AND b.%s = ? `, rag.WechatPlatformColumns.Enabled)
+		pars = append(pars, 1)
 
-	viewList := make([]rag.WechatArticleView, 0)
-	if list != nil && len(list) > 0 {
-		viewList = obj.ArticleAndPlatformListToViewList(list)
+		obj := new(rag.WechatArticle)
+		tmpTotal, list, err := obj.GetPageListByPlatformCondition(condition, pars, startSize, pageSize)
+		if err != nil {
+			br.Msg = "获取失败"
+			br.ErrMsg = "获取失败，Err:" + err.Error()
+			return
+		}
+		total = tmpTotal
+
+		if list != nil && len(list) > 0 {
+			viewList = obj.ArticleAndPlatformListToViewList(list)
+		}
+	} else {
+		sortMap := map[string]string{
+			"ArticleCreateTime": "desc",
+			"WechatArticleId":   "desc",
+		}
+		tmpTotal, list, err := elastic.WechatArticleEsSearch(keyWord, wechatPlatformId, startSize, pageSize, sortMap)
+		if err != nil {
+			br.Msg = "获取失败"
+			br.ErrMsg = "获取失败，Err:" + err.Error()
+			return
+		}
+		total = int(tmpTotal)
+		if list != nil && len(list) > 0 {
+			viewList = list[0].ArticleAndPlatformListToViewList(list)
+		}
 	}
+
 	page := paging.GetPaging(currentIndex, pageSize, total)
 	resp := response.WechatArticleListListResp{
 		List:   viewList,

services/elastic/wechat_article.go

@@ -2,8 +2,11 @@ package elastic
 
 import (
 	"context"
+	"encoding/json"
+	"eta/eta_api/models/rag"
 	"eta/eta_api/utils"
 	"fmt"
+	"github.com/olivere/elastic/v7"
 	"time"
 )
 
@@ -32,6 +35,49 @@ type WechatArticleAndPlatform struct {
 	RoundHeadImg      string    `gorm:"column:round_head_img;type:varchar(255);comment:头像;" description:"round_head_img"` // 头像
 }
 
+func (m *WechatArticleAndPlatform) ToView() rag.WechatArticleView {
+	var articleCreateTime, modifyTime, createTime string
+
+	if !m.ArticleCreateTime.IsZero() {
+		articleCreateTime = m.ArticleCreateTime.Format(utils.FormatDateTime)
+	}
+	if !m.CreateTime.IsZero() {
+		createTime = m.CreateTime.Format(utils.FormatDateTime)
+	}
+	if !m.ModifyTime.IsZero() {
+		modifyTime = m.ModifyTime.Format(utils.FormatDateTime)
+	}
+	return rag.WechatArticleView{
+		WechatArticleId:  m.WechatArticleId,
+		WechatPlatformId: m.WechatPlatformId,
+		FakeId:           m.FakeId,
+		Title:            m.Title,
+		Link:             m.Link,
+		CoverUrl:         m.CoverUrl,
+		Description:      m.Description,
+		Content:          m.Content,
+		TextContent:      m.TextContent,
+		//Abstract:                   m.Abstract,
+		Country:                    m.Country,
+		Province:                   m.Province,
+		City:                       m.City,
+		ArticleCreateTime:          articleCreateTime,
+		ModifyTime:                 modifyTime,
+		CreateTime:                 createTime,
+		WechatPlatformName:         m.Nickname,
+		WechatPlatformRoundHeadImg: m.RoundHeadImg,
+	}
+}
+
+func (m *WechatArticleAndPlatform) ArticleAndPlatformListToViewList(list []*WechatArticleAndPlatform) (wechatArticleViewList []rag.WechatArticleView) {
+	wechatArticleViewList = make([]rag.WechatArticleView, 0)
+
+	for _, v := range list {
+		wechatArticleViewList = append(wechatArticleViewList, v.ToView())
+	}
+	return
+}
+
 // WechatArticleEsAddOrEdit
 // @Description: 新增/编辑微信文章
 // @author: Roc
@@ -100,3 +146,157 @@ func WechatArticleEsDel(docId string) (err error) {
 
 	return
 }
+
+func WechatArticleEsSearch(keywordStr string, wechatPlatformId, from, size int, sortMap map[string]string) (total int64, list []*WechatArticleAndPlatform, err error) {
+	indexName := utils.DATA_INDEX_NAME
+	list = make([]*WechatArticleAndPlatform, 0)
+	defer func() {
+		if err != nil {
+			fmt.Println("SearchEdbInfoData Err:", err.Error())
+		}
+	}()
+
+	query := elastic.NewBoolQuery()
+
+	if wechatPlatformId >= 0 {
+		query = query.Must(elastic.NewTermQuery("WechatPlatformId", wechatPlatformId))
+	}
+
+	// 名字匹配
+	if keywordStr != `` {
+		query = query.Must(elastic.NewMultiMatchQuery(keywordStr, "Title"))
+	}
+
+	// 排序
+	sortList := make([]*elastic.FieldSort, 0)
+	// 如果没有关键字，那么就走指标id倒序
+
+	for orderKey, orderType := range sortMap {
+		switch orderType {
+		case "asc":
+			sortList = append(sortList, elastic.NewFieldSort(orderKey).Asc())
+		case "desc":
+			sortList = append(sortList, elastic.NewFieldSort(orderKey).Desc())
+
+		}
+
+	}
+
+	return searchWechatArticle(indexName, query, sortList, from, size)
+}
+
+// searchEdbInfoDataV2 查询es中的数据
+func searchWechatArticle(indexName string, query elastic.Query, sortList []*elastic.FieldSort, from, size int) (total int64, list []*WechatArticleAndPlatform, err error) {
+	total, err = searchWechatArticleTotal(indexName, query)
+	if err != nil {
+		return
+	}
+
+	// 获取列表数据
+	list, err = searchWechatArticleList(indexName, query, sortList, from, size)
+	if err != nil {
+		return
+	}
+
+	return
+}
+
+// searchEdbInfoDataTotal
+// @Description: 查询es中的数量
+// @author: Roc
+// @datetime 2024-12-23 11:19:04
+// @param indexName string
+// @param query elastic.Query
+// @return total int64
+// @return err error
+func searchWechatArticleTotal(indexName string, query elastic.Query) (total int64, err error) {
+	defer func() {
+		if err != nil {
+			fmt.Println("searchEdbInfoDataTotal Err:", err.Error())
+		}
+	}()
+	client := utils.EsClient
+
+	//根据条件数量统计
+	requestTotalHits := client.Count(indexName).Query(query)
+	total, err = requestTotalHits.Do(context.Background())
+	if err != nil {
+		return
+	}
+
+	return
+}
+
+// searchEdbInfoDataList
+// @Description: 查询es中的明细数据
+// @author: Roc
+// @datetime 2024-12-23 11:18:48
+// @param indexName string
+// @param query elastic.Query
+// @param sortList []*elastic.FieldSort
+// @param from int
+// @param size int
+// @return list []*data_manage.EdbInfoList
+// @return err error
+func searchWechatArticleList(indexName string, query elastic.Query, sortList []*elastic.FieldSort, from, size int) (list []*WechatArticleAndPlatform, err error) {
+	list = make([]*WechatArticleAndPlatform, 0)
+	defer func() {
+		if err != nil {
+			fmt.Println("searchEdbInfoDataList Err:", err.Error())
+		}
+	}()
+	client := utils.EsClient
+	// 高亮
+	highlight := elastic.NewHighlight()
+	highlight = highlight.Fields(elastic.NewHighlighterField("Title"))
+	highlight = highlight.PreTags("<font color='red'>").PostTags("</font>")
+
+	request := client.Search(indexName).Highlight(highlight).From(from).Size(size) // sets the JSON request
+	//request := client.Search(indexName).From(from).Size(size) // sets the JSON request
+
+	// 如果有指定排序，那么就按照排序来
+	if len(sortList) > 0 {
+		for _, v := range sortList {
+			request = request.SortBy(v)
+		}
+	}
+
+	searchMap := make(map[string]string)
+
+	searchResp, err := request.Query(query).Do(context.Background())
+	if err != nil {
+		return
+	}
+	//fmt.Println(searchResp)
+	//fmt.Println(searchResp.Status)
+	if searchResp.Status != 0 {
+		return
+	}
+	//total = searchResp.TotalHits()
+	if searchResp.Hits != nil {
+		for _, v := range searchResp.Hits.Hits {
+			if _, ok := searchMap[v.Id]; !ok {
+				itemJson, tmpErr := v.Source.MarshalJSON()
+				if tmpErr != nil {
+					err = tmpErr
+					fmt.Println("movieJson err:", err)
+					return
+				}
+				item := new(WechatArticleAndPlatform)
+				tmpErr = json.Unmarshal(itemJson, &item)
+				if tmpErr != nil {
+					fmt.Println("json.Unmarshal movieJson err:", tmpErr)
+					err = tmpErr
+					return
+				}
+				if len(v.Highlight["Title"]) > 0 {
+					item.Title = v.Highlight["Title"][0]
+				}
+				list = append(list, item)
+				searchMap[v.Id] = v.Id
+			}
+		}
+	}
+
+	return
+}