Merge remote-tracking branch 'origin/feature/deepseek_rag_1.0' into debug

services/llm/cache.go → cache/wechat_platform.go

@@ -1,10 +1,15 @@
-package llm
+package cache
 
 import (
 	"eta/eta_api/utils"
 	"fmt"
 )
 
+type WechatArticleOp struct {
+	Source           string
+	WechatPlatformId int
+}
+
 // AddWechatArticleOpToCache
 // @Description: 将公众号文章操作加入缓存
 // @param wechatPlatformId

controllers/rag/abstract.go → controllers/llm/abstract.go

@@ -1,4 +1,4 @@
-package rag
+package llm
 
 import (
 	"encoding/json"

controllers/rag/chat_ws_controller.go → controllers/llm/chat_ws_controller.go

@@ -1,4 +1,4 @@
-package rag
+package llm
 
 import (
 	"eta/eta_api/controllers"

controllers/rag/kb_controller.go → controllers/llm/kb_controller.go

@@ -1,4 +1,4 @@
-package rag
+package llm
 
 import (
 	"encoding/json"
@@ -49,4 +49,3 @@ func (kbctrl *KbController) SearchDocs() {
 	br.Success = true
 	br.Msg = "获取成功"
 }
-

controllers/llm/llm_http/response.go

@@ -0,0 +1,13 @@
+package llm_http
+
+import "eta/eta_api/models/llm"
+
+type UserChatListResp struct {
+	TodayList     []llm.UserLlmChatListViewItem
+	YesterdayList []llm.UserLlmChatListViewItem
+	WeekList      []llm.UserLlmChatListViewItem
+}
+type UserChatResp struct {
+	ChatId    int
+	ChatTitle string
+}

controllers/rag/question.go → controllers/llm/question.go

@@ -1,4 +1,4 @@
-package rag
+package llm
 
 import (
 	"encoding/json"

controllers/rag/user_chat_controller.go → controllers/llm/user_chat_controller.go

@@ -1,9 +1,9 @@
-package rag
+package llm
 
 import (
 	"encoding/json"
 	"eta/eta_api/controllers"
-	"eta/eta_api/controllers/rag/llm_http"
+	"eta/eta_api/controllers/llm/llm_http"
 	"eta/eta_api/models"
 	"eta/eta_api/models/llm"
 	llmService "eta/eta_api/services/llm"

controllers/rag/wechat_platform.go → controllers/llm/wechat_platform.go

@@ -1,14 +1,14 @@
-package rag
+package llm
 
 import (
 	"encoding/json"
+	"eta/eta_api/cache"
 	"eta/eta_api/controllers"
 	"eta/eta_api/models"
 	"eta/eta_api/models/rag"
 	"eta/eta_api/models/rag/request"
 	"eta/eta_api/models/rag/response"
 	"eta/eta_api/models/system"
-	"eta/eta_api/services/llm"
 	"eta/eta_api/utils"
 	"fmt"
 	"github.com/rdlucklib/rdluck_tools/paging"
@@ -158,7 +158,7 @@ func (c *WechatPlatformController) Add() {
 	// 异步新增公众号
 	//go llm.AddWechatPlatform(item)
 
-	go llm.AddWechatArticleOpToCache(item.WechatPlatformId, `add`)
+	go cache.AddWechatArticleOpToCache(item.WechatPlatformId, `add`)
 
 	br.Ret = 200
 	br.Success = true
@@ -438,7 +438,7 @@ func (c *WechatPlatformController) Refresh() {
 		return
 	}
 
-	go llm.AddWechatArticleOpToCache(wechatPlatform.WechatPlatformId, `add`)
+	go cache.AddWechatArticleOpToCache(wechatPlatform.WechatPlatformId, `add`)
 
 	br.Ret = 200
 	br.Success = true

routers/router.go

@@ -30,8 +30,8 @@ import (
 	"eta/eta_api/controllers/eta_forum"
 	"eta/eta_api/controllers/eta_trial"
 	"eta/eta_api/controllers/fe_calendar"
+	"eta/eta_api/controllers/llm"
 	"eta/eta_api/controllers/material"
-	"eta/eta_api/controllers/rag"
 	"eta/eta_api/controllers/report_approve"
 	"eta/eta_api/controllers/residual_analysis"
 	"eta/eta_api/controllers/roadshow"
@@ -72,12 +72,12 @@ func init() {
 		),
 		web.NSNamespace("/llm",
 			web.NSInclude(
-				&rag.ChatWsController{},
-				&rag.UserChatController{},
-				&rag.KbController{},
-				&rag.WechatPlatformController{},
-				&rag.QuestionController{},
-				&rag.AbstractController{},
+				&llm.ChatWsController{},
+				&llm.UserChatController{},
+				&llm.KbController{},
+				&llm.WechatPlatformController{},
+				&llm.QuestionController{},
+				&llm.AbstractController{},
 			),
 		),
 		web.NSNamespace("/banner",

services/llm.go

@@ -1,77 +0,0 @@
-package services
-
-import (
-	"eta/eta_api/models/rag"
-	"eta/eta_api/utils"
-	"fmt"
-	"os"
-	"path"
-	"time"
-)
-
-func ReplaceWechatPlatformPic(item *rag.WechatPlatform) {
-	var err error
-	defer func() {
-		if err != nil {
-			utils.FileLog.Error("替换公众号头像失败，err:%v", err)
-			fmt.Println("替换公众号头像失败，err:", err)
-		}
-	}()
-	if item.RoundHeadImg == `` {
-		return
-	}
-	resourceUrl, err := downloadWxPicAndUploadToOss(item.RoundHeadImg)
-	if err != nil {
-		return
-	}
-	item.RoundHeadImg = resourceUrl
-	err = item.Update([]string{"round_head_img"})
-
-}
-
-func ReplaceWechatArticlePic(item *rag.WechatArticle) {
-	var err error
-	defer func() {
-		if err != nil {
-			utils.FileLog.Error("替换公众号头像失败，err:%v", err)
-			fmt.Println("替换公众号头像失败，err:", err)
-		}
-	}()
-	if item.CoverUrl == `` {
-		return
-	}
-	resourceUrl, err := downloadWxPicAndUploadToOss(item.CoverUrl)
-	if err != nil {
-		return
-	}
-	item.CoverUrl = resourceUrl
-	err = item.Update([]string{"cover_url"})
-
-}
-
-func downloadWxPicAndUploadToOss(wxPicUrl string) (resourceUrl string, err error) {
-	localFilePath, err := utils.DownloadWxImage(wxPicUrl)
-	if err != nil {
-		return
-	}
-	defer func() {
-		os.Remove(localFilePath)
-	}()
-	ossClient := NewOssClient()
-	if ossClient == nil {
-		err = fmt.Errorf(`初始化OSS服务失败`)
-		return
-	}
-	ext := path.Ext(localFilePath)
-	randStr := utils.GetRandStringNoSpecialChar(28)
-	fileName := randStr + ext
-	savePath := utils.UploadDir + `wx/wx_article/` + time.Now().Format("200601/20060102/") + fileName
-	resourceUrl, err = ossClient.UploadFile(fileName, localFilePath, savePath)
-	if err != nil {
-		err = fmt.Errorf("文件上传失败，Err:" + err.Error())
-		return
-	}
-
-	return
-
-}

services/task.go

@@ -2,6 +2,7 @@ package services
 
 import (
 	"encoding/json"
+	"eta/eta_api/cache"
 	"eta/eta_api/models"
 	"eta/eta_api/models/rag"
 	"eta/eta_api/services/binlog"
@@ -593,7 +594,7 @@ func HandleWechatArticleOp() {
 	obj := rag.WechatPlatform{}
 	for {
 		utils.Rc.Brpop(utils.CACHE_WECHAT_PLATFORM_ARTICLE, func(b []byte) {
-			wechatArticleOp := new(llm.WechatArticleOp)
+			wechatArticleOp := new(cache.WechatArticleOp)
 			if err := json.Unmarshal(b, &wechatArticleOp); err != nil {
 				fmt.Println("json unmarshal wrong!")
 				return
@@ -606,9 +607,9 @@ func HandleWechatArticleOp() {
 
 			switch wechatArticleOp.Source {
 			case `add`:
-				llm.AddWechatPlatform(item)
+				AddWechatPlatform(item)
 			case `refresh`:
-				llm.BeachAddWechatArticle(item, 2)
+				BeachAddWechatArticle(item, 2)
 
 			}
 		})
@@ -626,7 +627,7 @@ func HandleWechatArticleLLmOp() {
 	obj := rag.WechatArticle{}
 	for {
 		utils.Rc.Brpop(utils.CACHE_WECHAT_PLATFORM_ARTICLE_KNOWLEDGE, func(b []byte) {
-			wechatArticleOp := new(llm.WechatArticleOp)
+			wechatArticleOp := new(cache.WechatArticleOp)
 			if err := json.Unmarshal(b, &wechatArticleOp); err != nil {
 				fmt.Println("json unmarshal wrong!")
 				return
@@ -638,9 +639,9 @@ func HandleWechatArticleLLmOp() {
 			}
 
 			// 文章加入到知识库
-			llm.ArticleToKnowledge(item)
+			ArticleToKnowledge(item)
 			// 生成摘要
-			llm.GenerateArticleAbstract(item)
+			GenerateArticleAbstract(item)
 		})
 	}
 }

services/llm/wechat_platform.go → services/wechat_platform.go

@@ -1,24 +1,21 @@
-package llm
+package services
 
 import (
+	"eta/eta_api/cache"
 	"eta/eta_api/models"
 	"eta/eta_api/models/rag"
+	"eta/eta_api/services/llm"
 	"eta/eta_api/utils"
 	"eta/eta_api/utils/llm/eta_llm/eta_llm_http"
 	"fmt"
 	"html"
 	"os"
-	"regexp"
+	"path"
 	"strconv"
 	"strings"
 	"time"
 )
 
-type WechatArticleOp struct {
-	Source           string
-	WechatPlatformId int
-}
-
 // AddWechatPlatform
 // @Description: 添加新的公众号
 // @param item
@@ -39,7 +36,7 @@ func AddWechatPlatform(item *rag.WechatPlatform) {
 
 	articleLink := item.ArticleLink
 
-	articleDetail, err := SearchByWechatArticle(item.ArticleLink)
+	articleDetail, err := llm.SearchByWechatArticle(item.ArticleLink)
 	if err != nil {
 		return
 	}
@@ -81,9 +78,12 @@ func AddWechatPlatform(item *rag.WechatPlatform) {
 		if err != nil {
 			return
 		}
+
+		// 修改公众号头像
+		go replaceWechatPlatformPic(item)
 	}
 
-	// 把刚搜索的文章加入到指标库
+	// 把刚搜索的文章加入到文章库中
 	AddWechatArticle(item, articleLink, articleDetail, nil)
 
 	BeachAddWechatArticle(item, 10)
@@ -99,7 +99,7 @@ func AddWechatPlatform(item *rag.WechatPlatform) {
 // @param item *rag.WechatPlatform
 // @param link string
 // @param articleDetail WechatArticleDataResp
-func AddWechatArticle(item *rag.WechatPlatform, articleLink string, articleDetail WechatArticleDataResp, articleMenu *ArticleMenu) {
+func AddWechatArticle(item *rag.WechatPlatform, articleLink string, articleDetail llm.WechatArticleDataResp, articleMenu *llm.ArticleMenu) {
 	var err error
 	defer func() {
 		if err != nil {
@@ -162,8 +162,11 @@ func AddWechatArticle(item *rag.WechatPlatform, articleLink string, articleDetai
 	}
 	err = obj.Create()
 
+	// 修改文章封面图
+	go replaceWechatArticlePic(obj)
+
 	// 文章入库成功后，需要将相关信息入摘要库
-	go AddWechatArticleLlmOpToCache(item.WechatPlatformId, ``)
+	go cache.AddWechatArticleLlmOpToCache(item.WechatPlatformId, ``)
 
 }
 
@@ -188,7 +191,7 @@ func BeachAddWechatArticle(item *rag.WechatPlatform, num int) {
 	wechatArticleObj := new(rag.WechatArticle)
 
 	// 获取公众号的文章列表
-	articleListResp, err := SearchByWechatArticleList(item.FakeId, num)
+	articleListResp, err := llm.SearchByWechatArticleList(item.FakeId, num)
 	if err != nil {
 		return
 	}
@@ -204,7 +207,7 @@ func BeachAddWechatArticle(item *rag.WechatPlatform, num int) {
 		}
 		err = nil
 
-		articleDetail, tmpErr := SearchByWechatArticle(articleMenu.Link)
+		articleDetail, tmpErr := llm.SearchByWechatArticle(articleMenu.Link)
 		if tmpErr != nil {
 			err = tmpErr
 			return
@@ -264,7 +267,7 @@ func GenerateArticleAbstract(item *rag.WechatArticle) {
 	}()
 
 	// 上传临时文件到LLM
-	tmpFileResp, err := UploadTempDocs(tmpFilePath)
+	tmpFileResp, err := llm.UploadTempDocs(tmpFilePath)
 	if err != nil {
 		err = fmt.Errorf("上传临时文件到LLM失败，Err:" + err.Error())
 		return
@@ -358,7 +361,7 @@ func GenerateArticleAbstract(item *rag.WechatArticle) {
 }
 
 func getAnswerByContent(docId, question string, historyList []eta_llm_http.HistoryContent) (originalAnswer, answer string, err error) {
-	originalAnswer, result, err := ChatByFile(docId, question, historyList)
+	originalAnswer, result, err := llm.ChatByFile(docId, question, historyList)
 	fmt.Println(result)
 	if err != nil {
 		err = fmt.Errorf("LLM对话失败，Err:" + err.Error())
@@ -404,7 +407,7 @@ func ArticleToKnowledge(item *rag.WechatArticle) {
 		err = fmt.Errorf("存储目录创建失败,Err:" + err.Error())
 		return
 	}
-	fileName := RemoveSpecialChars(item.Title) + `.md`
+	fileName := utils.RemoveSpecialChars(item.Title) + `.md`
 	tmpFilePath := uploadDir + "/" + fileName
 	err = utils.SaveToFile(item.TextContent, tmpFilePath)
 	if err != nil {
@@ -417,7 +420,7 @@ func ArticleToKnowledge(item *rag.WechatArticle) {
 
 	knowledgeArticleName := models.BusinessConfMap[models.KnowledgeArticleName]
 	// 上传临时文件到LLM
-	uploadFileResp, err := UploadDocsToKnowledge(tmpFilePath, knowledgeArticleName)
+	uploadFileResp, err := llm.UploadDocsToKnowledge(tmpFilePath, knowledgeArticleName)
 	if err != nil {
 		err = fmt.Errorf("上传文章原文到知识库失败，Err:" + err.Error())
 		return
@@ -466,7 +469,7 @@ func AbstractToKnowledge(wechatArticleItem *rag.WechatArticle, item *rag.WechatA
 		err = fmt.Errorf("存储目录创建失败,Err:" + err.Error())
 		return
 	}
-	fileName := RemoveSpecialChars(wechatArticleItem.Title) + `.md`
+	fileName := utils.RemoveSpecialChars(wechatArticleItem.Title) + `.md`
 	tmpFilePath := uploadDir + "/" + fileName
 	err = utils.SaveToFile(item.Content, tmpFilePath)
 	if err != nil {
@@ -479,7 +482,7 @@ func AbstractToKnowledge(wechatArticleItem *rag.WechatArticle, item *rag.WechatA
 
 	knowledgeArticleName := models.BusinessConfMap[models.KnowledgeBaseName]
 	// 上传临时文件到LLM
-	uploadFileResp, err := UploadDocsToKnowledge(tmpFilePath, knowledgeArticleName)
+	uploadFileResp, err := llm.UploadDocsToKnowledge(tmpFilePath, knowledgeArticleName)
 	if err != nil {
 		err = fmt.Errorf("上传文章原文到知识库失败，Err:" + err.Error())
 		return
@@ -497,8 +500,87 @@ func AbstractToKnowledge(wechatArticleItem *rag.WechatArticle, item *rag.WechatA
 
 }
 
-func RemoveSpecialChars(text string) string {
-	// 匹配非中文、非字母、非数字、非中文标点的字符
-	reg := regexp.MustCompile(`[^\p{Han}\p{L}\p{N}\x{3000}-\x{303F}]`)
-	return reg.ReplaceAllString(text, "")
+// replaceWechatPlatformPic
+// @Description: 替换公众号头像
+// @author: Roc
+// @datetime 2025-03-11 09:38:24
+// @param item *rag.WechatPlatform
+func replaceWechatPlatformPic(item *rag.WechatPlatform) {
+	var err error
+	defer func() {
+		if err != nil {
+			utils.FileLog.Error("替换公众号头像失败，err:%v", err)
+			fmt.Println("替换公众号头像失败，err:", err)
+		}
+	}()
+	if item.RoundHeadImg == `` {
+		return
+	}
+	resourceUrl, err := downloadWxPicAndUploadToOss(item.RoundHeadImg, `head_img`)
+	if err != nil {
+		return
+	}
+	item.RoundHeadImg = resourceUrl
+	err = item.Update([]string{"round_head_img"})
+
+}
+
+// replaceWechatArticlePic
+// @Description: 替换文章封面图
+// @author: Roc
+// @datetime 2025-03-11 09:38:35
+// @param item *rag.WechatArticle
+func replaceWechatArticlePic(item *rag.WechatArticle) {
+	var err error
+	defer func() {
+		if err != nil {
+			utils.FileLog.Error("替换公众号头像失败，err:%v", err)
+			fmt.Println("替换公众号头像失败，err:", err)
+		}
+	}()
+	if item.CoverUrl == `` {
+		return
+	}
+	resourceUrl, err := downloadWxPicAndUploadToOss(item.CoverUrl, `cover_url`)
+	if err != nil {
+		return
+	}
+	item.CoverUrl = resourceUrl
+	err = item.Update([]string{"cover_url"})
+
+}
+
+// downloadWxPicAndUploadToOss
+// @Description: 下载微信图片并上传到OSS
+// @author: Roc
+// @datetime 2025-03-11 09:28:49
+// @param wxPicUrl string
+// @return resourceUrl string
+// @return err error
+func downloadWxPicAndUploadToOss(wxPicUrl, source string) (resourceUrl string, err error) {
+	localFilePath, err := utils.DownloadWxImage(wxPicUrl)
+	if err != nil {
+		return
+	}
+	defer func() {
+		os.Remove(localFilePath)
+	}()
+	ossClient := NewOssClient()
+	if ossClient == nil {
+		err = fmt.Errorf(`初始化OSS服务失败`)
+		return
+	}
+	ext := path.Ext(localFilePath)
+	randStr := utils.GetRandStringNoSpecialChar(28)
+	fileName := randStr + ext
+	//savePath := utils.UploadDir + `wx/wx_article/` + time.Now().Format("200601/20060102/") + fileName
+	savePath := fmt.Sprintf(`%swx/%s/%s%s`, utils.UploadDir, source, time.Now().Format("200601/20060102/"), fileName)
+	resourceUrl, err = ossClient.UploadFile(fileName, localFilePath, savePath)
+	if err != nil {
+		err = fmt.Errorf("文件上传失败，Err:" + err.Error())
+		return
+	}
+
+	return
+
 }

utils/common.go

@@ -3082,3 +3082,15 @@ func DateStrToDateTimeStr(date time.Time) (formatStr string) {
 
 	return
 }
+
+// RemoveSpecialChars
+// @Description: 移除特殊字符
+// @author: Roc
+// @datetime 2025-03-10 21:06:38
+// @param text string
+// @return string
+func RemoveSpecialChars(text string) string {
+	// 匹配非中文、非字母、非数字、非中文标点的字符
+	reg := regexp.MustCompile(`[^\p{Han}\p{L}\p{N}\x{3000}-\x{303F}]`)
+	return reg.ReplaceAllString(text, "")
+}