Merge remote-tracking branch 'origin/feature/deepseek_rag_1.0' into debug

controllers/llm/abstract.go

@@ -56,9 +56,27 @@ func (c *AbstractController) List() {
 	}
 	startSize = utils.StartIndex(currentIndex, pageSize)
 
-	var total int
-	viewList := make([]rag.WechatArticleAbstractView, 0)
+	// 获取列表
+	total, viewList, err := getAbstractList(keyWord, tagId, startSize, pageSize)
+	if err != nil {
+		br.Msg = "获取失败"
+		br.ErrMsg = "获取失败，Err:" + err.Error()
+		return
+	}
 
+	page := paging.GetPaging(currentIndex, pageSize, total)
+	resp := response.AbstractListListResp{
+		List:   viewList,
+		Paging: page,
+	}
+
+	br.Ret = 200
+	br.Success = true
+	br.Msg = "获取成功"
+	br.Data = resp
+}
+
+func getAbstractList(keyWord string, tagId int, startSize, pageSize int) (total int, viewList []rag.WechatArticleAbstractView, err error) {
 	if keyWord == `` {
 		var condition string
 		var pars []interface{}
@@ -76,10 +94,9 @@ func (c *AbstractController) List() {
 		}
 
 		obj := new(rag.WechatArticleAbstract)
-		tmpTotal, list, err := obj.GetPageListByTagAndPlatformCondition(condition, pars, startSize, pageSize)
-		if err != nil {
-			br.Msg = "获取失败"
-			br.ErrMsg = "获取失败，Err:" + err.Error()
+		tmpTotal, list, tmpErr := obj.GetPageListByTagAndPlatformCondition(condition, pars, startSize, pageSize)
+		if tmpErr != nil {
+			err = tmpErr
 			return
 		}
 		total = tmpTotal
@@ -91,10 +108,9 @@ func (c *AbstractController) List() {
 		}
 
 		obj := new(rag.WechatPlatform)
-		platformList, err := obj.GetListByCondition(` AND enabled = 1 `, []interface{}{}, 0, 100000)
-		if err != nil {
-			br.Msg = "获取失败"
-			br.ErrMsg = "获取失败，Err:" + err.Error()
+		platformList, tmpErr := obj.GetListByCondition(` AND enabled = 1 `, []interface{}{}, 0, 100000)
+		if tmpErr != nil {
+			err = tmpErr
 			return
 		}
 		platformIdList := make([]int, 0)
@@ -105,10 +121,9 @@ func (c *AbstractController) List() {
 		if tagId > 0 {
 			tagList = append(tagList, tagId)
 		}
-		tmpTotal, list, err := elastic.WechatArticleAbstractEsSearch(keyWord, tagList, platformIdList, startSize, pageSize, sortMap)
-		if err != nil {
-			br.Msg = "获取失败"
-			br.ErrMsg = "获取失败，Err:" + err.Error()
+		tmpTotal, list, tmpErr := elastic.WechatArticleAbstractEsSearch(keyWord, tagList, platformIdList, startSize, pageSize, sortMap)
+		if tmpErr != nil {
+			err = tmpErr
 			return
 		}
 		total = int(tmpTotal)
@@ -117,22 +132,13 @@ func (c *AbstractController) List() {
 		}
 	}
 
-	page := paging.GetPaging(currentIndex, pageSize, total)
-	resp := response.AbstractListListResp{
-		List:   viewList,
-		Paging: page,
-	}
-
-	br.Ret = 200
-	br.Success = true
-	br.Msg = "获取成功"
-	br.Data = resp
+	return
 }
 
 // Del
 // @Title 删除摘要
 // @Description 删除摘要
-// @Param	request	body request.DelAbstractReq true "type json string"
+// @Param	request	body request.BeachOpAbstractReq true "type json string"
 // @Success 200 Ret=200 新增成功
 // @router /abstract/del [post]
 func (c *AbstractController) Del() {
@@ -141,54 +147,95 @@ func (c *AbstractController) Del() {
 		c.Data["json"] = br
 		c.ServeJSON()
 	}()
-	var req request.DelAbstractReq
+	var req request.BeachOpAbstractReq
 	err := json.Unmarshal(c.Ctx.Input.RequestBody, &req)
 	if err != nil {
 		br.Msg = "参数解析异常！"
 		br.ErrMsg = "参数解析失败，Err:" + err.Error()
 		return
 	}
-	if len(req.WechatArticleAbstractIdList) <= 0 {
+	if len(req.WechatArticleAbstractIdList) <= 0 && !req.IsSelectAll {
 		br.Msg = "请选择摘要"
 		br.IsSendEmail = false
 		return
 	}
 
+	vectorKeyList := make([]string, 0)
+	wechatArticleAbstractIdList := make([]int, 0)
+
 	obj := rag.WechatArticleAbstract{}
-	list, err := obj.GetByIdList(req.WechatArticleAbstractIdList)
-	if err != nil {
-		br.Msg = "修改失败"
-		br.ErrMsg = "修改失败，查找问题失败，Err:" + err.Error()
-		if utils.IsErrNoRow(err) {
-			br.Msg = "问题不存在"
-			br.IsSendEmail = false
-		}
-		return
-	}
 
-	if len(list) > 0 {
-		// 删除向量库
-		err = services.DelDoc(list)
+	if !req.IsSelectAll {
+		list, err := obj.GetByIdList(req.WechatArticleAbstractIdList)
 		if err != nil {
-			br.Msg = "删除失败"
-			br.ErrMsg = "删除向量库失败，Err:" + err.Error()
+			br.Msg = "修改失败"
+			br.ErrMsg = "修改失败，查找问题失败，Err:" + err.Error()
+			if utils.IsErrNoRow(err) {
+				br.Msg = "问题不存在"
+				br.IsSendEmail = false
+			}
 			return
 		}
+		if len(list) > 0 {
+			for _, v := range list {
+				// 有加入到向量库，那么就加入到待删除的向量库list中
+				if v.VectorKey != `` {
+					vectorKeyList = append(vectorKeyList, v.VectorKey)
+				}
+				wechatArticleAbstractIdList = append(wechatArticleAbstractIdList, v.WechatArticleAbstractId)
+			}
+		}
+	} else {
+		notIdMap := make(map[int]bool)
+		for _, v := range req.NotWechatArticleAbstractIdList {
+			notIdMap[v] = true
+		}
 
-		// 删除摘要
-		err = obj.DelByIdList(req.WechatArticleAbstractIdList)
+		_, list, err := getAbstractList(req.KeyWord, req.TagId, 0, 100000)
 		if err != nil {
-			br.Msg = "删除失败"
-			br.ErrMsg = "删除失败，Err:" + err.Error()
+			br.Msg = "修改失败"
+			br.ErrMsg = "修改失败，查找问题失败，Err:" + err.Error()
+			if utils.IsErrNoRow(err) {
+				br.Msg = "问题不存在"
+				br.IsSendEmail = false
+			}
 			return
 		}
-
-		// 删除es数据
-		for _, v := range list {
-			go services.DelEsWechatArticleAbstract(v.WechatArticleAbstractId)
+		if len(list) > 0 {
+			for _, v := range list {
+				if notIdMap[v.WechatArticleAbstractId] {
+					continue
+				}
+				// 有加入到向量库，那么就加入到待删除的向量库list中
+				if v.VectorKey != `` {
+					vectorKeyList = append(vectorKeyList, v.VectorKey)
+				}
+				wechatArticleAbstractIdList = append(wechatArticleAbstractIdList, v.WechatArticleAbstractId)
+			}
 		}
 	}
 
+	// 删除向量库
+	err = services.DelLlmDoc(vectorKeyList, wechatArticleAbstractIdList)
+	if err != nil {
+		br.Msg = "删除失败"
+		br.ErrMsg = "删除向量库失败，Err:" + err.Error()
+		return
+	}
+
+	// 删除摘要
+	err = obj.DelByIdList(req.WechatArticleAbstractIdList)
+	if err != nil {
+		br.Msg = "删除失败"
+		br.ErrMsg = "删除失败，Err:" + err.Error()
+		return
+	}
+
+	// 删除es数据
+	for _, wechatArticleAbstractId := range wechatArticleAbstractIdList {
+		go services.DelEsWechatArticleAbstract(wechatArticleAbstractId)
+	}
+
 	br.Ret = 200
 	br.Success = true
 	br.Msg = `删除成功`
@@ -197,7 +244,7 @@ func (c *AbstractController) Del() {
 // VectorDel
 // @Title 删除摘要向量库
 // @Description 删除摘要向量库
-// @Param	request	body request.EditQuestionReq true "type json string"
+// @Param	request	body request.BeachOpAbstractReq true "type json string"
 // @Success 200 Ret=200 新增成功
 // @router /abstract/vector/del [post]
 func (c *AbstractController) VectorDel() {
@@ -206,33 +253,76 @@ func (c *AbstractController) VectorDel() {
 		c.Data["json"] = br
 		c.ServeJSON()
 	}()
-	var req request.DelAbstractReq
+	var req request.BeachOpAbstractReq
 	err := json.Unmarshal(c.Ctx.Input.RequestBody, &req)
 	if err != nil {
 		br.Msg = "参数解析异常！"
 		br.ErrMsg = "参数解析失败，Err:" + err.Error()
 		return
 	}
-	if len(req.WechatArticleAbstractIdList) <= 0 {
+	if len(req.WechatArticleAbstractIdList) <= 0 && !req.IsSelectAll {
 		br.Msg = "请选择摘要"
 		br.IsSendEmail = false
 		return
 	}
 
+	vectorKeyList := make([]string, 0)
+	wechatArticleAbstractIdList := make([]int, 0)
+
 	obj := rag.WechatArticleAbstract{}
-	list, err := obj.GetByIdList(req.WechatArticleAbstractIdList)
-	if err != nil {
-		br.Msg = "修改失败"
-		br.ErrMsg = "修改失败，查找问题失败，Err:" + err.Error()
-		if utils.IsErrNoRow(err) {
-			br.Msg = "问题不存在"
-			br.IsSendEmail = false
+
+	if !req.IsSelectAll {
+		list, err := obj.GetByIdList(req.WechatArticleAbstractIdList)
+		if err != nil {
+			br.Msg = "修改失败"
+			br.ErrMsg = "修改失败，查找问题失败，Err:" + err.Error()
+			if utils.IsErrNoRow(err) {
+				br.Msg = "问题不存在"
+				br.IsSendEmail = false
+			}
+			return
+		}
+		if len(list) > 0 {
+			for _, v := range list {
+				// 有加入到向量库，那么就加入到待删除的向量库list中
+				if v.VectorKey != `` {
+					vectorKeyList = append(vectorKeyList, v.VectorKey)
+				}
+				wechatArticleAbstractIdList = append(wechatArticleAbstractIdList, v.WechatArticleAbstractId)
+			}
+		}
+	} else {
+		notIdMap := make(map[int]bool)
+		for _, v := range req.NotWechatArticleAbstractIdList {
+			notIdMap[v] = true
+		}
+		_, list, err := getAbstractList(req.KeyWord, req.TagId, 0, 100000)
+		if err != nil {
+			br.Msg = "修改失败"
+			br.ErrMsg = "修改失败，查找问题失败，Err:" + err.Error()
+			if utils.IsErrNoRow(err) {
+				br.Msg = "问题不存在"
+				br.IsSendEmail = false
+			}
+			return
+		}
+		if len(list) > 0 {
+			for _, v := range list {
+				if notIdMap[v.WechatArticleAbstractId] {
+					continue
+				}
+
+				// 有加入到向量库，那么就加入到待删除的向量库list中
+				if v.VectorKey != `` {
+					vectorKeyList = append(vectorKeyList, v.VectorKey)
+				}
+				wechatArticleAbstractIdList = append(wechatArticleAbstractIdList, v.WechatArticleAbstractId)
+			}
 		}
-		return
 	}
 
-	// 删除向量库
-	err = services.DelDoc(list)
+	// 删除摘要库
+	err = services.DelLlmDoc(vectorKeyList, wechatArticleAbstractIdList)
 	if err != nil {
 		br.Msg = "删除失败"
 		br.ErrMsg = "删除失败，Err:" + err.Error()
@@ -240,8 +330,8 @@ func (c *AbstractController) VectorDel() {
 	}
 
 	// 修改ES数据
-	for _, v := range list {
-		go services.AddOrEditEsWechatArticleAbstract(v.WechatArticleAbstractId)
+	for _, wechatArticleAbstractId := range wechatArticleAbstractIdList {
+		go services.AddOrEditEsWechatArticleAbstract(wechatArticleAbstractId)
 	}
 
 	br.Ret = 200
@@ -252,7 +342,7 @@ func (c *AbstractController) VectorDel() {
 // AddVector
 // @Title 删除摘要向量库
 // @Description 删除摘要向量库
-// @Param	request	body request.EditQuestionReq true "type json string"
+// @Param	request	body request.BeachOpAbstractReq true "type json string"
 // @Success 200 Ret=200 新增成功
 // @router /abstract/vector/add [post]
 func (c *AbstractController) AddVector() {
@@ -261,32 +351,67 @@ func (c *AbstractController) AddVector() {
 		c.Data["json"] = br
 		c.ServeJSON()
 	}()
-	var req request.DelAbstractReq
+	var req request.BeachOpAbstractReq
 	err := json.Unmarshal(c.Ctx.Input.RequestBody, &req)
 	if err != nil {
 		br.Msg = "参数解析异常！"
 		br.ErrMsg = "参数解析失败，Err:" + err.Error()
 		return
 	}
-	if len(req.WechatArticleAbstractIdList) <= 0 {
+	if len(req.WechatArticleAbstractIdList) <= 0 && !req.IsSelectAll {
 		br.Msg = "请选择摘要"
 		br.IsSendEmail = false
 		return
 	}
 
+	wechatArticleAbstractIdList := make([]int, 0)
+
 	obj := rag.WechatArticleAbstract{}
-	list, err := obj.GetByIdList(req.WechatArticleAbstractIdList)
-	if err != nil {
-		br.Msg = "修改失败"
-		br.ErrMsg = "修改失败，查找问题失败，Err:" + err.Error()
-		if utils.IsErrNoRow(err) {
-			br.Msg = "问题不存在"
-			br.IsSendEmail = false
+
+	if !req.IsSelectAll {
+		list, err := obj.GetByIdList(req.WechatArticleAbstractIdList)
+		if err != nil {
+			br.Msg = "修改失败"
+			br.ErrMsg = "修改失败，查找问题失败，Err:" + err.Error()
+			if utils.IsErrNoRow(err) {
+				br.Msg = "问题不存在"
+				br.IsSendEmail = false
+			}
+			return
+		}
+		if len(list) > 0 {
+			for _, v := range list {
+				wechatArticleAbstractIdList = append(wechatArticleAbstractIdList, v.WechatArticleAbstractId)
+			}
+		}
+	} else {
+		notIdMap := make(map[int]bool)
+		for _, v := range req.NotWechatArticleAbstractIdList {
+			notIdMap[v] = true
+		}
+
+		_, list, err := getAbstractList(req.KeyWord, req.TagId, 0, 100000)
+		if err != nil {
+			br.Msg = "修改失败"
+			br.ErrMsg = "修改失败，查找问题失败，Err:" + err.Error()
+			if utils.IsErrNoRow(err) {
+				br.Msg = "问题不存在"
+				br.IsSendEmail = false
+			}
+			return
+		}
+		if len(list) > 0 {
+			for _, v := range list {
+				if notIdMap[v.WechatArticleAbstractId] {
+					continue
+				}
+				wechatArticleAbstractIdList = append(wechatArticleAbstractIdList, v.WechatArticleAbstractId)
+			}
 		}
-		return
 	}
-	for _, item := range list {
-		cache.AddWechatArticleLlmOpToCache(item.WechatArticleId, ``)
+
+	for _, wechatArticleAbstractId := range wechatArticleAbstractIdList {
+		cache.AddWechatArticleLlmOpToCache(wechatArticleAbstractId, ``)
 	}
 
 	br.Ret = 200

models/rag/request/wechat_platform.go

@@ -24,8 +24,10 @@ type EditQuestionReq struct {
 	Content    string `description:"公众号名称"`
 }
 
-type DelAbstractReq struct {
-	WechatArticleAbstractIdList []int `description:"摘要id"`
-	TagId                       int   `description:"标签id"`
-	IsSelectAll                 bool  `description:"是否选择所有摘要"`
+type BeachOpAbstractReq struct {
+	WechatArticleAbstractIdList    []int  `description:"摘要id"`
+	NotWechatArticleAbstractIdList []int  `description:"不需要的摘要id"`
+	KeyWord                        string `description:"关键字"`
+	TagId                          int    `description:"标签id"`
+	IsSelectAll                    bool   `description:"是否选择所有摘要"`
 }

services/task.go

@@ -641,7 +641,7 @@ func HandleWechatArticleLLmOp() {
 			// 文章加入到知识库
 			ArticleToKnowledge(item)
 			// 生成摘要
-			GenerateArticleAbstract(item)
+			//GenerateArticleAbstract(item)
 		})
 	}
 }

services/wechat_platform.go

@@ -492,6 +492,37 @@ func DelDoc(wechatArticleAbstractList []*rag.WechatArticleAbstract) (err error)
 	return
 }
 
+// DelLlmDoc
+// @Description: 删除摘要向量库
+// @author: Roc
+// @datetime 2025-03-12 16:55:05
+// @param wechatArticleAbstractList []*rag.WechatArticleAbstract
+// @return err error
+func DelLlmDoc(vectorKeyList []string, wechatArticleAbstractIdList []int) (err error) {
+	defer func() {
+		if err != nil {
+			utils.FileLog.Error("删除摘要向量库文件失败，err:%v", err)
+			fmt.Println("删除摘要向量库文件失败，err:", err)
+		}
+	}()
+
+	// 没有就不删除
+	if len(vectorKeyList) <= 0 {
+		return
+	}
+
+	_, err = llm.DelDocsToKnowledge(models.BusinessConfMap[models.KnowledgeBaseName], vectorKeyList)
+	if err != nil {
+		err = fmt.Errorf("删除LLM摘要向量库文件失败，Err:" + err.Error())
+		return
+	}
+	//fmt.Println(resp)
+	obj := rag.WechatArticleAbstract{}
+	err = obj.DelVectorKey(wechatArticleAbstractIdList)
+
+	return
+}
+
 func getAnswerByContent(wechatArticleId int, docId string) (answer string, addArticleChatRecordList []*rag.WechatArticleChatRecord, err error) {
 	historyList := make([]eta_llm_http.HistoryContent, 0)
 	addArticleChatRecordList = make([]*rag.WechatArticleChatRecord, 0)