eta_api/services/llm/wechat_platform.go

package llm

import (
	"eta/eta_api/cache"
	"eta/eta_api/models"
	"eta/eta_api/models/rag"
	"eta/eta_api/utils"
	"eta/eta_api/utils/llm/eta_llm/eta_llm_http"
	"fmt"
	"html"
	"os"
	"regexp"
	"strconv"
	"strings"
	"time"
)

type WechatArticleOp struct {
	Source           string
	WechatPlatformId int
}

// AddWechatPlatform
// @Description: 添加新的公众号
// @param item
func AddWechatPlatform(item *rag.WechatPlatform) {
	var err error
	defer func() {
		if err != nil {
			utils.FileLog.Error("公众号入库后查找最新记录失败，err:%v", err)
		}
	}()
	if item.FakeId != `` {
		return
	}

	if item.ArticleLink == `` {
		return
	}

	articleLink := item.ArticleLink

	articleDetail, err := SearchByWechatArticle(item.ArticleLink)
	if err != nil {
		return
	}

	if articleDetail.Appuin == `` {
		err = fmt.Errorf("文章内未匹配到公众号唯一标识")
		return
	}

	wechatPlatform := new(rag.WechatPlatform)
	// 查找是否存在这个公众号id的
	wechatPlatformInfo, tmpErr := wechatPlatform.GetByFakeID(articleDetail.Appuin)
	if tmpErr != nil && !utils.IsErrNoRow(tmpErr) {
		err = tmpErr
		return
	}
	if tmpErr == nil {
		// 如果找到了，那么需要将当前的给移除掉
		err = item.Del()
		if err != nil {
			return
		}

		// 并将查出来的微信公众号摘出来的数据重新赋值
		item = wechatPlatformInfo

	} else if utils.IsErrNoRow(tmpErr) {
		// 如果没找到，那么就变更当前的信息
		item.FakeId = articleDetail.Appuin
		item.Nickname = articleDetail.Nickname
		//item.Alias = req.Alias
		item.RoundHeadImg = articleDetail.RoundHeadImg
		//item.ServiceType = req.ServiceType
		item.Signature = articleDetail.ProfileSignature
		//item.Verified = verified
		item.ModifyTime = time.Now()

		err = item.Update([]string{rag.WechatPlatformColumns.FakeID, rag.WechatPlatformColumns.Nickname, rag.WechatPlatformColumns.RoundHeadImg, rag.WechatPlatformColumns.Signature, rag.WechatPlatformColumns.ModifyTime})
		if err != nil {
			return
		}
	}

	// 把刚搜索的文章加入到指标库
	AddWechatArticle(item, articleLink, articleDetail, nil)

	BeachAddWechatArticle(item, 10)
	fmt.Println("公众号入库完成")

	return
}

// AddWechatArticle
// @Description: 添加公众号文章入库
// @author: Roc
// @datetime 2025-03-05 13:24:14
// @param item *rag.WechatPlatform
// @param link string
// @param articleDetail WechatArticleDataResp
func AddWechatArticle(item *rag.WechatPlatform, articleLink string, articleDetail WechatArticleDataResp, articleMenu *ArticleMenu) {
	var err error
	defer func() {
		if err != nil {
			utils.FileLog.Error("公众号文章入库失败，文章链接：%s ，err:%v", articleLink, err)
		}
	}()
	obj := new(rag.WechatArticle)

	_, err = obj.GetByLink(articleLink)
	if err == nil {
		// 文章已经入库了，不需要重复入库
		return
	}

	// 如果不是 ErrNoRow 的时候，那么就是查询数据库出问题了，需要直接返回
	if !utils.IsErrNoRow(err) {
		return
	}

	// 这个时候，说明数据库中没有这个文章，那么需要文章入库
	err = nil

	var publishAt time.Time
	if articleDetail.CreateAt != `` {
		createAtInt, tmpErr := strconv.Atoi(articleDetail.CreateAt)
		if tmpErr == nil {
			publishAt = time.Unix(int64(createAtInt), 1000)
		}
	} else if articleMenu != nil {
		publishAt = time.Unix(int64(articleMenu.UpdateTime), 1000)
	}

	obj = &rag.WechatArticle{
		WechatArticleId:  0,
		WechatPlatformId: item.WechatPlatformId,
		FakeId:           item.FakeId,
		Title:            articleDetail.Title,
		Link:             articleLink,
		CoverUrl:         articleDetail.CoverUrl,
		Description:      articleDetail.Desc,
		Content:          html.EscapeString(articleDetail.HtmlContent),
		TextContent:      articleDetail.TextContent,
		Country:          articleDetail.CountryName,
		Province:         articleDetail.ProvinceName,
		City:             articleDetail.CityName,
		//Abstract:          "",
		//ArticleCreateTime: createAt,
		ModifyTime: time.Now(),
		CreateTime: time.Now(),
	}
	if !publishAt.IsZero() {
		obj.ArticleCreateTime = publishAt
	}

	if articleMenu != nil {
		obj.Title = articleMenu.Title
		//obj.Link = articleMenu.Link
		obj.CoverUrl = articleMenu.Cover
		obj.Description = articleMenu.Digest
	}
	err = obj.Create()

	// 文章入库成功后，需要将相关信息入摘要库
	go cache.AddWechatArticleLlmOpToCache(item.WechatPlatformId, ``)

}

// BeachAddWechatArticle
// @Description: 批量添加公众号文章
// @param item
// @param num
// @return err
func BeachAddWechatArticle(item *rag.WechatPlatform, num int) {
	var err error
	defer func() {
		//fmt.Println("公众号文章批量入库完成")
		if err != nil {
			utils.FileLog.Error("公众号文章批量入库失败，err:%v", err)
			fmt.Println("公众号文章批量入库失败，err:", err)
		}
	}()
	if item.FakeId == `` {
		return
	}

	wechatArticleObj := new(rag.WechatArticle)

	// 获取公众号的文章列表
	articleListResp, err := SearchByWechatArticleList(item.FakeId, num)
	if err != nil {
		return
	}
	for _, articleMenu := range articleListResp.List {
		// 判断文章是否已经入库，如果已经入库了，那么就过滤，不去重复查询微信了
		_, err = wechatArticleObj.GetByLink(articleMenu.Link)
		if err == nil {
			// 文章已经入库了，不需要重复入库
			continue
		}
		if !utils.IsErrNoRow(err) {
			return
		}
		err = nil

		articleDetail, tmpErr := SearchByWechatArticle(articleMenu.Link)
		if tmpErr != nil {
			err = tmpErr
			return
		}

		// 把刚搜索的文章加入到指标库
		AddWechatArticle(item, articleMenu.Link, articleDetail, &articleMenu)

		time.Sleep(10 * time.Second)

	}
	return
}

// GenerateArticleAbstract
// @Description: 文章摘要生成
// @author: Roc
// @datetime 2025-03-10 16:17:53
// @param item *rag.WechatArticle
func GenerateArticleAbstract(item *rag.WechatArticle) {
	var err error
	defer func() {
		if err != nil {
			utils.FileLog.Error("文章转临时文件失败，err:%v", err)
			fmt.Println("文章转临时文件失败，err:", err)
		}
	}()

	abstractObj := rag.WechatArticleAbstract{}
	_, err = abstractObj.GetByWechatArticleId(item.WechatArticleId)
	if err == nil {
		// 摘要已经生成，不需要重复生成
		return
	}
	if !utils.IsErrNoRow(err) {
		return
	}

	// 生成临时文件
	dateDir := time.Now().Format("20060102")
	uploadDir := utils.STATIC_DIR + "ai/" + dateDir
	err = os.MkdirAll(uploadDir, utils.DIR_MOD)
	if err != nil {
		err = fmt.Errorf("存储目录创建失败,Err:" + err.Error())
		return
	}
	randStr := utils.GetRandStringNoSpecialChar(28)
	fileName := randStr + `.md`
	tmpFilePath := uploadDir + "/" + fileName
	err = utils.SaveToFile(item.TextContent, tmpFilePath)
	if err != nil {
		err = fmt.Errorf("生成临时文件失败，Err:" + err.Error())
		return
	}
	defer func() {
		os.Remove(tmpFilePath)
	}()

	// 上传临时文件到LLM
	tmpFileResp, err := UploadTempDocs(tmpFilePath)
	if err != nil {
		err = fmt.Errorf("上传临时文件到LLM失败，Err:" + err.Error())
		return
	}

	if tmpFileResp.Data.Id == `` {
		err = fmt.Errorf("上传临时文件到LLM失败，Err:上传失败")
		return
	}
	tmpDocId := tmpFileResp.Data.Id

	//tmpDocId := `c4d2ee902808408c8b8ed398b33be103` // 钢材
	//tmpDocId := `2dde8afe62d24525a814e74e0a5e35e4` // 钢材
	//tmpDocId := `7634cc1086c04b3687682220a2cf1a48` //

	historyList := make([]eta_llm_http.HistoryContent, 0)

	questionObj := rag.Question{}
	questionList, err := questionObj.GetListByCondition(``, []interface{}{}, 0, 100)
	if err != nil {
		err = fmt.Errorf("获取问题列表失败，Err:" + err.Error())
		return
	}

	addArticleChatRecordList := make([]*rag.WechatArticleChatRecord, 0)

	var abstract string
	//开始对话
	for _, question := range questionList {
		originalAnswer, tmpAnswer, tmpErr := getAnswerByContent(tmpDocId, question.QuestionContent, historyList)
		if tmpErr != nil {
			err = fmt.Errorf("LLM对话失败，Err:" + tmpErr.Error())
			return
		}
		abstract = tmpAnswer

		historyList = append(historyList, eta_llm_http.HistoryContent{
			Role:    `user`,
			Content: question.QuestionContent,
		}, eta_llm_http.HistoryContent{
			Role:    `assistant`,
			Content: tmpAnswer,
		})

		// 待入库的数据
		addArticleChatRecordList = append(addArticleChatRecordList, &rag.WechatArticleChatRecord{
			WechatArticleChatRecordId: 0,
			WechatArticleId:           item.WechatArticleId,
			ChatUserType:              "user",
			Content:                   question.QuestionContent,
			SendTime:                  time.Now(),
			CreatedTime:               time.Now(),
			UpdateTime:                time.Now(),
		}, &rag.WechatArticleChatRecord{
			WechatArticleChatRecordId: 0,
			WechatArticleId:           item.WechatArticleId,
			ChatUserType:              "assistant",
			Content:                   originalAnswer,
			SendTime:                  time.Now(),
			CreatedTime:               time.Now(),
			UpdateTime:                time.Now(),
		})
	}

	// 添加问答记录
	if len(addArticleChatRecordList) > 0 {
		recordObj := rag.WechatArticleChatRecord{}
		err = recordObj.CreateInBatches(addArticleChatRecordList)
		if err != nil {
			return
		}
	}

	if abstract != `` {
		abstractItem := &rag.WechatArticleAbstract{
			WechatArticleAbstractId: 0,
			WechatArticleId:         item.WechatArticleId,
			Content:                 abstract,
			Version:                 0,
			VectorKey:               "",
			ModifyTime:              time.Now(),
			CreateTime:              time.Now(),
		}
		err = abstractItem.Create()
		if err != nil {
			return
		}

		AbstractToKnowledge(item, abstractItem)
	}
}

func getAnswerByContent(docId, question string, historyList []eta_llm_http.HistoryContent) (originalAnswer, answer string, err error) {
	originalAnswer, result, err := ChatByFile(docId, question, historyList)
	fmt.Println(result)
	if err != nil {
		err = fmt.Errorf("LLM对话失败，Err:" + err.Error())
		return
	}

	// 提取 </think> 后面的内容
	thinkEndIndex := strings.Index(result.Answer, "</think>")
	if thinkEndIndex != -1 {
		answer = strings.TrimSpace(result.Answer[thinkEndIndex+len("</think>"):])
	} else {
		answer = result.Answer
	}

	answer = strings.TrimSpace(answer)

	return
}

// ArticleToKnowledge
// @Description: 原文入向量库
// @author: Roc
// @datetime 2025-03-10 16:13:16
// @param item *rag.WechatArticle
func ArticleToKnowledge(item *rag.WechatArticle) {
	if item.TextContent == `` {
		return
	}
	var err error
	defer func() {
		if err != nil {
			utils.FileLog.Error("上传文章原文到知识库失败，err:%v", err)
			fmt.Println("上传文章原文到知识库失败，err:", err)
		}
	}()

	// 生成临时文件
	//dateDir := time.Now().Format("20060102")
	//uploadDir := utils.STATIC_DIR + "ai/article/" + dateDir
	uploadDir := utils.STATIC_DIR + "ai/article"
	err = os.MkdirAll(uploadDir, utils.DIR_MOD)
	if err != nil {
		err = fmt.Errorf("存储目录创建失败,Err:" + err.Error())
		return
	}
	fileName := RemoveSpecialChars(item.Title) + `.md`
	tmpFilePath := uploadDir + "/" + fileName
	err = utils.SaveToFile(item.TextContent, tmpFilePath)
	if err != nil {
		err = fmt.Errorf("生成临时文件失败，Err:" + err.Error())
		return
	}
	defer func() {
		os.Remove(tmpFilePath)
	}()

	knowledgeArticleName := models.BusinessConfMap[models.KnowledgeArticleName]
	// 上传临时文件到LLM
	uploadFileResp, err := UploadDocsToKnowledge(tmpFilePath, knowledgeArticleName)
	if err != nil {
		err = fmt.Errorf("上传文章原文到知识库失败，Err:" + err.Error())
		return
	}

	if len(uploadFileResp.FailedFiles) > 0 {
		for _, v := range uploadFileResp.FailedFiles {
			err = fmt.Errorf("上传文章原文到知识库失败，Err:" + v)
		}
	}

	item.VectorKey = tmpFilePath
	item.ModifyTime = time.Now()
	err = item.Update([]string{"vector_key", "modify_time"})

}

// AbstractToKnowledge
// @Description: 摘要入向量库
// @author: Roc
// @datetime 2025-03-10 16:14:59
// @param wechatArticleItem *rag.WechatArticle
// @param item *rag.WechatArticleAbstract
func AbstractToKnowledge(wechatArticleItem *rag.WechatArticle, item *rag.WechatArticleAbstract) {
	if item.Content == `` {
		return
	}
	// 已经生成了，那就不处理了
	if item.VectorKey != `` {
		return
	}
	var err error
	defer func() {
		if err != nil {
			utils.FileLog.Error("摘要入向量库失败，err:%v", err)
			fmt.Println("摘要入向量库失败，err:", err)
		}
	}()

	// 生成临时文件
	//dateDir := time.Now().Format("20060102")
	//uploadDir := utils.STATIC_DIR + "ai/article/" + dateDir
	uploadDir := utils.STATIC_DIR + "ai/abstract"
	err = os.MkdirAll(uploadDir, utils.DIR_MOD)
	if err != nil {
		err = fmt.Errorf("存储目录创建失败,Err:" + err.Error())
		return
	}
	fileName := RemoveSpecialChars(wechatArticleItem.Title) + `.md`
	tmpFilePath := uploadDir + "/" + fileName
	err = utils.SaveToFile(item.Content, tmpFilePath)
	if err != nil {
		err = fmt.Errorf("生成临时文件失败，Err:" + err.Error())
		return
	}
	defer func() {
		os.Remove(tmpFilePath)
	}()

	knowledgeArticleName := models.BusinessConfMap[models.KnowledgeBaseName]
	// 上传临时文件到LLM
	uploadFileResp, err := UploadDocsToKnowledge(tmpFilePath, knowledgeArticleName)
	if err != nil {
		err = fmt.Errorf("上传文章原文到知识库失败，Err:" + err.Error())
		return
	}

	if len(uploadFileResp.FailedFiles) > 0 {
		for _, v := range uploadFileResp.FailedFiles {
			err = fmt.Errorf("上传文章原文到知识库失败，Err:" + v)
		}
	}

	item.VectorKey = tmpFilePath
	item.ModifyTime = time.Now()
	err = item.Update([]string{"vector_key", "modify_time"})

}

func RemoveSpecialChars(text string) string {
	// 匹配非中文、非字母、非数字、非中文标点的字符
	reg := regexp.MustCompile(`[^\p{Han}\p{L}\p{N}\x{3000}-\x{303F}]`)
	return reg.ReplaceAllString(text, "")
}