|
@@ -5,6 +5,7 @@ import (
|
|
|
"eta/eta_api/cache"
|
|
|
"eta/eta_api/models"
|
|
|
"eta/eta_api/models/rag"
|
|
|
+ "eta/eta_api/services/elastic"
|
|
|
"eta/eta_api/services/llm"
|
|
|
"eta/eta_api/utils"
|
|
|
"eta/eta_api/utils/llm/eta_llm/eta_llm_http"
|
|
@@ -245,10 +246,17 @@ func GenerateArticleAbstract(item *rag.WechatArticle) {
|
|
|
}
|
|
|
}()
|
|
|
|
|
|
+
|
|
|
+ if item.TextContent == `` {
|
|
|
+ return
|
|
|
+ }
|
|
|
+
|
|
|
abstractObj := rag.WechatArticleAbstract{}
|
|
|
- _, err = abstractObj.GetByWechatArticleId(item.WechatArticleId)
|
|
|
+ tmpAbstractItem, err := abstractObj.GetByWechatArticleId(item.WechatArticleId)
|
|
|
if err == nil {
|
|
|
|
|
|
+ AbstractToKnowledge(item, tmpAbstractItem, false)
|
|
|
+
|
|
|
return
|
|
|
}
|
|
|
if !utils.IsErrNoRow(err) {
|
|
@@ -292,53 +300,11 @@ func GenerateArticleAbstract(item *rag.WechatArticle) {
|
|
|
|
|
|
|
|
|
|
|
|
- historyList := make([]eta_llm_http.HistoryContent, 0)
|
|
|
-
|
|
|
- questionObj := rag.Question{}
|
|
|
- questionList, err := questionObj.GetListByCondition(``, []interface{}{}, 0, 100)
|
|
|
- if err != nil {
|
|
|
- err = fmt.Errorf("获取问题列表失败,Err:" + err.Error())
|
|
|
- return
|
|
|
- }
|
|
|
-
|
|
|
- addArticleChatRecordList := make([]*rag.WechatArticleChatRecord, 0)
|
|
|
-
|
|
|
- var abstract string
|
|
|
|
|
|
- for _, question := range questionList {
|
|
|
- originalAnswer, tmpAnswer, tmpErr := getAnswerByContent(tmpDocId, question.QuestionContent, historyList)
|
|
|
- if tmpErr != nil {
|
|
|
- err = fmt.Errorf("LLM对话失败,Err:" + tmpErr.Error())
|
|
|
- return
|
|
|
- }
|
|
|
- abstract = tmpAnswer
|
|
|
-
|
|
|
- historyList = append(historyList, eta_llm_http.HistoryContent{
|
|
|
- Role: `user`,
|
|
|
- Content: question.QuestionContent,
|
|
|
- }, eta_llm_http.HistoryContent{
|
|
|
- Role: `assistant`,
|
|
|
- Content: tmpAnswer,
|
|
|
- })
|
|
|
-
|
|
|
-
|
|
|
- addArticleChatRecordList = append(addArticleChatRecordList, &rag.WechatArticleChatRecord{
|
|
|
- WechatArticleChatRecordId: 0,
|
|
|
- WechatArticleId: item.WechatArticleId,
|
|
|
- ChatUserType: "user",
|
|
|
- Content: question.QuestionContent,
|
|
|
- SendTime: time.Now(),
|
|
|
- CreatedTime: time.Now(),
|
|
|
- UpdateTime: time.Now(),
|
|
|
- }, &rag.WechatArticleChatRecord{
|
|
|
- WechatArticleChatRecordId: 0,
|
|
|
- WechatArticleId: item.WechatArticleId,
|
|
|
- ChatUserType: "assistant",
|
|
|
- Content: originalAnswer,
|
|
|
- SendTime: time.Now(),
|
|
|
- CreatedTime: time.Now(),
|
|
|
- UpdateTime: time.Now(),
|
|
|
- })
|
|
|
+ abstract, addArticleChatRecordList, tmpErr := getAnswerByContent(item.WechatArticleId, tmpDocId)
|
|
|
+ if tmpErr != nil {
|
|
|
+ err = fmt.Errorf("LLM对话失败,Err:" + tmpErr.Error())
|
|
|
+ return
|
|
|
}
|
|
|
|
|
|
|
|
@@ -351,6 +317,16 @@ func GenerateArticleAbstract(item *rag.WechatArticle) {
|
|
|
}
|
|
|
|
|
|
if abstract != `` {
|
|
|
+ if abstract == `sorry` || strings.Index(abstract, `根据已知信息无法回答该问题`) == 0 {
|
|
|
+ item.AbstractStatus = 2
|
|
|
+ item.ModifyTime = time.Now()
|
|
|
+ err = item.Update([]string{"AbstractStatus", "ModifyTime"})
|
|
|
+ return
|
|
|
+ }
|
|
|
+ item.AbstractStatus = 1
|
|
|
+ item.ModifyTime = time.Now()
|
|
|
+ err = item.Update([]string{"AbstractStatus", "ModifyTime"})
|
|
|
+
|
|
|
abstractItem := &rag.WechatArticleAbstract{
|
|
|
WechatArticleAbstractId: 0,
|
|
|
WechatArticleId: item.WechatArticleId,
|
|
@@ -365,12 +341,181 @@ func GenerateArticleAbstract(item *rag.WechatArticle) {
|
|
|
return
|
|
|
}
|
|
|
|
|
|
- AbstractToKnowledge(item, abstractItem)
|
|
|
+
|
|
|
+ go AddOrEditEsWechatArticleAbstract(abstractItem.WechatArticleAbstractId)
|
|
|
+
|
|
|
+ AbstractToKnowledge(item, abstractItem, false)
|
|
|
}
|
|
|
}
|
|
|
|
|
|
-func getAnswerByContent(docId, question string, historyList []eta_llm_http.HistoryContent) (originalAnswer, answer string, err error) {
|
|
|
- originalAnswer, result, err := llm.ChatByFile(docId, question, historyList)
|
|
|
+
|
|
|
+
|
|
|
+
|
|
|
+
|
|
|
+
|
|
|
+func ReGenerateArticleAbstract(item *rag.WechatArticle) {
|
|
|
+ var err error
|
|
|
+ defer func() {
|
|
|
+ if err != nil {
|
|
|
+ utils.FileLog.Error("文章转临时文件失败,err:%v", err)
|
|
|
+ fmt.Println("文章转临时文件失败,err:", err)
|
|
|
+ }
|
|
|
+ }()
|
|
|
+
|
|
|
+ abstractObj := rag.WechatArticleAbstract{}
|
|
|
+ abstractItem, err := abstractObj.GetByWechatArticleId(item.WechatArticleId)
|
|
|
+ if err != nil {
|
|
|
+ if utils.IsErrNoRow(err) {
|
|
|
+
|
|
|
+ GenerateArticleAbstract(item)
|
|
|
+ return
|
|
|
+ }
|
|
|
+
|
|
|
+ return
|
|
|
+ }
|
|
|
+
|
|
|
+
|
|
|
+ dateDir := time.Now().Format("20060102")
|
|
|
+ uploadDir := utils.STATIC_DIR + "ai/" + dateDir
|
|
|
+ err = os.MkdirAll(uploadDir, utils.DIR_MOD)
|
|
|
+ if err != nil {
|
|
|
+ err = fmt.Errorf("存储目录创建失败,Err:" + err.Error())
|
|
|
+ return
|
|
|
+ }
|
|
|
+ randStr := utils.GetRandStringNoSpecialChar(28)
|
|
|
+ fileName := randStr + `.md`
|
|
|
+ tmpFilePath := uploadDir + "/" + fileName
|
|
|
+ err = utils.SaveToFile(item.TextContent, tmpFilePath)
|
|
|
+ if err != nil {
|
|
|
+ err = fmt.Errorf("生成临时文件失败,Err:" + err.Error())
|
|
|
+ return
|
|
|
+ }
|
|
|
+ defer func() {
|
|
|
+ os.Remove(tmpFilePath)
|
|
|
+ }()
|
|
|
+
|
|
|
+
|
|
|
+ tmpFileResp, err := llm.UploadTempDocs(tmpFilePath)
|
|
|
+ if err != nil {
|
|
|
+ err = fmt.Errorf("上传临时文件到LLM失败,Err:" + err.Error())
|
|
|
+ return
|
|
|
+ }
|
|
|
+
|
|
|
+ if tmpFileResp.Data.Id == `` {
|
|
|
+ err = fmt.Errorf("上传临时文件到LLM失败,Err:上传失败")
|
|
|
+ return
|
|
|
+ }
|
|
|
+ tmpDocId := tmpFileResp.Data.Id
|
|
|
+
|
|
|
+
|
|
|
+
|
|
|
+
|
|
|
+
|
|
|
+
|
|
|
+ abstract, addArticleChatRecordList, tmpErr := getAnswerByContent(item.WechatArticleId, tmpDocId)
|
|
|
+ if tmpErr != nil {
|
|
|
+ err = fmt.Errorf("LLM对话失败,Err:" + tmpErr.Error())
|
|
|
+ return
|
|
|
+ }
|
|
|
+
|
|
|
+
|
|
|
+ if len(addArticleChatRecordList) > 0 {
|
|
|
+ recordObj := rag.WechatArticleChatRecord{}
|
|
|
+ err = recordObj.CreateInBatches(addArticleChatRecordList)
|
|
|
+ if err != nil {
|
|
|
+ return
|
|
|
+ }
|
|
|
+ }
|
|
|
+
|
|
|
+ if abstract != `` {
|
|
|
+ if abstract == `sorry` || strings.Index(abstract, `根据已知信息无法回答该问题`) == 0 {
|
|
|
+ item.AbstractStatus = 2
|
|
|
+ item.ModifyTime = time.Now()
|
|
|
+ err = item.Update([]string{"AbstractStatus", "ModifyTime"})
|
|
|
+ return
|
|
|
+ }
|
|
|
+ item.AbstractStatus = 1
|
|
|
+ item.ModifyTime = time.Now()
|
|
|
+ err = item.Update([]string{"AbstractStatus", "ModifyTime"})
|
|
|
+
|
|
|
+ abstractItem.Content = abstract
|
|
|
+ abstractItem.Version = abstractObj.Version + 1
|
|
|
+ abstractItem.ModifyTime = time.Now()
|
|
|
+ err = abstractItem.Update([]string{"content", "version", "modify_time"})
|
|
|
+ if err != nil {
|
|
|
+ return
|
|
|
+ }
|
|
|
+
|
|
|
+ AbstractToKnowledge(item, abstractItem, true)
|
|
|
+ }
|
|
|
+}
|
|
|
+
|
|
|
+
|
|
|
+
|
|
|
+
|
|
|
+
|
|
|
+
|
|
|
+
|
|
|
+func DelDoc(wechatArticleAbstractList []*rag.WechatArticleAbstract) (err error) {
|
|
|
+ defer func() {
|
|
|
+ if err != nil {
|
|
|
+ utils.FileLog.Error("删除摘要向量库文件失败,err:%v", err)
|
|
|
+ fmt.Println("删除摘要向量库文件失败,err:", err)
|
|
|
+ }
|
|
|
+ }()
|
|
|
+
|
|
|
+ vectorKeyList := make([]string, 0)
|
|
|
+ wechatArticleAbstractIdList := make([]int, 0)
|
|
|
+
|
|
|
+ for _, v := range wechatArticleAbstractList {
|
|
|
+ if v.VectorKey == `` {
|
|
|
+ continue
|
|
|
+ }
|
|
|
+ vectorKeyList = append(vectorKeyList, v.VectorKey)
|
|
|
+ wechatArticleAbstractIdList = append(wechatArticleAbstractIdList, v.WechatArticleAbstractId)
|
|
|
+ }
|
|
|
+
|
|
|
+
|
|
|
+ if len(vectorKeyList) <= 0 {
|
|
|
+ return
|
|
|
+ }
|
|
|
+
|
|
|
+ _, err = llm.DelDocsToKnowledge(models.BusinessConfMap[models.KnowledgeBaseName], vectorKeyList)
|
|
|
+ if err != nil {
|
|
|
+ err = fmt.Errorf("删除LLM摘要向量库文件失败,Err:" + err.Error())
|
|
|
+ return
|
|
|
+ }
|
|
|
+
|
|
|
+ obj := rag.WechatArticleAbstract{}
|
|
|
+ err = obj.DelVectorKey(wechatArticleAbstractIdList)
|
|
|
+
|
|
|
+ return
|
|
|
+}
|
|
|
+
|
|
|
+func getAnswerByContent(wechatArticleId int, docId string) (answer string, addArticleChatRecordList []*rag.WechatArticleChatRecord, err error) {
|
|
|
+ historyList := make([]eta_llm_http.HistoryContent, 0)
|
|
|
+ addArticleChatRecordList = make([]*rag.WechatArticleChatRecord, 0)
|
|
|
+
|
|
|
+ questionObj := rag.Question{}
|
|
|
+ questionList, err := questionObj.GetListByCondition(``, []interface{}{}, 0, 100)
|
|
|
+ if err != nil {
|
|
|
+ err = fmt.Errorf("获取问题列表失败,Err:" + err.Error())
|
|
|
+ return
|
|
|
+ }
|
|
|
+
|
|
|
+
|
|
|
+ if len(questionList) <= 0 {
|
|
|
+ return
|
|
|
+ }
|
|
|
+
|
|
|
+
|
|
|
+ questionStrList := []string{`你现在是一名资深的期货行业分析师,请基于以下的问题进行汇总总结,如果不能正常总结出来,那么就只需要回复我:sorry。以下是问题:`}
|
|
|
+ for _, v := range questionList {
|
|
|
+ questionStrList = append(questionStrList, v.QuestionContent)
|
|
|
+ }
|
|
|
+ questionStr := strings.Join(questionStrList, "\n")
|
|
|
+
|
|
|
+ originalAnswer, result, err := llm.ChatByFile(docId, questionStr, historyList)
|
|
|
fmt.Println(result)
|
|
|
if err != nil {
|
|
|
err = fmt.Errorf("LLM对话失败,Err:" + err.Error())
|
|
@@ -387,6 +532,25 @@ func getAnswerByContent(docId, question string, historyList []eta_llm_http.Histo
|
|
|
|
|
|
answer = strings.TrimSpace(answer)
|
|
|
|
|
|
+
|
|
|
+ addArticleChatRecordList = append(addArticleChatRecordList, &rag.WechatArticleChatRecord{
|
|
|
+ WechatArticleChatRecordId: 0,
|
|
|
+ WechatArticleId: wechatArticleId,
|
|
|
+ ChatUserType: "user",
|
|
|
+ Content: questionStr,
|
|
|
+ SendTime: time.Now(),
|
|
|
+ CreatedTime: time.Now(),
|
|
|
+ UpdateTime: time.Now(),
|
|
|
+ }, &rag.WechatArticleChatRecord{
|
|
|
+ WechatArticleChatRecordId: 0,
|
|
|
+ WechatArticleId: wechatArticleId,
|
|
|
+ ChatUserType: "assistant",
|
|
|
+ Content: originalAnswer,
|
|
|
+ SendTime: time.Now(),
|
|
|
+ CreatedTime: time.Now(),
|
|
|
+ UpdateTime: time.Now(),
|
|
|
+ })
|
|
|
+
|
|
|
return
|
|
|
}
|
|
|
|
|
@@ -453,12 +617,12 @@ func ArticleToKnowledge(item *rag.WechatArticle) {
|
|
|
|
|
|
|
|
|
|
|
|
-func AbstractToKnowledge(wechatArticleItem *rag.WechatArticle, item *rag.WechatArticleAbstract) {
|
|
|
+func AbstractToKnowledge(wechatArticleItem *rag.WechatArticle, item *rag.WechatArticleAbstract, isReUpload bool) {
|
|
|
if item.Content == `` {
|
|
|
return
|
|
|
}
|
|
|
|
|
|
- if item.VectorKey != `` {
|
|
|
+ if item.VectorKey != `` && !isReUpload {
|
|
|
return
|
|
|
}
|
|
|
var err error
|
|
@@ -546,6 +710,9 @@ func replaceWechatArticleCoverPic(item *rag.WechatArticle) {
|
|
|
utils.FileLog.Error("替换公众号头像失败,err:%v", err)
|
|
|
fmt.Println("替换公众号头像失败,err:", err)
|
|
|
}
|
|
|
+
|
|
|
+
|
|
|
+ AddOrEditEsWechatArticle(item.WechatArticleId)
|
|
|
}()
|
|
|
if item.CoverUrl == `` {
|
|
|
return
|
|
@@ -674,3 +841,156 @@ func handleNode(n *html2.Node) {
|
|
|
handleNode(c)
|
|
|
}
|
|
|
}
|
|
|
+
|
|
|
+
|
|
|
+
|
|
|
+
|
|
|
+
|
|
|
+
|
|
|
+func AddOrEditEsWechatPlatformId(wechatPlatformId int) {
|
|
|
+ if utils.EsWechatArticleName == `` {
|
|
|
+ return
|
|
|
+ }
|
|
|
+ obj := rag.WechatArticle{}
|
|
|
+ list, _ := obj.GetListByCondition(` wechat_article_id `, ` AND wechat_platform_id = ? `, []interface{}{wechatPlatformId}, 0, 1000000)
|
|
|
+ for _, item := range list {
|
|
|
+ AddOrEditEsWechatArticle(item.WechatArticleId)
|
|
|
+ }
|
|
|
+}
|
|
|
+
|
|
|
+
|
|
|
+
|
|
|
+
|
|
|
+
|
|
|
+
|
|
|
+func AddOrEditEsWechatArticle(articleId int) {
|
|
|
+ if utils.EsWechatArticleName == `` {
|
|
|
+ return
|
|
|
+ }
|
|
|
+
|
|
|
+ var err error
|
|
|
+ defer func() {
|
|
|
+ if err != nil {
|
|
|
+ utils.FileLog.Error("添加公众号微信信息到ES失败,err:%v", err)
|
|
|
+ fmt.Println("添加公众号微信信息到ES失败,err:", err)
|
|
|
+ }
|
|
|
+ }()
|
|
|
+ obj := rag.WechatArticle{}
|
|
|
+ articleInfo, err := obj.GetById(articleId)
|
|
|
+ if err != nil {
|
|
|
+ err = fmt.Errorf("获取公众号文章信息失败,Err:" + err.Error())
|
|
|
+ return
|
|
|
+ }
|
|
|
+ platformObj := rag.WechatPlatform{}
|
|
|
+ platformInfo, err := platformObj.GetById(articleInfo.WechatPlatformId)
|
|
|
+ if err != nil {
|
|
|
+ err = fmt.Errorf("获取公众号平台信息失败,Err:" + err.Error())
|
|
|
+ return
|
|
|
+ }
|
|
|
+
|
|
|
+ esItem := elastic.WechatArticleAndPlatform{
|
|
|
+ WechatArticleId: articleInfo.WechatArticleId,
|
|
|
+ WechatPlatformId: articleInfo.WechatPlatformId,
|
|
|
+ FakeId: articleInfo.FakeId,
|
|
|
+ Title: articleInfo.Title,
|
|
|
+ Link: articleInfo.Link,
|
|
|
+ CoverUrl: articleInfo.CoverUrl,
|
|
|
+ Description: articleInfo.Description,
|
|
|
+
|
|
|
+
|
|
|
+
|
|
|
+ Country: articleInfo.Country,
|
|
|
+ Province: articleInfo.Province,
|
|
|
+ City: articleInfo.City,
|
|
|
+ ArticleCreateTime: articleInfo.ArticleCreateTime,
|
|
|
+ IsDeleted: articleInfo.IsDeleted,
|
|
|
+ ModifyTime: articleInfo.ModifyTime,
|
|
|
+ CreateTime: articleInfo.CreateTime,
|
|
|
+ Nickname: platformInfo.Nickname,
|
|
|
+ Alias: platformInfo.Alias,
|
|
|
+ RoundHeadImg: platformInfo.RoundHeadImg,
|
|
|
+ }
|
|
|
+
|
|
|
+ err = elastic.WechatArticleEsAddOrEdit(strconv.Itoa(articleInfo.WechatArticleId), esItem)
|
|
|
+}
|
|
|
+
|
|
|
+
|
|
|
+
|
|
|
+
|
|
|
+
|
|
|
+
|
|
|
+func AddOrEditEsWechatArticleAbstract(articleAbstractId int) {
|
|
|
+ if utils.EsWechatArticleAbstractName == `` {
|
|
|
+ return
|
|
|
+ }
|
|
|
+
|
|
|
+ var err error
|
|
|
+ defer func() {
|
|
|
+ if err != nil {
|
|
|
+ utils.FileLog.Error("添加公众号微信信息到ES失败,err:%v", err)
|
|
|
+ fmt.Println("添加公众号微信信息到ES失败,err:", err)
|
|
|
+ }
|
|
|
+ }()
|
|
|
+ obj := rag.WechatArticleAbstract{}
|
|
|
+ abstractInfo, err := obj.GetById(articleAbstractId)
|
|
|
+ if err != nil {
|
|
|
+ err = fmt.Errorf("获取公众号文章信息失败,Err:" + err.Error())
|
|
|
+ return
|
|
|
+ }
|
|
|
+ articleObj := rag.WechatArticle{}
|
|
|
+ articleInfo, err := articleObj.GetById(abstractInfo.WechatArticleId)
|
|
|
+ if err != nil {
|
|
|
+ err = fmt.Errorf("获取公众号文章信息失败,Err:" + err.Error())
|
|
|
+ return
|
|
|
+ }
|
|
|
+
|
|
|
+
|
|
|
+ tagObj := rag.WechatPlatformTagMapping{}
|
|
|
+ tagMappingList, err := tagObj.GetListByCondition(` AND wechat_platform_id = ? `, []interface{}{articleInfo.WechatPlatformId}, 0, 10000)
|
|
|
+ if err != nil {
|
|
|
+ err = fmt.Errorf("获取公众号平台关联的品种信息失败,Err:" + err.Error())
|
|
|
+ return
|
|
|
+ }
|
|
|
+
|
|
|
+ tagIdList := make([]int, 0)
|
|
|
+ for _, v := range tagMappingList {
|
|
|
+ tagIdList = append(tagIdList, v.TagId)
|
|
|
+ }
|
|
|
+
|
|
|
+ esItem := elastic.WechatArticleAbstractItem{
|
|
|
+ WechatArticleAbstractId: abstractInfo.WechatArticleAbstractId,
|
|
|
+ WechatArticleId: abstractInfo.WechatArticleId,
|
|
|
+ WechatPlatformId: articleInfo.WechatPlatformId,
|
|
|
+ Abstract: abstractInfo.Content,
|
|
|
+ Version: abstractInfo.Version,
|
|
|
+ VectorKey: abstractInfo.VectorKey,
|
|
|
+ ModifyTime: articleInfo.ModifyTime,
|
|
|
+ CreateTime: articleInfo.CreateTime,
|
|
|
+ Title: articleInfo.Title,
|
|
|
+ Link: articleInfo.Link,
|
|
|
+ TagIdList: tagIdList,
|
|
|
+ }
|
|
|
+
|
|
|
+ err = elastic.WechatArticleAbstractEsAddOrEdit(strconv.Itoa(articleAbstractId), esItem)
|
|
|
+}
|
|
|
+
|
|
|
+
|
|
|
+
|
|
|
+
|
|
|
+
|
|
|
+
|
|
|
+func DelEsWechatArticleAbstract(articleAbstractId int) {
|
|
|
+ if utils.EsWechatArticleAbstractName == `` {
|
|
|
+ return
|
|
|
+ }
|
|
|
+
|
|
|
+ var err error
|
|
|
+ defer func() {
|
|
|
+ if err != nil {
|
|
|
+ utils.FileLog.Error("添加公众号微信信息到ES失败,err:%v", err)
|
|
|
+ fmt.Println("添加公众号微信信息到ES失败,err:", err)
|
|
|
+ }
|
|
|
+ }()
|
|
|
+
|
|
|
+ err = elastic.WechatArticleAbstractEsDel(strconv.Itoa(articleAbstractId))
|
|
|
+}
|