llm_report.go 22 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751
  1. package services
  2. import (
  3. "encoding/json"
  4. "errors"
  5. "eta/eta_api/cache"
  6. "eta/eta_api/models"
  7. "eta/eta_api/models/rag"
  8. "eta/eta_api/services/elastic"
  9. "eta/eta_api/services/llm"
  10. "eta/eta_api/utils"
  11. "fmt"
  12. "golang.org/x/net/html"
  13. "golang.org/x/net/html/atom"
  14. "os"
  15. "regexp"
  16. "strconv"
  17. "strings"
  18. "time"
  19. )
  20. // ReportAddOrModifyKnowledge
  21. // @Description: ETA报告加入/修改到知识库
  22. // @author: Roc
  23. // @datetime 2025-04-07 14:41:45
  24. // @param reportId int
  25. // @param reportChapterId int
  26. func ReportAddOrModifyKnowledge(reportId, reportChapterId int) {
  27. if reportId <= 0 {
  28. return
  29. }
  30. var err error
  31. defer func() {
  32. if err != nil {
  33. //fmt.Println("ReportAddOrModifyKnowledge error:", err)
  34. utils.FileLog.Error("ReportAddOrModifyKnowledge error:", err)
  35. }
  36. }()
  37. var title, author, htmlContent string
  38. var publishTime time.Time
  39. if reportChapterId > 0 {
  40. chapterInfo, tmpErr := models.GetReportChapterInfoById(reportChapterId)
  41. if tmpErr != nil {
  42. return
  43. }
  44. title = chapterInfo.Title
  45. author = chapterInfo.Author
  46. publishTime = chapterInfo.PublishTime
  47. htmlContent = chapterInfo.Content
  48. } else {
  49. reportInfo, tmpErr := models.GetReportByReportId(reportId)
  50. if tmpErr != nil {
  51. return
  52. }
  53. title = reportInfo.Title
  54. author = reportInfo.Author
  55. publishTime = reportInfo.PublishTime
  56. htmlContent = reportInfo.Content
  57. }
  58. err = handleReportAddOrModifyKnowledge(reportId, reportChapterId, title, author, htmlContent, publishTime)
  59. return
  60. }
  61. // ReportAddOrModifyKnowledgeByReportId
  62. // @Description: ETA报告加入/修改到知识库(只传id的情况)
  63. // @author: Roc
  64. // @datetime 2025-04-07 15:41:15
  65. // @param reportId int
  66. func ReportAddOrModifyKnowledgeByReportId(reportId int) {
  67. if reportId <= 0 {
  68. return
  69. }
  70. errList := make([]string, 0)
  71. defer func() {
  72. if len(errList) > 0 {
  73. utils.FileLog.Error("ReportAddOrModifyKnowledge error,报告ID:%d:%s", reportId, strings.Join(errList, "\n"))
  74. }
  75. }()
  76. reportInfo, err := models.GetReportByReportId(reportId)
  77. if err != nil {
  78. errList = append(errList, err.Error())
  79. return
  80. }
  81. // 如果是单篇报告,那么直接处理
  82. if reportInfo.HasChapter == 0 {
  83. err = handleReportAddOrModifyKnowledge(reportId, 0, reportInfo.Title, reportInfo.Author, reportInfo.Content, reportInfo.PublishTime)
  84. if err != nil {
  85. errList = append(errList, err.Error())
  86. }
  87. return
  88. }
  89. // 章节类型的报告,需要查询出来后再处理
  90. chapterInfoList, err := models.GetPublishedChapterListByReportId(reportId)
  91. if err != nil {
  92. errList = append(errList, err.Error())
  93. return
  94. }
  95. for _, v := range chapterInfoList {
  96. err = handleReportAddOrModifyKnowledge(reportId, v.ReportChapterId, v.Title, reportInfo.Author, v.Content, v.PublishTime)
  97. if err != nil {
  98. errList = append(errList, fmt.Sprintf("第%d章:%s,异常:\n%s", v.ReportChapterId, v.Title, err.Error()))
  99. continue
  100. }
  101. }
  102. return
  103. }
  104. // handleReportAddOrModifyKnowledge
  105. // @Description: 处理ETA报告加入/修改到知识库
  106. // @author: Roc
  107. // @datetime 2025-04-07 15:33:38
  108. // @param reportId int
  109. // @param reportChapterId int
  110. // @param title string
  111. // @param author string
  112. // @param htmlContent string
  113. // @param publishTime time.Time
  114. // @return err error
  115. func handleReportAddOrModifyKnowledge(reportId, reportChapterId int, title, author, htmlContent string, publishTime time.Time) (err error) {
  116. htmlContent = html.UnescapeString(htmlContent)
  117. doc, err := html.Parse(strings.NewReader(htmlContent))
  118. if err != nil {
  119. return
  120. }
  121. // 只获取文本内容
  122. content := &strings.Builder{}
  123. getArticleContent(content, doc)
  124. textContent := content.String()
  125. textContent = regexp.MustCompile(`\n+`).ReplaceAllString(textContent, "\n")
  126. textContent = strings.Trim(textContent, "\n")
  127. publishTimeStr := `未知`
  128. if !publishTime.IsZero() {
  129. title = fmt.Sprintf("%s(%s)", title, publishTime.Format(utils.FormatMonthDayUnSpace))
  130. publishTimeStr = publishTime.Format(utils.FormatDateTime)
  131. }
  132. textContent = fmt.Sprintf("标题:%s\n发布时间:%s\n%s", title, publishTimeStr, textContent)
  133. obj := rag.RagEtaReport{}
  134. item, err := obj.GetByReportAndChapterId(reportId, reportChapterId)
  135. if err != nil && !utils.IsErrNoRow(err) {
  136. // 查询异常,且不是没找到数据的报错
  137. return
  138. }
  139. if err == nil {
  140. // 标记删除了的话,那就不处理了
  141. if item.IsDeleted == 1 {
  142. return
  143. }
  144. item.Title = title
  145. item.Author = author
  146. item.TextContent = textContent
  147. item.IsPublished = 1
  148. //item.PublishTime = publishTime
  149. item.ModifyTime = time.Now()
  150. //err = item.Update([]string{"title", "author", "text_content", "is_published", "publish_time", "modify_time"})
  151. err = item.Update([]string{"title", "author", "text_content", "is_published", "modify_time"})
  152. } else {
  153. // 无数据的时候,需要新增
  154. err = nil
  155. item = &rag.RagEtaReport{
  156. RagEtaReportId: 0,
  157. ReportId: reportId,
  158. ReportChapterId: reportChapterId,
  159. Title: title,
  160. Author: author,
  161. TextContent: textContent,
  162. VectorKey: "",
  163. IsPublished: 1,
  164. IsDeleted: 0,
  165. PublishTime: publishTime,
  166. ModifyTime: time.Now(),
  167. CreateTime: time.Now(),
  168. }
  169. err = item.Create()
  170. }
  171. cache.AddRagEtaReportLlmOpToCache(item.RagEtaReportId)
  172. return
  173. }
  174. // ReportUnPublishedKnowledge
  175. // @Description: 知识库取消发布
  176. // @author: Roc
  177. // @datetime 2025-04-07 14:58:25
  178. // @param reportId int
  179. // @param reportChapterId int
  180. func ReportUnPublishedKnowledge(reportId, reportChapterId int) {
  181. if reportId <= 0 && reportChapterId <= 0 {
  182. return
  183. }
  184. var err error
  185. defer func() {
  186. if err != nil {
  187. //fmt.Println("ReportAddOrModifyKnowledge error:", err)
  188. utils.FileLog.Error("ReportAddOrModifyKnowledge error:", err)
  189. }
  190. }()
  191. obj := rag.RagEtaReport{}
  192. item, err := obj.GetByReportAndChapterId(reportId, reportChapterId)
  193. if err != nil && !utils.IsErrNoRow(err) {
  194. // 查询异常,且不是没找到数据的报错
  195. return
  196. }
  197. if item.RagEtaReportId > 0 {
  198. item.IsPublished = 0
  199. item.ModifyTime = time.Now()
  200. err = item.Update([]string{"is_published", "modify_time"})
  201. }
  202. return
  203. }
  204. // ReportUnPublishedKnowledgeByReportId
  205. // @Description: ETA报告取消发布同步到知识库(只传报告id的情况)
  206. // @author: Roc
  207. // @datetime 2025-04-07 15:41:15
  208. // @param reportId int
  209. func ReportUnPublishedKnowledgeByReportId(reportId int) {
  210. errList := make([]string, 0)
  211. defer func() {
  212. if len(errList) > 0 {
  213. utils.FileLog.Error("ReportUnPublishedKnowledgeByReportId error,报告ID:%d:%s", reportId, strings.Join(errList, "\n"))
  214. }
  215. }()
  216. obj := rag.RagEtaReport{}
  217. list, err := obj.GetListByCondition(``, ` AND report_id = ? `, []interface{}{reportId}, 0, 1000)
  218. if err != nil && !utils.IsErrNoRow(err) {
  219. // 查询异常,且不是没找到数据的报错
  220. return
  221. }
  222. for _, item := range list {
  223. item.IsPublished = 0
  224. item.ModifyTime = time.Now()
  225. err = item.Update([]string{"is_published", "modify_time"})
  226. if err != nil {
  227. errList = append(errList, fmt.Sprintf("第%d章:%s,异常:\n%s", item.ReportChapterId, item.Title, err.Error()))
  228. continue
  229. }
  230. // 删除摘要
  231. err = DelRagEtaReportAbstract([]int{item.RagEtaReportId})
  232. }
  233. return
  234. }
  235. func getArticleContent(content *strings.Builder, htmlContentNode *html.Node) {
  236. if htmlContentNode.Type == html.TextNode {
  237. cleanData := strings.TrimSpace(htmlContentNode.Data)
  238. if cleanData != `` && cleanData != "</p>" {
  239. content.WriteString(cleanData)
  240. }
  241. } else if htmlContentNode.Type == html.ElementNode {
  242. switch htmlContentNode.DataAtom {
  243. case atom.Ul:
  244. content.WriteString("\n")
  245. case atom.Br:
  246. // 遇到 <br> 标签时添加换行符
  247. content.WriteString("\n")
  248. case atom.P:
  249. content.WriteString("\n")
  250. }
  251. }
  252. for c := htmlContentNode.FirstChild; c != nil; c = c.NextSibling {
  253. getArticleContent(content, c)
  254. }
  255. }
  256. // GenerateArticleAbstract
  257. // @Description: 文章摘要生成(默认提示词批量生成)
  258. // @author: Roc
  259. // @datetime 2025-03-10 16:17:53
  260. // @param item *rag.RagEtaReport
  261. func GenerateRagEtaReportAbstract(item *rag.RagEtaReport, forceGenerate bool) {
  262. var err error
  263. defer func() {
  264. if err != nil {
  265. utils.FileLog.Error("文章转临时文件失败,err:%v", err)
  266. fmt.Println("文章转临时文件失败,err:", err)
  267. }
  268. }()
  269. // 内容为空,那就不需要生成摘要
  270. if item.TextContent == `` {
  271. return
  272. }
  273. questionObj := rag.Question{}
  274. questionList, err := questionObj.GetListByCondition(``, ` AND is_default = 1 `, []interface{}{}, 0, 100)
  275. if err != nil {
  276. err = fmt.Errorf("获取问题列表失败,Err:" + err.Error())
  277. return
  278. }
  279. // 没问题就不生成了
  280. if len(questionList) <= 0 {
  281. return
  282. }
  283. for _, question := range questionList {
  284. GenerateRagEtaReportAbstractByQuestion(item, question, forceGenerate)
  285. }
  286. return
  287. }
  288. // GenerateRagEtaReportAbstractByQuestion
  289. // @Description: 文章摘要生成(根据提示词生成)
  290. // @author: Roc
  291. // @datetime 2025-03-10 16:17:53
  292. // @param item *rag.RagEtaReport
  293. func GenerateRagEtaReportAbstractByQuestion(item *rag.RagEtaReport, question *rag.Question, forceGenerate bool) {
  294. var err error
  295. defer func() {
  296. if err != nil {
  297. utils.FileLog.Error("文章转临时文件失败,err:%v", err)
  298. fmt.Println("文章转临时文件失败,err:", err)
  299. }
  300. }()
  301. // 内容为空,那就不需要生成摘要
  302. if item.TextContent == `` {
  303. return
  304. }
  305. abstractObj := rag.RagEtaReportAbstract{}
  306. abstractItem, err := abstractObj.GetByRagEtaReportIdAndQuestionId(item.RagEtaReportId, question.QuestionId)
  307. // 如果找到了,同时不是强制生成,那么就直接处理到知识库中
  308. if err == nil && !forceGenerate {
  309. // 摘要已经生成,不需要重复生成,只需要重新加入到向量库中
  310. ReportAbstractToKnowledge(item, abstractItem, false)
  311. return
  312. }
  313. //你现在是一名资深的期货行业分析师,请基于以下的问题进行汇总总结,如果不能正常总结出来,那么就只需要回复我:sorry
  314. questionStr := fmt.Sprintf(`%s\n%s`, `你现在是一名资深的期货行业分析师,请基于以下的问题进行汇总总结,如果不能正常总结出来,那么就只需要回复我:sorry。以下是问题:`, question.QuestionContent)
  315. //开始对话
  316. abstract, industryTags, _, tmpErr := getAnswerByContent(item.RagEtaReportId, utils.AI_ARTICLE_SOURCE_ETA_REPORT, questionStr)
  317. if tmpErr != nil {
  318. err = fmt.Errorf("LLM对话失败,Err:" + tmpErr.Error())
  319. return
  320. }
  321. // 添加问答记录
  322. //if len(addArticleChatRecordList) > 0 {
  323. // recordObj := rag.RagEtaReportChatRecord{}
  324. // err = recordObj.CreateInBatches(addArticleChatRecordList)
  325. // if err != nil {
  326. // return
  327. // }
  328. //}
  329. if abstract == `` {
  330. return
  331. }
  332. //if abstract == `sorry` || strings.Index(abstract, `根据已知信息无法回答该问题`) == 0 {
  333. // item.AbstractStatus = 2
  334. // item.ModifyTime = time.Now()
  335. // err = item.Update([]string{"AbstractStatus", "ModifyTime"})
  336. // return
  337. //}
  338. //item.AbstractStatus = 1
  339. //item.ModifyTime = time.Now()
  340. //err = item.Update([]string{"AbstractStatus", "ModifyTime"})
  341. var tagIdJsonStr string
  342. var tagNameJsonStr string
  343. // 标签ID
  344. {
  345. tagIdList := make([]int, 0)
  346. tagNameList := make([]string, 0)
  347. tagIdMap := make(map[int]bool)
  348. if abstractItem != nil && abstractItem.Tags != `` {
  349. tmpErr = json.Unmarshal([]byte(abstractItem.Tags), &tagIdList)
  350. if tmpErr != nil {
  351. utils.FileLog.Info(fmt.Sprintf("json.Unmarshal 失败,标签数据:%s,Err:%s", abstractItem.Tags, tmpErr.Error()))
  352. } else {
  353. for _, tagId := range tagIdList {
  354. tagIdMap[tagId] = true
  355. }
  356. }
  357. if abstractItem.TagsName != `` {
  358. tagNameList = strings.Split(abstractItem.TagsName, ",")
  359. }
  360. }
  361. for _, tagName := range industryTags {
  362. tagId, tmpErr := GetTagIdByName(tagName)
  363. if tmpErr != nil {
  364. utils.FileLog.Info(fmt.Sprintf("获取标签ID失败,标签名称:%s,Err:%s", tagName, tmpErr.Error()))
  365. }
  366. if _, ok := tagIdMap[tagId]; !ok {
  367. tagIdList = append(tagIdList, tagId)
  368. tagNameList = append(tagNameList, tagName)
  369. tagIdMap[tagId] = true
  370. }
  371. }
  372. //for _, tagName := range varietyTags {
  373. // tagId, tmpErr := GetTagIdByName(tagName)
  374. // if tmpErr != nil {
  375. // utils.FileLog.Info(fmt.Sprintf("获取标签ID失败,标签名称:%s,Err:%s", tagName, tmpErr.Error()))
  376. // }
  377. // if _, ok := tagIdMap[tagId]; !ok {
  378. // tagIdList = append(tagIdList, tagId)
  379. // tagIdMap[tagId] = true
  380. // }
  381. //}
  382. tagIdJsonByte, err := json.Marshal(tagIdList)
  383. if err != nil {
  384. utils.FileLog.Info(fmt.Sprintf("标签ID序列化失败,Err:%s", tmpErr.Error()))
  385. } else {
  386. tagIdJsonStr = string(tagIdJsonByte)
  387. }
  388. tagNameJsonStr = strings.Join(tagNameList, `,`)
  389. }
  390. if abstractItem == nil || abstractItem.RagEtaReportAbstractId <= 0 {
  391. abstractItem = &rag.RagEtaReportAbstract{
  392. RagEtaReportAbstractId: 0,
  393. RagEtaReportId: item.RagEtaReportId,
  394. Content: abstract,
  395. QuestionId: question.QuestionId,
  396. QuestionContent: question.QuestionContent,
  397. Version: 1,
  398. Tags: tagIdJsonStr,
  399. TagsName: tagNameJsonStr,
  400. VectorKey: "",
  401. ModifyTime: time.Now(),
  402. CreateTime: time.Now(),
  403. }
  404. err = abstractItem.Create()
  405. } else {
  406. // 添加历史记录
  407. rag.AddArticleAbstractHistoryByRagEtaReportAbstract(abstractItem)
  408. abstractItem.Content = abstract
  409. abstractItem.Version++
  410. abstractItem.ModifyTime = time.Now()
  411. abstractItem.Tags = tagIdJsonStr
  412. abstractItem.TagsName = tagNameJsonStr
  413. abstractItem.QuestionContent = question.QuestionContent
  414. err = abstractItem.Update([]string{"content", "version", "modify_time", "tags", "tags_name", "question_content"})
  415. }
  416. if err != nil {
  417. return
  418. }
  419. // 数据入ES库
  420. go AddOrEditEsRagEtaReportAbstract(abstractItem.RagEtaReportAbstractId)
  421. ReportAbstractToKnowledge(item, abstractItem, false)
  422. }
  423. // AddOrEditEsRagEtaReportAbstract
  424. // @Description: 新增/编辑微信文章摘要入ES
  425. // @author: Roc
  426. // @datetime 2025-03-13 14:13:47
  427. // @param articleAbstractId int
  428. func AddOrEditEsRagEtaReportAbstract(ragEtaReportAbstractId int) {
  429. if utils.EsRagEtaReportAbstractName == `` {
  430. return
  431. }
  432. var err error
  433. defer func() {
  434. if err != nil {
  435. utils.FileLog.Error("添加ETA报告微信信息到ES失败,err:%v", err)
  436. fmt.Println("添加ETA报告微信信息到ES失败,err:", err)
  437. }
  438. }()
  439. obj := rag.RagEtaReportAbstract{}
  440. abstractInfo, err := obj.GetById(ragEtaReportAbstractId)
  441. if err != nil {
  442. err = fmt.Errorf("获取ETA报告文章信息失败,Err:" + err.Error())
  443. return
  444. }
  445. ragEtaReportObj := rag.RagEtaReport{}
  446. articleInfo, err := ragEtaReportObj.GetById(abstractInfo.RagEtaReportAbstractId)
  447. if err != nil {
  448. err = fmt.Errorf("获取ETA报告文章信息失败,Err:" + err.Error())
  449. return
  450. }
  451. tagIdList := make([]int, 0)
  452. if abstractInfo.Tags != `` {
  453. err = json.Unmarshal([]byte(abstractInfo.Tags), &tagIdList)
  454. if err != nil {
  455. err = fmt.Errorf("报告标签ID转int失败,Err:" + err.Error())
  456. utils.FileLog.Info(fmt.Sprintf("json.Unmarshal 报告标签ID转int失败,标签数据:%s,Err:%s", abstractInfo.Tags, err.Error()))
  457. }
  458. }
  459. tagNameList := make([]string, 0)
  460. if abstractInfo.TagsName != `` {
  461. tagNameList = strings.Split(abstractInfo.TagsName, ",")
  462. }
  463. esItem := elastic.RagEtaReportAbstractItem{
  464. RagEtaReportAbstractId: abstractInfo.RagEtaReportAbstractId,
  465. RagEtaReportId: abstractInfo.RagEtaReportId,
  466. Abstract: abstractInfo.Content,
  467. QuestionId: abstractInfo.QuestionId,
  468. Version: abstractInfo.Version,
  469. VectorKey: abstractInfo.VectorKey,
  470. ModifyTime: abstractInfo.ModifyTime,
  471. CreateTime: abstractInfo.CreateTime,
  472. Title: articleInfo.Title,
  473. TagIdList: tagIdList,
  474. TagNameList: tagNameList,
  475. }
  476. err = elastic.RagEtaReportAbstractEsAddOrEdit(strconv.Itoa(abstractInfo.RagEtaReportAbstractId), esItem)
  477. }
  478. // DelEsRagEtaReportAbstract
  479. // @Description: 删除ES中的ETA报告
  480. // @author: Roc
  481. // @datetime 2025-04-21 11:08:09
  482. // @param articleAbstractId int
  483. func DelEsRagEtaReportAbstract(articleAbstractId int) {
  484. if utils.EsRagEtaReportAbstractName == `` {
  485. return
  486. }
  487. var err error
  488. defer func() {
  489. if err != nil {
  490. utils.FileLog.Error("删除ES中的ETA报告失败,err:%v", err)
  491. fmt.Println("删除ES中的ETA报告失败,err:", err)
  492. }
  493. }()
  494. err = elastic.RagEtaReportAbstractEsDel(strconv.Itoa(articleAbstractId))
  495. }
  496. // WechatArticleAbstractToKnowledge
  497. // @Description: 摘要入向量库
  498. // @author: Roc
  499. // @datetime 2025-03-10 16:14:59
  500. // @param wechatArticleItem *rag.RagEtaReport
  501. // @param abstractItem *rag.RagEtaReportAbstract
  502. func ReportAbstractToKnowledge(ragEtaReport *rag.RagEtaReport, abstractItem *rag.RagEtaReportAbstract, isReUpload bool) {
  503. if abstractItem.Content == `` {
  504. return
  505. }
  506. // 已经生成了,那就不处理了
  507. if abstractItem.VectorKey != `` && !isReUpload {
  508. return
  509. }
  510. var err error
  511. defer func() {
  512. if err != nil {
  513. utils.FileLog.Error("摘要入向量库失败,err:%v", err)
  514. fmt.Println("摘要入向量库失败,err:", err)
  515. }
  516. // 数据入ES库
  517. go AddOrEditEsRagEtaReportAbstract(abstractItem.RagEtaReportAbstractId)
  518. }()
  519. // 生成临时文件
  520. //dateDir := time.Now().Format("20060102")
  521. //uploadDir := + "./static/ai/article/" + dateDir
  522. uploadDir := "./static/ai/abstract"
  523. err = os.MkdirAll(uploadDir, utils.DIR_MOD)
  524. if err != nil {
  525. err = fmt.Errorf("存储目录创建失败,Err:" + err.Error())
  526. return
  527. }
  528. fileName := utils.MD5(fmt.Sprintf("%d_%d", utils.AI_ARTICLE_SOURCE_ETA_REPORT, ragEtaReport.RagEtaReportId)) + `.md`
  529. tmpFilePath := uploadDir + "/" + fileName
  530. err = utils.SaveToFile(abstractItem.Content, tmpFilePath)
  531. if err != nil {
  532. err = fmt.Errorf("生成临时文件失败,Err:" + err.Error())
  533. return
  534. }
  535. defer func() {
  536. os.Remove(tmpFilePath)
  537. }()
  538. knowledgeArticleName := models.BusinessConfMap[models.PrivateKnowledgeBaseName]
  539. // 上传临时文件到LLM
  540. uploadFileResp, err := llm.UploadDocsToKnowledge(tmpFilePath, knowledgeArticleName)
  541. if err != nil {
  542. err = fmt.Errorf("上传文章原文到知识库失败,Err:" + err.Error())
  543. return
  544. }
  545. if len(uploadFileResp.FailedFiles) > 0 {
  546. for _, v := range uploadFileResp.FailedFiles {
  547. err = fmt.Errorf("上传文章原文到知识库失败,Err:" + v)
  548. }
  549. }
  550. abstractItem.VectorKey = tmpFilePath
  551. abstractItem.ModifyTime = time.Now()
  552. err = abstractItem.Update([]string{"vector_key", "modify_time"})
  553. }
  554. // DelRagReportLlmDoc
  555. // @Description: 删除ETA报告的摘要向量库
  556. // @author: Roc
  557. // @datetime 2025-04-23 13:24:51
  558. // @param vectorKeyList []string
  559. // @param abstractIdList []int
  560. // @return err error
  561. func DelRagReportLlmDoc(vectorKeyList []string, abstractIdList []int) (err error) {
  562. defer func() {
  563. if err != nil {
  564. utils.FileLog.Error("删除摘要向量库文件失败,err:%v", err)
  565. fmt.Println("删除摘要向量库文件失败,err:", err)
  566. }
  567. }()
  568. // 没有就不删除
  569. if len(vectorKeyList) <= 0 {
  570. return
  571. }
  572. _, err = llm.DelDocsToKnowledge(models.BusinessConfMap[models.PrivateKnowledgeBaseName], vectorKeyList)
  573. obj := rag.RagEtaReportAbstract{}
  574. err = obj.DelVectorKey(abstractIdList)
  575. return
  576. }
  577. // DelRagEtaReportAbstract
  578. // @Description: 删除ETA报告摘要
  579. // @author: Roc
  580. // @datetime 2025-04-23 17:36:22
  581. // @param abstractIdList []int
  582. // @return err error
  583. func DelRagEtaReportAbstract(abstractIdList []int) (err error) {
  584. vectorKeyList := make([]string, 0)
  585. newAbstractIdList := make([]int, 0)
  586. obj := rag.RagEtaReportAbstract{}
  587. list, err := obj.GetByIdList(abstractIdList)
  588. if err != nil {
  589. if !utils.IsErrNoRow(err) {
  590. err = errors.New("删除向量库失败,Err:" + err.Error())
  591. } else {
  592. err = nil
  593. }
  594. return
  595. }
  596. if len(list) > 0 {
  597. for _, v := range list {
  598. // 有加入到向量库,那么就加入到待删除的向量库list中
  599. if v.VectorKey != `` {
  600. vectorKeyList = append(vectorKeyList, v.VectorKey)
  601. }
  602. newAbstractIdList = append(newAbstractIdList, v.RagEtaReportAbstractId)
  603. }
  604. }
  605. //if !req.IsSelectAll {
  606. // list, err := obj.GetByIdList(req.RagEtaReportAbstractIdList)
  607. // if err != nil {
  608. // br.Msg = "修改失败"
  609. // br.ErrMsg = "修改失败,查找问题失败,Err:" + err.Error()
  610. // if utils.IsErrNoRow(err) {
  611. // br.Msg = "问题不存在"
  612. // br.IsSendEmail = false
  613. // }
  614. // return
  615. // }
  616. // if len(list) > 0 {
  617. // for _, v := range list {
  618. // // 有加入到向量库,那么就加入到待删除的向量库list中
  619. // if v.VectorKey != `` {
  620. // vectorKeyList = append(vectorKeyList, v.VectorKey)
  621. // }
  622. // wechatArticleAbstractIdList = append(wechatArticleAbstractIdList, v.RagEtaReportAbstractId)
  623. // }
  624. // }
  625. //} else {
  626. // notIdMap := make(map[int]bool)
  627. // for _, v := range req.NotRagEtaReportAbstractIdList {
  628. // notIdMap[v] = true
  629. // }
  630. //
  631. // _, list, err := getRagEtaReportAbstractList(req.KeyWord, req.TagId, 0, 100000)
  632. // if err != nil {
  633. // br.Msg = "修改失败"
  634. // br.ErrMsg = "修改失败,查找问题失败,Err:" + err.Error()
  635. // if utils.IsErrNoRow(err) {
  636. // br.Msg = "问题不存在"
  637. // br.IsSendEmail = false
  638. // }
  639. // return
  640. // }
  641. // if len(list) > 0 {
  642. // for _, v := range list {
  643. // if notIdMap[v.RagEtaReportAbstractId] {
  644. // continue
  645. // }
  646. // // 有加入到向量库,那么就加入到待删除的向量库list中
  647. // if v.VectorKey != `` {
  648. // vectorKeyList = append(vectorKeyList, v.VectorKey)
  649. // }
  650. // wechatArticleAbstractIdList = append(wechatArticleAbstractIdList, v.RagEtaReportAbstractId)
  651. // }
  652. // }
  653. //}
  654. // 删除向量库
  655. err = DelRagReportLlmDoc(vectorKeyList, newAbstractIdList)
  656. if err != nil {
  657. err = errors.New("删除向量库失败,Err:" + err.Error())
  658. return
  659. }
  660. // 删除摘要
  661. err = obj.DelByIdList(newAbstractIdList)
  662. if err != nil {
  663. err = errors.New("删除失败,Err:" + err.Error())
  664. return
  665. }
  666. // 删除es数据
  667. for _, wechatArticleAbstractId := range newAbstractIdList {
  668. go DelEsRagEtaReportAbstract(wechatArticleAbstractId)
  669. }
  670. return
  671. }