llm_report.go 23 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789
  1. package services
  2. import (
  3. "encoding/json"
  4. "errors"
  5. "eta/eta_api/cache"
  6. "eta/eta_api/models"
  7. "eta/eta_api/models/rag"
  8. "eta/eta_api/services/elastic"
  9. "eta/eta_api/services/llm"
  10. "eta/eta_api/utils"
  11. "fmt"
  12. "golang.org/x/net/html"
  13. "golang.org/x/net/html/atom"
  14. "os"
  15. "regexp"
  16. "strconv"
  17. "strings"
  18. "time"
  19. )
  20. // ReportAddOrModifyKnowledge
  21. // @Description: ETA报告加入/修改到知识库
  22. // @author: Roc
  23. // @datetime 2025-04-07 14:41:45
  24. // @param reportId int
  25. // @param reportChapterId int
  26. func ReportAddOrModifyKnowledge(reportId, reportChapterId int) {
  27. if reportId <= 0 {
  28. return
  29. }
  30. var err error
  31. defer func() {
  32. if err != nil {
  33. //fmt.Println("ReportAddOrModifyKnowledge error:", err)
  34. utils.FileLog.Error("ReportAddOrModifyKnowledge error:", err)
  35. }
  36. }()
  37. var title, author, htmlContent string
  38. var publishTime time.Time
  39. if reportChapterId > 0 {
  40. chapterInfo, tmpErr := models.GetReportChapterInfoById(reportChapterId)
  41. if tmpErr != nil {
  42. return
  43. }
  44. title = chapterInfo.Title
  45. author = chapterInfo.Author
  46. publishTime = chapterInfo.PublishTime
  47. htmlContent = chapterInfo.Content
  48. } else {
  49. reportInfo, tmpErr := models.GetReportByReportId(reportId)
  50. if tmpErr != nil {
  51. return
  52. }
  53. title = reportInfo.Title
  54. author = reportInfo.Author
  55. publishTime = reportInfo.PublishTime
  56. htmlContent = reportInfo.Content
  57. }
  58. err = handleReportAddOrModifyKnowledge(reportId, reportChapterId, title, author, htmlContent, publishTime)
  59. return
  60. }
  61. // ReportAddOrModifyKnowledgeByReportId
  62. // @Description: ETA报告加入/修改到知识库(只传id的情况)
  63. // @author: Roc
  64. // @datetime 2025-04-07 15:41:15
  65. // @param reportId int
  66. func ReportAddOrModifyKnowledgeByReportId(reportId int) {
  67. if reportId <= 0 {
  68. return
  69. }
  70. errList := make([]string, 0)
  71. defer func() {
  72. if len(errList) > 0 {
  73. utils.FileLog.Error("ReportAddOrModifyKnowledge error,报告ID:%d:%s", reportId, strings.Join(errList, "\n"))
  74. }
  75. }()
  76. reportInfo, err := models.GetReportByReportId(reportId)
  77. if err != nil {
  78. errList = append(errList, err.Error())
  79. return
  80. }
  81. // 如果是单篇报告,那么直接处理
  82. if reportInfo.HasChapter == 0 {
  83. err = handleReportAddOrModifyKnowledge(reportId, 0, reportInfo.Title, reportInfo.Author, reportInfo.Content, reportInfo.PublishTime)
  84. if err != nil {
  85. errList = append(errList, err.Error())
  86. }
  87. return
  88. }
  89. // 章节类型的报告,需要查询出来后再处理
  90. chapterInfoList, err := models.GetPublishedChapterListByReportId(reportId)
  91. if err != nil {
  92. errList = append(errList, err.Error())
  93. return
  94. }
  95. for _, v := range chapterInfoList {
  96. err = handleReportAddOrModifyKnowledge(reportId, v.ReportChapterId, v.Title, reportInfo.Author, v.Content, v.PublishTime)
  97. if err != nil {
  98. errList = append(errList, fmt.Sprintf("第%d章:%s,异常:\n%s", v.ReportChapterId, v.Title, err.Error()))
  99. continue
  100. }
  101. }
  102. return
  103. }
  104. // handleReportAddOrModifyKnowledge
  105. // @Description: 处理ETA报告加入/修改到知识库
  106. // @author: Roc
  107. // @datetime 2025-04-07 15:33:38
  108. // @param reportId int
  109. // @param reportChapterId int
  110. // @param title string
  111. // @param author string
  112. // @param htmlContent string
  113. // @param publishTime time.Time
  114. // @return err error
  115. func handleReportAddOrModifyKnowledge(reportId, reportChapterId int, title, author, htmlContent string, publishTime time.Time) (err error) {
  116. htmlContent = html.UnescapeString(htmlContent)
  117. doc, err := html.Parse(strings.NewReader(htmlContent))
  118. if err != nil {
  119. return
  120. }
  121. // 只获取文本内容
  122. content := &strings.Builder{}
  123. getArticleContent(content, doc)
  124. textContent := content.String()
  125. textContent = regexp.MustCompile(`\n+`).ReplaceAllString(textContent, "\n")
  126. textContent = strings.Trim(textContent, "\n")
  127. publishTimeStr := `未知`
  128. if !publishTime.IsZero() {
  129. title = fmt.Sprintf("%s(%s)", title, publishTime.Format(utils.FormatMonthDayUnSpace))
  130. publishTimeStr = publishTime.Format(utils.FormatDateTime)
  131. }
  132. textContent = fmt.Sprintf("标题:%s\n发布时间:%s\n%s", title, publishTimeStr, textContent)
  133. obj := rag.RagEtaReport{}
  134. item, err := obj.GetByReportAndChapterId(reportId, reportChapterId)
  135. if err != nil && !utils.IsErrNoRow(err) {
  136. // 查询异常,且不是没找到数据的报错
  137. return
  138. }
  139. if err == nil {
  140. // 标记删除了的话,那就不处理了
  141. if item.IsDeleted == 1 {
  142. return
  143. }
  144. item.Title = title
  145. item.Author = author
  146. item.TextContent = textContent
  147. item.IsPublished = 1
  148. //item.PublishTime = publishTime
  149. item.ModifyTime = time.Now()
  150. //err = item.Update([]string{"title", "author", "text_content", "is_published", "publish_time", "modify_time"})
  151. err = item.Update([]string{"title", "author", "text_content", "is_published", "modify_time"})
  152. } else {
  153. // 无数据的时候,需要新增
  154. err = nil
  155. item = &rag.RagEtaReport{
  156. RagEtaReportId: 0,
  157. ReportId: reportId,
  158. ReportChapterId: reportChapterId,
  159. Title: title,
  160. Author: author,
  161. TextContent: textContent,
  162. VectorKey: "",
  163. IsPublished: 1,
  164. IsDeleted: 0,
  165. PublishTime: publishTime,
  166. ModifyTime: time.Now(),
  167. CreateTime: time.Now(),
  168. }
  169. err = item.Create()
  170. }
  171. cache.AddRagEtaReportLlmOpToCache(item.RagEtaReportId, 0, true)
  172. return
  173. }
  174. // ReportUnPublishedKnowledge
  175. // @Description: 知识库取消发布
  176. // @author: Roc
  177. // @datetime 2025-04-07 14:58:25
  178. // @param reportId int
  179. // @param reportChapterId int
  180. func ReportUnPublishedKnowledge(reportId, reportChapterId int) {
  181. if reportId <= 0 && reportChapterId <= 0 {
  182. return
  183. }
  184. var err error
  185. defer func() {
  186. if err != nil {
  187. //fmt.Println("ReportAddOrModifyKnowledge error:", err)
  188. utils.FileLog.Error("ReportAddOrModifyKnowledge error:", err)
  189. }
  190. }()
  191. obj := rag.RagEtaReport{}
  192. item, err := obj.GetByReportAndChapterId(reportId, reportChapterId)
  193. if err != nil && !utils.IsErrNoRow(err) {
  194. // 查询异常,且不是没找到数据的报错
  195. return
  196. }
  197. if item.RagEtaReportId > 0 {
  198. item.IsPublished = 0
  199. item.ModifyTime = time.Now()
  200. err = item.Update([]string{"is_published", "modify_time"})
  201. }
  202. return
  203. }
  204. // ReportUnPublishedKnowledgeByReportId
  205. // @Description: ETA报告取消发布同步到知识库(只传报告id的情况)
  206. // @author: Roc
  207. // @datetime 2025-04-07 15:41:15
  208. // @param reportId int
  209. func ReportUnPublishedKnowledgeByReportId(reportId int) {
  210. errList := make([]string, 0)
  211. defer func() {
  212. if len(errList) > 0 {
  213. utils.FileLog.Error("ReportUnPublishedKnowledgeByReportId error,报告ID:%d:%s", reportId, strings.Join(errList, "\n"))
  214. }
  215. }()
  216. obj := rag.RagEtaReport{}
  217. list, err := obj.GetListByCondition(``, ` AND report_id = ? `, []interface{}{reportId}, 0, 1000)
  218. if err != nil && !utils.IsErrNoRow(err) {
  219. // 查询异常,且不是没找到数据的报错
  220. return
  221. }
  222. for _, item := range list {
  223. item.IsPublished = 0
  224. item.ModifyTime = time.Now()
  225. err = item.Update([]string{"is_published", "modify_time"})
  226. if err != nil {
  227. errList = append(errList, fmt.Sprintf("第%d章:%s,异常:\n%s", item.ReportChapterId, item.Title, err.Error()))
  228. continue
  229. }
  230. // 删除摘要
  231. err = DelRagEtaReportAbstract([]int{item.RagEtaReportId})
  232. }
  233. return
  234. }
  235. func getArticleContent(content *strings.Builder, htmlContentNode *html.Node) {
  236. if htmlContentNode.Type == html.TextNode {
  237. cleanData := strings.TrimSpace(htmlContentNode.Data)
  238. if cleanData != `` && cleanData != "</p>" {
  239. content.WriteString(cleanData)
  240. }
  241. } else if htmlContentNode.Type == html.ElementNode {
  242. switch htmlContentNode.DataAtom {
  243. case atom.Ul:
  244. content.WriteString("\n")
  245. case atom.Br:
  246. // 遇到 <br> 标签时添加换行符
  247. content.WriteString("\n")
  248. case atom.P:
  249. content.WriteString("\n")
  250. }
  251. }
  252. for c := htmlContentNode.FirstChild; c != nil; c = c.NextSibling {
  253. getArticleContent(content, c)
  254. }
  255. }
  256. // GenerateRagEtaReportAbstract
  257. // @Description: 文章摘要生成(默认提示词批量生成)
  258. // @author: Roc
  259. // @datetime 2025-04-24 11:24:53
  260. // @param item *rag.RagEtaReport
  261. // @param forceGenerate bool
  262. func GenerateRagEtaReportAbstract(item *rag.RagEtaReport, forceGenerate bool) {
  263. var err error
  264. defer func() {
  265. if err != nil {
  266. utils.FileLog.Error("文章摘要生成(默认提示词批量生成)失败,err:%v", err)
  267. }
  268. }()
  269. // 内容为空,那就不需要生成摘要
  270. if item.TextContent == `` {
  271. return
  272. }
  273. questionObj := rag.Question{}
  274. questionList, err := questionObj.GetListByCondition(``, ` AND is_default = 1 `, []interface{}{}, 0, 100)
  275. if err != nil {
  276. err = fmt.Errorf("获取问题列表失败,Err:" + err.Error())
  277. return
  278. }
  279. // 没问题就不生成了
  280. if len(questionList) <= 0 {
  281. return
  282. }
  283. for _, question := range questionList {
  284. GenerateRagEtaReportAbstractByQuestion(item, question, forceGenerate)
  285. }
  286. return
  287. }
  288. // GenerateRagEtaReportAbstractByQuestion
  289. // @Description: ETA报告摘要生成(根据提示词生成)
  290. // @author: Roc
  291. // @datetime 2025-04-24 11:23:49
  292. // @param item *rag.RagEtaReport
  293. // @param question *rag.Question
  294. // @param forceGenerate bool
  295. // @return err error
  296. func GenerateRagEtaReportAbstractByQuestion(item *rag.RagEtaReport, question *rag.Question, forceGenerate bool) (err error) {
  297. defer func() {
  298. if err != nil {
  299. utils.FileLog.Error("文章摘要生成(根据提示词生成)失败,err:%v", err)
  300. }
  301. }()
  302. // 内容为空,那就不需要生成摘要
  303. if item.TextContent == `` {
  304. return
  305. }
  306. abstractObj := rag.RagEtaReportAbstract{}
  307. abstractItem, err := abstractObj.GetByRagEtaReportIdAndQuestionId(item.RagEtaReportId, question.QuestionId)
  308. // 如果找到了,同时不是强制生成,那么就直接处理到知识库中
  309. if err == nil && !forceGenerate {
  310. // 摘要已经生成,不需要重复生成,只需要重新加入到向量库中
  311. ReportAbstractToKnowledge(item, abstractItem, false)
  312. return
  313. }
  314. // 如果是没找到数据,那么就将报错置空
  315. if err != nil && utils.IsErrNoRow(err) {
  316. err = nil
  317. }
  318. //你现在是一名资深的期货行业分析师,请基于以下的问题进行汇总总结,如果不能正常总结出来,那么就只需要回复我:sorry
  319. questionStr := fmt.Sprintf(`%s\n%s`, `你现在是一名资深的期货行业分析师,请基于以下的问题进行汇总总结,如果不能正常总结出来,那么就只需要回复我:sorry。以下是问题:`, question.QuestionContent)
  320. //开始对话
  321. abstract, industryTags, tmpErr := getAnswerByContent(item.RagEtaReportId, utils.AI_ARTICLE_SOURCE_ETA_REPORT, questionStr)
  322. if tmpErr != nil {
  323. err = fmt.Errorf("LLM对话失败,Err:" + tmpErr.Error())
  324. return
  325. }
  326. // 添加问答记录
  327. //if len(addArticleChatRecordList) > 0 {
  328. // recordObj := rag.RagEtaReportChatRecord{}
  329. // err = recordObj.CreateInBatches(addArticleChatRecordList)
  330. // if err != nil {
  331. // return
  332. // }
  333. //}
  334. if abstract == `` {
  335. return
  336. }
  337. if abstract == `sorry` || strings.Index(abstract, `根据已知信息无法回答该问题`) == 0 {
  338. return
  339. }
  340. //if abstract == `sorry` || strings.Index(abstract, `根据已知信息无法回答该问题`) == 0 {
  341. // item.AbstractStatus = 2
  342. // item.ModifyTime = time.Now()
  343. // err = item.Update([]string{"AbstractStatus", "ModifyTime"})
  344. // return
  345. //}
  346. //item.AbstractStatus = 1
  347. //item.ModifyTime = time.Now()
  348. //err = item.Update([]string{"AbstractStatus", "ModifyTime"})
  349. var tagIdJsonStr string
  350. var tagNameJsonStr string
  351. // 标签ID
  352. {
  353. tagIdList := make([]int, 0)
  354. tagNameList := make([]string, 0)
  355. tagIdMap := make(map[int]bool)
  356. if abstractItem != nil && abstractItem.Tags != `` {
  357. tmpErr = json.Unmarshal([]byte(abstractItem.Tags), &tagIdList)
  358. if tmpErr != nil {
  359. utils.FileLog.Info(fmt.Sprintf("json.Unmarshal 失败,标签数据:%s,Err:%s", abstractItem.Tags, tmpErr.Error()))
  360. } else {
  361. for _, tagId := range tagIdList {
  362. tagIdMap[tagId] = true
  363. }
  364. }
  365. if abstractItem.TagsName != `` {
  366. tagNameList = strings.Split(abstractItem.TagsName, ",")
  367. }
  368. }
  369. for _, tagName := range industryTags {
  370. tagId, tmpErr := GetTagIdByName(tagName)
  371. if tmpErr != nil {
  372. utils.FileLog.Info(fmt.Sprintf("获取标签ID失败,标签名称:%s,Err:%s", tagName, tmpErr.Error()))
  373. }
  374. if _, ok := tagIdMap[tagId]; !ok {
  375. tagIdList = append(tagIdList, tagId)
  376. tagNameList = append(tagNameList, tagName)
  377. tagIdMap[tagId] = true
  378. }
  379. }
  380. //for _, tagName := range varietyTags {
  381. // tagId, tmpErr := GetTagIdByName(tagName)
  382. // if tmpErr != nil {
  383. // utils.FileLog.Info(fmt.Sprintf("获取标签ID失败,标签名称:%s,Err:%s", tagName, tmpErr.Error()))
  384. // }
  385. // if _, ok := tagIdMap[tagId]; !ok {
  386. // tagIdList = append(tagIdList, tagId)
  387. // tagIdMap[tagId] = true
  388. // }
  389. //}
  390. tagIdJsonByte, err := json.Marshal(tagIdList)
  391. if err != nil {
  392. utils.FileLog.Info(fmt.Sprintf("标签ID序列化失败,Err:%s", tmpErr.Error()))
  393. } else {
  394. tagIdJsonStr = string(tagIdJsonByte)
  395. }
  396. tagNameJsonStr = strings.Join(tagNameList, `,`)
  397. }
  398. if abstractItem == nil || abstractItem.RagEtaReportAbstractId <= 0 {
  399. abstractItem = &rag.RagEtaReportAbstract{
  400. RagEtaReportAbstractId: 0,
  401. RagEtaReportId: item.RagEtaReportId,
  402. Content: abstract,
  403. QuestionId: question.QuestionId,
  404. QuestionContent: question.QuestionContent,
  405. Version: 1,
  406. Tags: tagIdJsonStr,
  407. TagsName: tagNameJsonStr,
  408. VectorKey: "",
  409. ModifyTime: time.Now(),
  410. CreateTime: time.Now(),
  411. }
  412. err = abstractItem.Create()
  413. } else {
  414. // 添加历史记录
  415. rag.AddArticleAbstractHistoryByRagEtaReportAbstract(abstractItem)
  416. abstractItem.Content = abstract
  417. abstractItem.Version++
  418. abstractItem.ModifyTime = time.Now()
  419. abstractItem.Tags = tagIdJsonStr
  420. abstractItem.TagsName = tagNameJsonStr
  421. abstractItem.QuestionContent = question.QuestionContent
  422. err = abstractItem.Update([]string{"content", "version", "modify_time", "tags", "tags_name", "question_content"})
  423. }
  424. if err != nil {
  425. return
  426. }
  427. // 数据入ES库
  428. go AddOrEditEsRagEtaReportAbstract(abstractItem.RagEtaReportAbstractId)
  429. ReportAbstractToKnowledge(item, abstractItem, false)
  430. return
  431. }
  432. // AddOrEditEsRagEtaReportAbstract
  433. // @Description: 新增/编辑微信文章摘要入ES
  434. // @author: Roc
  435. // @datetime 2025-03-13 14:13:47
  436. // @param articleAbstractId int
  437. func AddOrEditEsRagEtaReportAbstract(ragEtaReportAbstractId int) {
  438. if utils.EsRagEtaReportAbstractName == `` {
  439. return
  440. }
  441. var err error
  442. defer func() {
  443. if err != nil {
  444. utils.FileLog.Error("添加ETA报告微信信息到ES失败,err:%v", err)
  445. fmt.Println("添加ETA报告微信信息到ES失败,err:", err)
  446. }
  447. }()
  448. obj := rag.RagEtaReportAbstract{}
  449. abstractInfo, err := obj.GetById(ragEtaReportAbstractId)
  450. if err != nil {
  451. err = fmt.Errorf("获取ETA报告文章信息失败,Err:" + err.Error())
  452. return
  453. }
  454. ragEtaReportObj := rag.RagEtaReport{}
  455. articleInfo, err := ragEtaReportObj.GetById(abstractInfo.RagEtaReportAbstractId)
  456. if err != nil {
  457. err = fmt.Errorf("获取ETA报告文章信息失败,Err:" + err.Error())
  458. return
  459. }
  460. tagIdList := make([]int, 0)
  461. if abstractInfo.Tags != `` {
  462. err = json.Unmarshal([]byte(abstractInfo.Tags), &tagIdList)
  463. if err != nil {
  464. err = fmt.Errorf("报告标签ID转int失败,Err:" + err.Error())
  465. utils.FileLog.Info(fmt.Sprintf("json.Unmarshal 报告标签ID转int失败,标签数据:%s,Err:%s", abstractInfo.Tags, err.Error()))
  466. }
  467. }
  468. tagNameList := make([]string, 0)
  469. if abstractInfo.TagsName != `` {
  470. tagNameList = strings.Split(abstractInfo.TagsName, ",")
  471. }
  472. esItem := elastic.RagEtaReportAbstractItem{
  473. RagEtaReportAbstractId: abstractInfo.RagEtaReportAbstractId,
  474. RagEtaReportId: abstractInfo.RagEtaReportId,
  475. Abstract: abstractInfo.Content,
  476. QuestionId: abstractInfo.QuestionId,
  477. Version: abstractInfo.Version,
  478. VectorKey: abstractInfo.VectorKey,
  479. ModifyTime: abstractInfo.ModifyTime,
  480. CreateTime: abstractInfo.CreateTime,
  481. Title: articleInfo.Title,
  482. TagIdList: tagIdList,
  483. TagNameList: tagNameList,
  484. }
  485. err = elastic.RagEtaReportAbstractEsAddOrEdit(strconv.Itoa(abstractInfo.RagEtaReportAbstractId), esItem)
  486. }
  487. // DelEsRagEtaReportAbstract
  488. // @Description: 删除ES中的ETA报告
  489. // @author: Roc
  490. // @datetime 2025-04-21 11:08:09
  491. // @param articleAbstractId int
  492. func DelEsRagEtaReportAbstract(articleAbstractId int) {
  493. if utils.EsRagEtaReportAbstractName == `` {
  494. return
  495. }
  496. var err error
  497. defer func() {
  498. if err != nil {
  499. utils.FileLog.Error("删除ES中的ETA报告失败,err:%v", err)
  500. fmt.Println("删除ES中的ETA报告失败,err:", err)
  501. }
  502. }()
  503. err = elastic.RagEtaReportAbstractEsDel(strconv.Itoa(articleAbstractId))
  504. }
  505. // WechatArticleAbstractToKnowledge
  506. // @Description: 摘要入向量库
  507. // @author: Roc
  508. // @datetime 2025-03-10 16:14:59
  509. // @param wechatArticleItem *rag.RagEtaReport
  510. // @param abstractItem *rag.RagEtaReportAbstract
  511. func ReportAbstractToKnowledge(ragEtaReport *rag.RagEtaReport, abstractItem *rag.RagEtaReportAbstract, isReUpload bool) {
  512. if abstractItem.Content == `` {
  513. return
  514. }
  515. // 已经生成了,那就不处理了
  516. if abstractItem.VectorKey != `` && !isReUpload {
  517. return
  518. }
  519. var err error
  520. defer func() {
  521. if err != nil {
  522. utils.FileLog.Error("摘要入向量库失败,err:%v", err)
  523. fmt.Println("摘要入向量库失败,err:", err)
  524. }
  525. // 数据入ES库
  526. go AddOrEditEsRagEtaReportAbstract(abstractItem.RagEtaReportAbstractId)
  527. }()
  528. // 生成临时文件
  529. //dateDir := time.Now().Format("20060102")
  530. //uploadDir := + "./static/ai/article/" + dateDir
  531. uploadDir := "./static/ai/abstract"
  532. err = os.MkdirAll(uploadDir, utils.DIR_MOD)
  533. if err != nil {
  534. err = fmt.Errorf("存储目录创建失败,Err:" + err.Error())
  535. return
  536. }
  537. fileName := utils.MD5(fmt.Sprintf("%d_%d", utils.AI_ARTICLE_SOURCE_ETA_REPORT, ragEtaReport.RagEtaReportId)) + `.md`
  538. tmpFilePath := uploadDir + "/" + fileName
  539. err = utils.SaveToFile(abstractItem.Content, tmpFilePath)
  540. if err != nil {
  541. err = fmt.Errorf("生成临时文件失败,Err:" + err.Error())
  542. return
  543. }
  544. defer func() {
  545. os.Remove(tmpFilePath)
  546. }()
  547. knowledgeArticleName := models.BusinessConfMap[models.PrivateKnowledgeBaseName]
  548. // 上传临时文件到LLM
  549. uploadFileResp, err := llm.UploadDocsToKnowledge(tmpFilePath, knowledgeArticleName)
  550. if err != nil {
  551. err = fmt.Errorf("上传文章原文到知识库失败,Err:" + err.Error())
  552. return
  553. }
  554. if len(uploadFileResp.FailedFiles) > 0 {
  555. for _, v := range uploadFileResp.FailedFiles {
  556. err = fmt.Errorf("上传文章原文到知识库失败,Err:" + v)
  557. }
  558. }
  559. abstractItem.VectorKey = tmpFilePath
  560. abstractItem.ModifyTime = time.Now()
  561. err = abstractItem.Update([]string{"vector_key", "modify_time"})
  562. }
  563. // DelRagReportLlmDoc
  564. // @Description: 删除ETA报告的摘要向量库
  565. // @author: Roc
  566. // @datetime 2025-04-23 13:24:51
  567. // @param vectorKeyList []string
  568. // @param abstractIdList []int
  569. // @return err error
  570. func DelRagReportLlmDoc(vectorKeyList []string, abstractIdList []int) (err error) {
  571. defer func() {
  572. if err != nil {
  573. utils.FileLog.Error("删除摘要向量库文件失败,err:%v", err)
  574. fmt.Println("删除摘要向量库文件失败,err:", err)
  575. }
  576. }()
  577. // 没有就不删除
  578. if len(vectorKeyList) <= 0 {
  579. return
  580. }
  581. _, err = llm.DelDocsToKnowledge(models.BusinessConfMap[models.PrivateKnowledgeBaseName], vectorKeyList)
  582. obj := rag.RagEtaReportAbstract{}
  583. err = obj.DelVectorKey(abstractIdList)
  584. return
  585. }
  586. // DelRagEtaReportAbstract
  587. // @Description: 删除ETA报告摘要
  588. // @author: Roc
  589. // @datetime 2025-04-23 17:36:22
  590. // @param abstractIdList []int
  591. // @return err error
  592. func DelRagEtaReportAbstract(abstractIdList []int) (err error) {
  593. obj := rag.RagEtaReportAbstract{}
  594. list, err := obj.GetByIdList(abstractIdList)
  595. if err != nil {
  596. if !utils.IsErrNoRow(err) {
  597. err = errors.New("删除向量库失败,Err:" + err.Error())
  598. } else {
  599. err = nil
  600. }
  601. return
  602. }
  603. err = delRagEtaReportAbstract(list)
  604. return
  605. }
  606. // DelRagEtaReportAbstractByQuestionId
  607. // @Description: 根据提示词ID删除ETA报告摘要
  608. // @author: Roc
  609. // @datetime 2025-04-23 17:36:22
  610. // @param abstractIdList []int
  611. // @return err error
  612. func DelRagEtaReportAbstractByQuestionId(questionId int) (err error) {
  613. obj := rag.RagEtaReportAbstract{}
  614. list, err := obj.GetListByQuestionId(questionId)
  615. if err != nil {
  616. if !utils.IsErrNoRow(err) {
  617. err = errors.New("删除向量库失败,Err:" + err.Error())
  618. } else {
  619. err = nil
  620. }
  621. return
  622. }
  623. err = delRagEtaReportAbstract(list)
  624. return
  625. }
  626. // delRagEtaReportAbstract
  627. // @Description: 删除摘要
  628. // @author: Roc
  629. // @datetime 2025-04-24 15:19:19
  630. // @param list []*rag.RagEtaReportAbstract
  631. // @return err error
  632. func delRagEtaReportAbstract(list []*rag.RagEtaReportAbstract) (err error) {
  633. obj := rag.RagEtaReportAbstract{}
  634. vectorKeyList := make([]string, 0)
  635. newAbstractIdList := make([]int, 0)
  636. if len(list) > 0 {
  637. for _, v := range list {
  638. // 有加入到向量库,那么就加入到待删除的向量库list中
  639. if v.VectorKey != `` {
  640. vectorKeyList = append(vectorKeyList, v.VectorKey)
  641. }
  642. newAbstractIdList = append(newAbstractIdList, v.RagEtaReportAbstractId)
  643. }
  644. }
  645. // 删除向量库
  646. err = DelRagReportLlmDoc(vectorKeyList, newAbstractIdList)
  647. if err != nil {
  648. err = errors.New("删除向量库失败,Err:" + err.Error())
  649. return
  650. }
  651. // 删除摘要
  652. err = obj.DelByIdList(newAbstractIdList)
  653. if err != nil {
  654. err = errors.New("删除失败,Err:" + err.Error())
  655. return
  656. }
  657. // 删除es数据
  658. for _, wechatArticleAbstractId := range newAbstractIdList {
  659. go DelEsRagEtaReportAbstract(wechatArticleAbstractId)
  660. }
  661. return
  662. }
  663. // GetDelAbstractByQuestionIdCacheKey
  664. // @Description: 获取删除微信文章/ETA报告摘要的缓存key
  665. // @author: Roc
  666. // @datetime 2025-04-24 15:44:41
  667. // @param questionId int
  668. // @return string
  669. func GetDelAbstractByQuestionIdCacheKey(questionId int) string {
  670. return fmt.Sprintf("%s%d", utils.CACHE_AI_ARTICLE_ABSTRACT_DEL, questionId)
  671. }
  672. // DelAbstractByQuestionId
  673. // @Description: 根据提示词ID删除微信文章/报告摘要
  674. // @author: Roc
  675. // @datetime 2025-04-24 15:37:28
  676. // @param questionId int
  677. func DelAbstractByQuestionId(questionId int) {
  678. cacheKey := GetDelAbstractByQuestionIdCacheKey(questionId)
  679. if !utils.Rc.SetNX(cacheKey, 1, 30*time.Minute) {
  680. utils.FileLog.Error("根据提示词删除摘要失败,提示词ID:%d,系统处理中,请稍后重试!", questionId)
  681. return
  682. }
  683. defer func() {
  684. utils.Rc.Delete(cacheKey)
  685. }()
  686. // 删除微信文章摘要
  687. err := DelWechatArticleAbstractByQuestionId(questionId)
  688. if err != nil {
  689. utils.FileLog.Error("根据提示词摘要删除微信文章摘要失败,提示词ID:%d;原因:%s", questionId, err.Error())
  690. }
  691. // 删除ETA报告摘要
  692. err = DelRagEtaReportAbstractByQuestionId(questionId)
  693. if err != nil {
  694. utils.FileLog.Error("根据提示词删除ETA报告摘要失败,提示词ID:%d;原因:%s", questionId, err.Error())
  695. }
  696. return
  697. }