Merge branch 'feature/yb11.1_rai_report' into debug

# Conflicts:
#	models/classify.go
#	models/report.go
#	utils/constants.go

models/classify.go

@@ -767,3 +767,8 @@ func GetAllEnabledClassify() (list []*Classify, err error) {
 
 	return
 }
+func GetReportClassifyByClassifyName(classifyNameList []string) (items []*Classify, err error) {
+	sql := `SELECT * FROM classify WHERE classify_name IN (?) and enabled = 1`
+	err = global.DbMap[utils.DbNameReport].Raw(sql, classifyNameList).Find(&items).Error
+	return items, err
+}

models/report.go

@@ -91,6 +91,7 @@ type Report struct {
 	ReportCreateTime    time.Time `description:"报告时间创建时间"`
 	InheritReportId     int       `description:"待继承的报告ID"`
 	VoiceGenerateType   int       `description:"音频生成方式，0：系统生成，1：人工上传"`
+	RaiReportId         int       `description:"RAI报告ID"`
 }
 
 func (m *Report) AfterFind(db *gorm.DB) (err error) {
@@ -169,6 +170,7 @@ type ReportList struct {
 	ClassifyNameThird   string    `description:"三级分类名称"`
 	InheritReportId     int       `description:"待继承的报告ID"`
 	ClassifyEnabled     bool      `description:"分类是否禁用"`
+	RaiReportId         int       `description:"RAI报告ID"`
 }
 
 func (m *ReportList) AfterFind(db *gorm.DB) (err error) {
@@ -446,6 +448,7 @@ type ReportDetail struct {
 	ReportLayout        int8      `description:"报告布局，1：常规布局，2：智能布局。默认：1"`
 	IsPublicPublish     int8      `description:"是否公开发布，1：是，2：否"`
 	ReportCreateTime    time.Time `description:"报告时间创建时间"`
+	RaiReportId         int       `description:"RAI报告ID"`
 }
 
 func (m *ReportDetail) AfterFind(db *gorm.DB) (err error) {

models/report_rai.go

@@ -0,0 +1,104 @@
+package models
+
+import (
+	"eta/eta_api/global"
+	"eta/eta_api/utils"
+	"time"
+)
+
+type RaiReportNotifyRedis struct {
+	ArticleId  int       `description:"文章ID"`
+	Action     string    `description:"日志类型：add，edit，move"`
+	CreateTime time.Time `description:"创建时间"`
+}
+
+type ArticleDetailResultApi struct {
+	Data ArticleResultApidate `json:"data"`
+	Code int                  `json:"code"`
+	Msg  string               `json:"msg"`
+}
+
+type ArticleResultApidate struct {
+	ArticleId     int                      `json:"id"`
+	Title         string                   `json:"title"`
+	File          string                   `json:"file"`
+	TitleEn       string                   `json:"title_en"`
+	Frequency     string                   `json:"frequency"`
+	CreateDate    string                   `json:"create_date"`
+	UpdateDate    string                   `json:"update_date"`
+	PublishDate   time.Time                `json:"publish_date"`
+	PublishStatus int                      `json:"publish_status" description:"发布状态: 0未发布，2已发布，4已发布"`
+	IndustrId     int                      `json:"industry_id"`
+	SeriesId      int                      `json:"series_id"`
+	Series        ArticleSeries            `json:"series"`
+	Content       ArticleResultApiContent  `json:"content"`
+	Author        ArticleResultApiAuthor   `json:"author"`
+	Industry      ArticleResultApiIndustry `json:"industry"`
+	Type          ArticleResultApiType     `json:"type"`
+	Stock         []string                 `json:"stock"`
+	Field         ArticleField             `json:"field"`
+	Corpus        Corpus                   `json:"corpus"`
+	Cover         string                   `json:"cover"`
+	TypeId        int                      `json:"type_id"`
+	IsActive      bool                     `json:"is_active" description:"是否有效: 0无效，1有效"`
+	PublishArea   string                   `json:"publish_area"`
+}
+
+type ArticleField struct {
+	Id          int    `json:"id"`
+	Name        string `json:"name"`
+	Description string `json:"description"`
+	IndustryId  int    `json:"industry_id"`
+}
+
+type Corpus struct {
+	Id        int    `json:"id"`
+	ArticleId int    `json:"article_id"`
+	Corpus    string `json:"corpus"`
+}
+
+type ArticleSeries struct {
+	Name string `json:"name"`
+}
+type ArticleResultApiContent struct {
+	ArticleId  int    `json:"id"`
+	Body       string `json:"body"`
+	Abstract   string `json:"abstract"`
+	Annotation string `json:"annotation"`
+}
+
+type ArticleResultApiAuthor struct {
+	PhoneNumber string `json:"phone_number"`
+	Name        string `json:"name"`
+}
+
+type ArticleResultApiIndustry struct {
+	Name string `json:"name"`
+}
+
+type ArticleResultApiType struct {
+	Name string `json:"name"`
+}
+
+type ArticleIndustryApi struct {
+	Data []ArticleResultApiIndustrdate `json:"data"`
+	Code int                           `json:"code"`
+	Msg  string                        `json:"msg"`
+}
+
+type ArticleResultApiIndustrdate struct {
+	Id     int                          `json:"id"`
+	Name   string                       `json:"name"`
+	Series []ArticleResultApiSeriesdate `json:"series"`
+}
+
+type ArticleResultApiSeriesdate struct {
+	Id   int    `json:"id"`
+	Name string `json:"name"`
+}
+
+func GetReportByRaiReportId(raiReportId int) (report *Report, err error) {
+	report = new(Report)
+	err = global.DbMap[utils.DbNameReport].Model(&Report{}).Where("rai_report_id = ?", raiReportId).First(report).Error
+	return
+}

services/report_rai.go

@@ -0,0 +1,304 @@
+package services
+
+import (
+	"encoding/json"
+	"eta/eta_api/models"
+	"eta/eta_api/models/report"
+	"eta/eta_api/services/alarm_msg"
+	"eta/eta_api/utils"
+	"fmt"
+	"html"
+	"io/ioutil"
+	"net/http"
+	"strconv"
+	"strings"
+	"time"
+)
+
+func AutoInsertRaiReport() {
+	for {
+		utils.Rc.Brpop(utils.FICC_ARTICLE_UPDATE_KEY, func(b []byte) {
+			var log models.RaiReportNotifyRedis
+			if err := json.Unmarshal(b, &log); err != nil {
+				utils.FileLog.Info("获取权益报告并更新处理Redis队列消息失败： json unmarshal wrong!", err.Error())
+				go alarm_msg.SendAlarmMsg(fmt.Sprintf("获取权益报告并更新处理Redis队列消息失败： json unmarshal wrong! %s", err.Error()), 2)
+			}
+			// 这里直接go出去会出现并发，导致文章md5ID唯一索引限制报错
+			err := HandleInsertRaiReport(log.ArticleId)
+			if err != nil {
+				utils.FileLog.Info("获取权益报告并更新处理Redis队列消息失败： HandleInsertRaiReport ", err.Error())
+				go alarm_msg.SendAlarmMsg(fmt.Sprintf("获取权益报告并更新处理Redis队列消息失败： HandleInsertRaiReport %s", err.Error()), 2)
+			}
+		})
+	}
+}
+
+func HandleInsertRaiReport(raiReportId int) (err error) {
+	// 设置缓存，防止重复处理
+	cacheKey := fmt.Sprintf("rai_report_notify_redis_%d", raiReportId)
+	cacheValue := utils.Rc.GetStr(cacheKey)
+	if cacheValue != "" {
+		return nil
+	}
+	utils.Rc.SetNX(cacheKey, "1", 10*time.Second)
+	defer func() {
+		if err != nil {
+			msg := fmt.Sprintf("处理同步过来的文章失败"+"HandleArticleListByApi ErrMsg:%s artcleId：%d", err.Error(), raiReportId)
+			utils.FileLog.Info(msg, 2)
+			go alarm_msg.SendAlarmMsg(msg, 2)
+		}
+		utils.Rc.Delete(cacheKey)
+	}()
+	var clueApiUrl string
+	clueApiUrl = fmt.Sprint(utils.RaiReportLibUrl, "articles/", raiReportId)
+	fmt.Println(clueApiUrl)
+
+	body, err := getRaiReportLib(clueApiUrl)
+	if err != nil {
+		fmt.Println(err)
+		return
+	}
+	var articleResultDate models.ArticleDetailResultApi
+	err = json.Unmarshal(body, &articleResultDate)
+	if err != nil {
+		fmt.Println("Getres.PublicGetDate Err:", err.Error())
+		return err
+	}
+	
+	articleResult := articleResultDate.Data
+	// 判断是否是固收研究
+	if articleResult.IndustrId != 12 {
+		return nil
+	}
+	// 根据分类名称查找分类信息
+	classifyItemList, e := models.GetReportClassifyByClassifyName([]string{articleResult.Industry.Name, articleResult.Series.Name})
+	if e != nil {
+		err = fmt.Errorf("GetReportClassifyByClassifyName err: %s", e.Error())
+		return err
+	}
+	classifyMap := make(map[string]*models.Classify)
+	for _, v := range classifyItemList {
+		classifyMap[v.ClassifyName] = v
+	}
+	classifyFirst, ok := classifyMap[articleResult.Industry.Name]
+	if !ok {
+		err = fmt.Errorf("一级分类不存在")
+		return err
+	}
+	classifySecond, ok := classifyMap[articleResult.Series.Name]
+	if !ok {
+		// 新增二级分类
+		err, _, _ = AddReportClassify(articleResult.Series.Name, classifyFirst.Id, []int{})
+		if err != nil {
+			err = fmt.Errorf("添加二级分类失败, Err: %s", err.Error())
+			return err
+		}
+		item, err := models.GetClassifyByName(articleResult.Series.Name, classifyFirst.Id)
+		if err != nil {
+			err = fmt.Errorf("添加二级分类失败, Err: %s", err.Error())
+			return err
+		}
+		classifySecond = item
+	}
+	// 判断分类的层级关系是否合理
+	if classifyFirst.Id != classifySecond.ParentId {
+		err = fmt.Errorf("分类层级关系不合理")
+		return err
+	}
+	// 判断报告是否已存在， 如果存在则更新报告，如果不存在则创建报告
+	reportInfo, err := models.GetReportByRaiReportId(articleResult.ArticleId)
+	if err != nil && err.Error() != utils.ErrNoRow() {
+		return err
+	}
+	if err == nil && reportInfo.Id > 0 {
+		var contentSub string
+		if articleResult.Content.Body != "" {
+			contentSub, err = GetReportContentSub(articleResult.Content.Body)
+			if err != nil {
+				go alarm_msg.SendAlarmMsg("ContentSub 失败，Err:"+err.Error(), 3)
+			}
+		}
+		state := reportInfo.State
+		// 报告已存在，更新报告
+		if (articleResult.PublishStatus == 2 || articleResult.PublishStatus == 4) && articleResult.IsActive {
+			// 报告状态为未发布，则更新报告
+			state = models.ReportStatePublished
+			reportInfo.PublishTime = articleResult.PublishDate
+		}else if articleResult.IsActive == false {
+			// 删除报告
+			err = models.DeleteReport(reportInfo.Id)
+			if err != nil {
+				err = fmt.Errorf("删除报告失败, Err: %s", err.Error())
+				return
+			}
+			go UpdateReportEs(reportInfo.Id, 1)
+		}else {
+			// 报告状态为未发布，则更新报告
+			state = models.ReportStateUnpublished
+			reportInfo.PublishTime = articleResult.PublishDate
+			
+		}
+		// 过滤Abstracthtml标签，把<p>标签去掉
+		abstract := strings.ReplaceAll(articleResult.Content.Abstract, "<p>", "")
+		abstract = strings.ReplaceAll(abstract, "</p>", "")
+
+		reportInfo.ClassifyIdFirst = classifyFirst.Id
+		reportInfo.ClassifyNameFirst = articleResult.Industry.Name
+		reportInfo.ClassifyIdSecond = classifySecond.Id
+		reportInfo.ClassifyNameSecond = articleResult.Series.Name
+		reportInfo.Title = articleResult.Title
+		reportInfo.Abstract = abstract
+		reportInfo.Author = articleResult.Author.Name
+		reportInfo.Frequency = articleResult.Frequency
+		reportInfo.State = state
+		reportInfo.Content = html.EscapeString(articleResult.Content.Body)
+		reportInfo.ContentSub = html.EscapeString(contentSub)
+		updateTime, _ := time.ParseInLocation(utils.FormatDate, articleResult.UpdateDate, time.Local)
+		reportInfo.ModifyTime = updateTime
+
+		// 报告更新
+		updateCols := []string{"ClassifyIdFirst","ClassifyNameFirst","ClassifyIdSecond","ClassifyNameSecond","Title","Abstract","Author","Frequency","State","Content","ContentSub","ModifyTime","PublishTime"}
+		err = reportInfo.UpdateReport(updateCols)
+		if err != nil {
+			err = fmt.Errorf("更新报告失败, Err: %s", err.Error())
+			return
+		}
+		go UpdateReportEs(reportInfo.Id, state)
+		if state == models.ReportStatePublished {
+			// 报告权限处理
+			go handleReportPermission(int64(reportInfo.Id), reportInfo.ClassifyIdSecond)
+		}else {
+			// 重置小程序详情页海报
+			_ = ResetMiniProgramReportDetailCover(reportInfo.Id)
+		}
+
+	}else if reportInfo.Id == 0 {
+		// 报告不存在，创建报告
+		// 判断状态
+		if (articleResult.PublishStatus == 2 || articleResult.PublishStatus == 4) && articleResult.IsActive {
+			var contentSub string
+			if articleResult.Content.Body != "" {
+				contentSub, err = GetReportContentSub(articleResult.Content.Body)
+				if err != nil {
+					go alarm_msg.SendAlarmMsg("ContentSub 失败，Err:"+err.Error(), 3)
+				}
+			}
+
+			// 已发布状态
+			state :=  models.ReportStatePublished
+
+			// 协作方式，1：个人，2：多人协作。默认：1
+			collaborateType := 1
+	
+			// 报告布局，1：常规布局，2：智能布局。默认：1
+			reportLayout := 1
+	
+			// 是否公开发布，1：是，2：否
+			isPublicPublish := 1
+	
+			abstract := strings.ReplaceAll(articleResult.Content.Abstract, "<p>", "")
+			abstract = strings.ReplaceAll(abstract, "</p>", "")
+
+			item := new(models.Report)
+			item.AddType = 	1
+			item.ReportVersion = 2
+			item.ClassifyIdFirst = classifyFirst.Id
+			item.ClassifyNameFirst = articleResult.Industry.Name
+			item.ClassifyIdSecond = classifySecond.Id
+			item.ClassifyNameSecond = articleResult.Series.Name
+			item.Title = articleResult.Title
+			item.Abstract = abstract
+			item.Author = articleResult.Author.Name
+			item.Frequency = articleResult.Frequency
+			item.State = state
+			item.Content = html.EscapeString(articleResult.Content.Body)
+			item.Stage = 0
+			item.ContentSub = html.EscapeString(contentSub)
+			item.CreateTime = time.Now().Format(utils.FormatDateTime)
+			updateTime, _ := time.ParseInLocation(utils.FormatDate, articleResult.UpdateDate, time.Local)
+			item.ModifyTime = updateTime
+			item.ReportVersion = 2
+			item.AdminId = 0
+			item.AdminRealName = ""
+			item.PublishTime = articleResult.PublishDate
+
+			item.ClassifyIdThird = 0
+			item.ClassifyNameThird = ""
+
+			item.LastModifyAdminId = 0
+			item.LastModifyAdminName = ""
+			item.ContentModifyTime = time.Now()
+			item.NeedSplice = 1
+			item.ContentStruct = ""
+			item.HeadImg = ""
+			item.EndImg = ""
+			item.CanvasColor = ""
+			item.HeadResourceId = 0
+			item.EndResourceId = 0
+			item.CollaborateType = int8(collaborateType)
+			item.ReportLayout = int8(reportLayout)
+			item.IsPublicPublish = int8(isPublicPublish)
+			createTime, _ := time.ParseInLocation(utils.FormatDate, articleResult.CreateDate, time.Local)
+			item.ReportCreateTime = createTime
+			item.RaiReportId = articleResult.ArticleId
+			// 新增报告及章节
+			var reportId int64
+			allGrantUserList := make([]*report.ReportGrant, 0)
+			reportId, err = models.AddReportAndChapter(item, allGrantUserList, []models.AddReportChapter{})
+			if err != nil {
+				err = fmt.Errorf("新增报告及章节失败, Err: " + err.Error())
+				return
+			}
+			reportCode := utils.MD5(strconv.Itoa(int(reportId)))
+			item.ReportCode = reportCode
+
+			// 修改唯一编码
+			{
+				go models.ModifyReportCode(reportId, reportCode)
+			}
+
+
+			// 报告权限处理
+			go handleReportPermission(reportId, item.ClassifyIdSecond)
+
+			// 更新报告Es
+			_ = UpdateReportEs(int(reportId), 2)
+		}
+	}
+
+	return
+}
+
+// get公共请求方法
+func getRaiReportLib(url string) (body []byte, err error) {
+	if url == "" {
+		err = fmt.Errorf("url is empty")
+		return
+	}
+	if utils.RaiReportLibAuthorization == "" {
+		err = fmt.Errorf("authorization is empty")
+		return
+	}
+	defer func() {
+		if err != nil {
+			go utils.SendEmail(utils.APPNAME+"【"+utils.RunMode+"】"+"失败提醒", url+"Get ErrMsg:"+err.Error(), utils.EmailSendToUsers)
+		}
+	}()
+	method := "GET"
+	client := &http.Client{}
+	req, err := http.NewRequest(method, url, nil)
+	if err != nil {
+		return
+	}
+	req.Header.Add("Authorization", utils.RaiReportLibAuthorization)
+	res, err := client.Do(req)
+	if err != nil {
+		return
+	}
+	defer res.Body.Close()
+	body, err = ioutil.ReadAll(res.Body) 
+	if err != nil {
+		return
+	}
+	return
+}

services/task.go

@@ -74,6 +74,9 @@ func Task() {
 	// 定时任务进行微信文章LLM操作
 	go HandleWechatArticleLLmOp()
 
+	// 权益报告监听入库
+	go AutoInsertRaiReport()
+
 	// TODO:数据修复
 	//FixNewEs()
 	fmt.Println("task end")

utils/config.go

@@ -301,6 +301,11 @@ var (
 	CommandPython string // python命令
 )
 
+var (
+	RaiReportLibUrl string // 权益报告库地址
+	RaiReportLibAuthorization string // 权益报告库鉴权
+)
+
 func init() {
 	tmpRunMode, err := web.AppConfig.String("run_mode")
 	if err != nil {
@@ -672,4 +677,7 @@ func init() {
 		CommandPython = "python3"
 	}
 	fmt.Println(CommandPython)
+
+	RaiReportLibUrl = config["rai_report_lib_url"]
+	RaiReportLibAuthorization = config["rai_report_lib_authorization"]
 }

utils/constants.go

@@ -591,3 +591,6 @@ const (
 const (
 	DATA_SOURCE_NAME_RADISH_RESEARCH = "萝卜投研" // 萝卜投研 -> 105
 )
+const (
+	FICC_ARTICLE_UPDATE_KEY = "FICC_ARTICLE_UPDATE_KEY" //权益报告通知给FICC这边的缓存key
+)