Merge branch 'feature/ppt_batch_refresh'

controllers/data_manage/chart_info.go

@@ -2940,6 +2940,14 @@ func (this *EdbInfoController) BatchChartInfoRefresh() {
 		br.ErrMsg = "参数解析失败，Err:" + err.Error()
 		return
 	}
+	if req.Source == "" {
+		br.Msg = "刷新来源有误"
+		return
+	}
+	if req.PrimaryId <= 0 {
+		br.Msg = "刷新对象有误"
+		return
+	}
 
 	//获取所有的图表列表
 	_, chartInfoList, err := data_manage.GetChartInfoListByUniqueCodeSlice(req.ChartInfoCode)
@@ -2950,7 +2958,7 @@ func (this *EdbInfoController) BatchChartInfoRefresh() {
 		return
 	}
 
-	redisKey := data.GetBatchChartRefreshKey(req.Source, req.ReportId, req.ReportChapterId)
+	redisKey := data.GetBatchChartRefreshKey(req.Source, req.PrimaryId, req.SubId)
 
 	// 图表中的指标刷新
 	err, isAsync := data.BatchChartInfoRefreshV2(chartInfoList, redisKey)
@@ -3462,9 +3470,17 @@ func (this *EdbInfoController) GetBatchChartRefreshResult() {
 		br.ErrMsg = "参数解析失败，Err:" + err.Error()
 		return
 	}
+	if req.Source == "" {
+		br.Msg = "刷新来源有误"
+		return
+	}
+	if req.PrimaryId <= 0 {
+		br.Msg = "刷新对象有误"
+		return
+	}
 
 	// 刷新结果，默认是刷新完成了
-	refreshResult := data.CheckBatchChartRefreshResult(req.Source, req.ReportId, req.ReportChapterId)
+	refreshResult := data.CheckBatchChartRefreshResult(req.Source, req.PrimaryId, req.SubId)
 
 	resp := response.ChartRefreshResp{
 		RefreshResult: refreshResult,

controllers/data_manage/excel/excel_info.go

@@ -2694,6 +2694,14 @@ func (c *ExcelInfoController) BatchRefresh() {
 		br.Msg = "刷新成功"
 		return
 	}
+	if req.Source == "" {
+		br.Msg = "刷新来源有误"
+		return
+	}
+	if req.PrimaryId <= 0 {
+		br.Msg = "刷新对象有误"
+		return
+	}
 
 	// 获取表格关联的指标IDs
 	edbIds, e := excel2.GetEdbIdsFromExcelCodes(req.ExcelCodes, sysUser.AdminId, c.Lang)
@@ -2703,7 +2711,7 @@ func (c *ExcelInfoController) BatchRefresh() {
 		return
 	}
 
-	redisKey := data.GetBatchChartRefreshKey(req.Source, req.ReportId, req.ReportChapterId)
+	redisKey := data.GetBatchChartRefreshKey(req.Source, req.PrimaryId, req.SubId)
 	refreshKeys := make([]string, 0)
 	for _, v := range req.ExcelCodes {
 		refreshKeys = append(refreshKeys, fmt.Sprint(utils.HZ_CHART_LIB_EXCEL_TABLE_DETAIL, v))
@@ -2751,10 +2759,18 @@ func (c *ExcelInfoController) GetBatchChartRefreshResult() {
 		br.ErrMsg = "参数解析失败，Err:" + err.Error()
 		return
 	}
+	if req.Source == "" {
+		br.Msg = "刷新来源有误"
+		return
+	}
+	if req.PrimaryId <= 0 {
+		br.Msg = "刷新对象有误"
+		return
+	}
 
 	// 校验缓存是否存在, 存在说明还在刷新中
 	result := true
-	redisKey := excel2.GetExcelEdbBatchRefreshKey(req.Source, req.ReportId, req.ReportChapterId)
+	redisKey := excel2.GetExcelEdbBatchRefreshKey(req.Source, req.PrimaryId, req.SubId)
 	if redisKey != `` {
 		// 如果找到了key，那么就是还在更新中
 		ok := utils.Rc.IsExist(redisKey)

models/data_manage/chart_info.go

@@ -1763,10 +1763,10 @@ func GetChartInfoByClassifyIdAndName(classifyId int, chartName string) (item *Ch
 
 // BatchChartRefreshReq 批量刷新图表请求
 type BatchChartRefreshReq struct {
-	ChartInfoCode   []string `description:"图表编码数组"`
-	ReportId        int      `description:"报告id"`
-	ReportChapterId int      `description:"报告章节id"`
-	Source          string   `description:"来源，枚举值：report、english_report、smart_report"`
+	ChartInfoCode []string `description:"图表编码数组"`
+	PrimaryId     int      `description:"研报/智能研报/PPT主键"`
+	SubId         int      `description:"报告章节ID"`
+	Source        string   `description:"来源，枚举值：report、english_report、smart_report、ppt"`
 }
 
 // GetChartInfoListByUniqueCodeSlice 根据图表编码获取图表列表数据

models/data_manage/excel/excel_info.go

@@ -615,10 +615,10 @@ func SaveExcelInfoAndSheet(excelInfo *ExcelInfo, updateExcelInfoParam []string,
 
 // BatchRefreshExcelReq 批量刷新表格请求
 type BatchRefreshExcelReq struct {
-	ExcelCodes      []string `description:"表格编码"`
-	ReportId        int      `description:"报告ID"`
-	ReportChapterId int      `description:"报告章节ID"`
-	Source          string   `description:"来源，枚举值：report、english_report、smart_report"`
+	ExcelCodes []string `description:"表格编码"`
+	PrimaryId  int      `description:"研报/智能研报/PPT主键"`
+	SubId      int      `description:"报告章节ID"`
+	Source     string   `description:"来源，枚举值：report、english_report、smart_report"`
 }
 
 // GetExcelMaxSortByClassifyId 获取当前分类下，且排序数最大的excel

services/data/chart_info.go

@@ -1350,7 +1350,22 @@ func BatchChartInfoRefreshV2(chartInfoList []*data_manage.ChartInfo, redisKey st
 	if totalEdbInfo > 20 { // 关联指标过多的时候，异步刷新
 		isAsync = true
 		isDeleteCache = false // 不删除缓存
-		go refreshChartEdbInfo(chartInfoList, redisKey, newBaseEdbInfoArr, newBasePredictEdbInfoArr, newCalculateMap, newPredictCalculateMap, calculateArr, predictCalculateArr)
+		//go refreshChartEdbInfo(chartInfoList, redisKey, newBaseEdbInfoArr, newBasePredictEdbInfoArr, newCalculateMap, newPredictCalculateMap, calculateArr, predictCalculateArr)
+
+		// 加入队列刷新
+		var refreshItem EdbQueueRefreshReq
+		refreshItem.RefreshKey = redisKey
+		for _, v := range chartInfoList {
+			key := utils.HZ_CHART_LIB_DETAIL + v.UniqueCode
+			refreshItem.ItemRefreshKeys = append(refreshItem.ItemRefreshKeys, key)
+		}
+		refreshItem.BaseEdbInfoArr = newBaseEdbInfoArr
+		refreshItem.BasePredictEdbInfoArr = newBasePredictEdbInfoArr
+		refreshItem.CalculateMap = newCalculateMap
+		refreshItem.PredictCalculateMap = newPredictCalculateMap
+		refreshItem.CalculateArr = calculateArr
+		refreshItem.PredictCalculateArr = predictCalculateArr
+		go PushEdb2Refresh(refreshItem)
 	} else {
 		err = edbInfoRefreshAll(false, newBaseEdbInfoArr, newBasePredictEdbInfoArr, newCalculateMap, newPredictCalculateMap, calculateArr, predictCalculateArr)
 
@@ -1395,15 +1410,15 @@ func refreshChartEdbInfo(chartInfoList []*data_manage.ChartInfo, redisKey string
 // @author: Roc
 // @datetime 2023-11-30 13:30:26
 // @param source string
-// @param reportId int
-// @param reportChapterId int
+// @param primaryId int
+// @param subId int
 // @return string
-func GetBatchChartRefreshKey(source string, reportId, reportChapterId int) string {
+func GetBatchChartRefreshKey(source string, primaryId, subId int) string {
 	if source == `` {
 		return ``
 	}
 
-	return fmt.Sprint("batch_chart_refresh:", source, ":", reportId, ":", reportChapterId)
+	return fmt.Sprint("batch_chart_refresh:", source, ":", primaryId, ":", subId)
 }
 
 // CheckBatchChartRefreshResult
@@ -1411,12 +1426,12 @@ func GetBatchChartRefreshKey(source string, reportId, reportChapterId int) strin
 // @author: Roc
 // @datetime 2023-11-30 13:30:26
 // @param source string
-// @param reportId int
-// @param reportChapterId int
+// @param primaryId int
+// @param subId int
 // @return string
-func CheckBatchChartRefreshResult(source string, reportId, reportChapterId int) (refreshResult bool) {
+func CheckBatchChartRefreshResult(source string, primaryId, subId int) (refreshResult bool) {
 	refreshResult = true
-	redisKey := GetBatchChartRefreshKey(source, reportId, reportChapterId)
+	redisKey := GetBatchChartRefreshKey(source, primaryId, subId)
 	if redisKey != `` {
 		// 如果找到了key，那么就是还在更新中
 		isOk := utils.Rc.IsExist(redisKey)

services/data/edb_info.go

@@ -2619,24 +2619,36 @@ func BatchRefreshEdbByEdbIds(edbIdList []int, redisKey string, refreshKeys []str
 	if totalEdbInfo > 20 {
 		syncing = true
 
-		go func() {
-			defer func() {
-				if err != nil {
-					alarm_msg.SendAlarmMsg("BatchRefreshEdbByEdbIds, ErrMsg: "+err.Error(), 3)
-				}
-			}()
-
-			err = edbInfoRefreshAll(false, newBaseEdbInfoArr, newBasePredictEdbInfoArr, newCalculateMap, newPredictCalculateMap, calculateArr, predictCalculateArr)
-
-			if redisKey != `` {
-				_ = utils.Rc.Delete(redisKey)
-			}
-			if len(refreshKeys) > 0 {
-				for _, v := range refreshKeys {
-					_ = utils.Rc.Delete(v)
-				}
-			}
-		}()
+		//go func() {
+		//	defer func() {
+		//		if err != nil {
+		//			alarm_msg.SendAlarmMsg("BatchRefreshEdbByEdbIds, ErrMsg: "+err.Error(), 3)
+		//		}
+		//	}()
+		//
+		//	err = edbInfoRefreshAll(false, newBaseEdbInfoArr, newBasePredictEdbInfoArr, newCalculateMap, newPredictCalculateMap, calculateArr, predictCalculateArr)
+		//
+		//	if redisKey != `` {
+		//		_ = utils.Rc.Delete(redisKey)
+		//	}
+		//	if len(refreshKeys) > 0 {
+		//		for _, v := range refreshKeys {
+		//			_ = utils.Rc.Delete(v)
+		//		}
+		//	}
+		//}()
+
+		// 加入队列刷新
+		var refreshItem EdbQueueRefreshReq
+		refreshItem.RefreshKey = redisKey
+		refreshItem.ItemRefreshKeys = refreshKeys
+		refreshItem.BaseEdbInfoArr = newBaseEdbInfoArr
+		refreshItem.BasePredictEdbInfoArr = newBasePredictEdbInfoArr
+		refreshItem.CalculateMap = newCalculateMap
+		refreshItem.PredictCalculateMap = newPredictCalculateMap
+		refreshItem.CalculateArr = calculateArr
+		refreshItem.PredictCalculateArr = predictCalculateArr
+		go PushEdb2Refresh(refreshItem)
 		return
 	}
 

services/data/edb_refresh.go

@@ -0,0 +1,105 @@
+package data
+
+import (
+	"eta/eta_api/models/data_manage"
+	"eta/eta_api/services/alarm_msg"
+	"eta/eta_api/utils"
+	"fmt"
+	"sync"
+)
+
+var (
+	EdbRefreshLock     sync.Map                // 指标刷新锁, 避免同时间重复key的刷新
+	EdbRefreshHandling chan EdbQueueRefreshReq // 排队刷新的20+指标的图表/表格
+	EdbRefreshWorker   chan struct{}           // 同时允许N组刷新
+)
+
+func init() {
+	EdbRefreshHandling = make(chan EdbQueueRefreshReq, 50) // N为最大排队等候的goroutine数
+	EdbRefreshWorker = make(chan struct{}, 5)              // 控制刷新速率修改此N值
+}
+
+type EdbQueueRefreshReq struct {
+	RefreshKey            string   `description:"刷新请求的缓存key"`
+	ItemRefreshKeys       []string `description:"图表/表格刷新后要删除的缓存key"`
+	BaseEdbInfoArr        []*data_manage.EdbInfo
+	BasePredictEdbInfoArr []*data_manage.EdbInfo
+	CalculateMap          map[int]*data_manage.EdbInfo
+	PredictCalculateMap   map[int]*data_manage.EdbInfo
+	CalculateArr          []int
+	PredictCalculateArr   []int
+}
+
+// PushEdb2Refresh 写入指标刷新队列
+func PushEdb2Refresh(item EdbQueueRefreshReq) {
+	refreshKey := item.RefreshKey
+	if refreshKey == "" {
+		return
+	}
+
+	// 检查是否已在队列中
+	_, ok := EdbRefreshLock.Load(refreshKey)
+	if ok {
+		return
+	}
+
+	// 追加至刷新队列
+	EdbRefreshHandling <- item
+
+	EdbRefreshLock.Store(refreshKey, true)
+}
+
+// HandleEdbRefreshQueue 排队刷新指标
+func HandleEdbRefreshQueue() {
+	defer func() {
+		if err := recover(); err != nil {
+			tips := fmt.Sprintf("[HandleEdbRefreshQueue] panic: %v", err)
+			fmt.Println(tips)
+			utils.FileLog.Info(tips)
+		}
+	}()
+	fmt.Println("HandleEdbRefreshQueue start")
+
+	for {
+		select {
+		case item, ok := <-EdbRefreshHandling:
+			if !ok {
+				return
+			}
+			go EdbRefreshTask(item)
+		}
+	}
+}
+
+// EdbRefreshTask 指标刷新任务
+func EdbRefreshTask(item EdbQueueRefreshReq) {
+	var err error
+	defer func() {
+		if err != nil {
+			tips := fmt.Sprintf("[EdbRefreshTask] ErrMsg: %s", err.Error())
+			utils.FileLog.Info(tips)
+			go alarm_msg.SendAlarmMsg(tips, 3)
+		}
+		<-EdbRefreshWorker
+	}()
+	EdbRefreshWorker <- struct{}{}
+
+	fmt.Printf("EdbRefreshTask开始刷新: %s\n", item.RefreshKey)
+	// 刷新指标
+	e := edbInfoRefreshAll(false, item.BaseEdbInfoArr, item.BasePredictEdbInfoArr, item.CalculateMap, item.PredictCalculateMap, item.CalculateArr, item.PredictCalculateArr)
+	if e != nil {
+		err = fmt.Errorf("edbInfoRefreshAll err: %s", e.Error())
+	}
+
+	// 清除缓存
+	_ = utils.Rc.Delete(item.RefreshKey)
+	if len(item.ItemRefreshKeys) > 0 {
+		for _, v := range item.ItemRefreshKeys {
+			_ = utils.Rc.Delete(v)
+		}
+	}
+
+	// 解除锁
+	EdbRefreshLock.Delete(item.RefreshKey)
+	fmt.Printf("EdbRefreshTask结束刷新: %s\n", item.RefreshKey)
+}

services/data/excel/excel_info.go

@@ -520,10 +520,10 @@ func GetEdbIdsFromExcelCodes(excelCodes []string, sysUserId int, lang string) (e
 }
 
 // GetExcelEdbBatchRefreshKey 获取批量刷新表格指标缓存key
-func GetExcelEdbBatchRefreshKey(source string, reportId, chapterId int) string {
+func GetExcelEdbBatchRefreshKey(source string, primaryId, subId int) string {
 	if source == `` {
 		return ``
 	}
 
-	return fmt.Sprint("batch_refresh_excel_edb:", source, ":", reportId, ":", chapterId)
+	return fmt.Sprint("batch_refresh_excel_edb:", source, ":", primaryId, ":", subId)
 }

services/task.go

@@ -40,8 +40,8 @@ func Task() {
 
 	go AutoInsertAdminOperateRecordToDB()
 
-	// 测试用-生成bloomberg数据
-	//go InsertBloombergIndex()
+	// 指标刷新
+	go data.HandleEdbRefreshQueue()
 
 	// TODO:修复权限
 	//FixEnCompanyPermission()