Merge branch 'feature/excel_refresh_1215'

controllers/data_manage/excel/excel_info.go

@@ -1804,6 +1804,9 @@ func (c *ExcelInfoController) GetHistoryDateData() {
 func (c *ExcelInfoController) Refresh() {
 	br := new(models.BaseResponse).Init()
 	defer func() {
+		if br.ErrMsg == "" {
+			br.IsSendEmail = false
+		}
 		c.Data["json"] = br
 		c.ServeJSON()
 	}()
@@ -1844,7 +1847,7 @@ func (c *ExcelInfoController) Refresh() {
 	}
 
 	// 数据刷新（只有自定义表格有刷新）
-	if excelDetail.Source == 2 {
+	if excelDetail.Source == utils.TIME_TABLE {
 		jsonStrByte, err := json.Marshal(excelDetail.TableData)
 		if err != nil {
 			br.Msg = "自定义表格数据获取失败"
@@ -1869,6 +1872,48 @@ func (c *ExcelInfoController) Refresh() {
 		}
 	}
 
+	// 数据刷新-混合表格
+	if excelDetail.Source == utils.MIXED_TABLE {
+		jsonByte, e := json.Marshal(excelDetail.TableData)
+		if e != nil {
+			br.Msg = "刷新失败"
+			br.ErrMsg = "JSON格式化混合表格数据失败, Err: " + e.Error()
+			return
+		}
+		var tableData request.MixedTableReq
+		if e = json.Unmarshal(jsonByte, &tableData); e != nil {
+			br.Msg = "刷新失败"
+			br.ErrMsg = "解析混合表格数据失败, Err: " + e.Error()
+			return
+		}
+		edbInfoIds := make([]int, 0)
+		edbInfoIdExist := make(map[int]bool)
+		if len(tableData.Data) > 0 {
+			for _, t := range tableData.Data {
+				for _, v := range t {
+					if v.EdbInfoId > 0 && !edbInfoIdExist[v.EdbInfoId] {
+						edbInfoIdExist[v.EdbInfoId] = true
+						edbInfoIds = append(edbInfoIds, v.EdbInfoId)
+					}
+				}
+			}
+		}
+		if len(edbInfoIds) > 0 {
+			e, _ = data.EdbInfoRefreshAllFromBaseV3(edbInfoIds, false, true, false)
+			if e != nil {
+				br.Msg = "刷新失败"
+				br.ErrMsg = "刷新混合表格数据失败, Err: " + err.Error()
+				return
+			}
+		}
+	}
+
+	// 清除缓存
+	key := utils.HZ_CHART_LIB_EXCEL_TABLE_DETAIL + ":" + excelDetail.UniqueCode
+	if utils.Re == nil {
+		_ = utils.Rc.Delete(key)
+	}
+
 	br.Ret = 200
 	br.Success = true
 	br.Msg = "刷新成功"
@@ -2244,3 +2289,117 @@ func (this *ExcelInfoController) MarkEditStatus() {
 	br.Msg = msg
 	br.Data = data
 }
+
+// BatchRefresh
+// @Title 批量刷新表格接口
+// @Description 批量刷新图表接口
+// @Param	request	body excel3.BatchRefreshExcelReq true "type json string"
+// @Success Ret=200 刷新成功
+// @router /excel_info/table/batch_refresh [post]
+func (this *ExcelInfoController) BatchRefresh() {
+	br := new(models.BaseResponse).Init()
+	defer func() {
+		if br.ErrMsg == "" {
+			br.IsSendEmail = false
+		}
+		this.Data["json"] = br
+		this.ServeJSON()
+	}()
+	sysUser := this.SysUser
+	if sysUser == nil {
+		br.Msg = "请登录"
+		br.ErrMsg = "请登录，SysUser Is Empty"
+		br.Ret = 408
+		return
+	}
+	var req excel3.BatchRefreshExcelReq
+	err := json.Unmarshal(this.Ctx.Input.RequestBody, &req)
+	if err != nil {
+		br.Msg = "参数解析异常！"
+		br.ErrMsg = "参数解析失败，Err:" + err.Error()
+		return
+	}
+	if len(req.ExcelCodes) == 0 {
+		br.Ret = 200
+		br.Success = true
+		br.Msg = "刷新成功"
+		return
+	}
+
+	// 获取表格关联的指标IDs
+	edbIds, e := excel2.GetEdbIdsFromExcelCodes(req.ExcelCodes)
+	if e != nil {
+		br.Msg = "刷新失败"
+		br.ErrMsg = "获取表格关联的指标IDs失败, Err: " + e.Error()
+		return
+	}
+
+	redisKey := data.GetBatchChartRefreshKey(req.Source, req.ReportId, req.ReportChapterId)
+	refreshKeys := make([]string, 0)
+	for _, v := range req.ExcelCodes {
+		refreshKeys = append(refreshKeys, fmt.Sprint(utils.HZ_CHART_LIB_EXCEL_TABLE_DETAIL, v))
+	}
+
+	// 刷新相关指标
+	syncing, e := data.BatchRefreshEdbByEdbIds(edbIds, redisKey, refreshKeys)
+	if e != nil {
+		br.Msg = "刷新失败"
+		br.ErrMsg = "刷新表格关联指标信息失败，Err:" + err.Error()
+		return
+	}
+
+	br.Msg = "刷新成功"
+	if syncing {
+		br.Msg = "表格关联指标较多，请10分钟后刷新页面查看最新数据"
+	}
+	br.Ret = 200
+	br.Success = true
+}
+
+// GetBatchChartRefreshResult
+// @Title 获取批量刷新表格结果
+// @Description 获取批量刷新表格结果
+// @Param	request	body excel3.BatchRefreshExcelReq true "type json string"
+// @Success Ret=200 刷新成功
+// @router /excel_info/table/batch_refresh/result [post]
+func (this *ExcelInfoController) GetBatchChartRefreshResult() {
+	br := new(models.BaseResponse).Init()
+	defer func() {
+		this.Data["json"] = br
+		this.ServeJSON()
+	}()
+	sysUser := this.SysUser
+	if sysUser == nil {
+		br.Msg = "请登录"
+		br.ErrMsg = "请登录，SysUser Is Empty"
+		br.Ret = 408
+		return
+	}
+	var req excel3.BatchRefreshExcelReq
+	err := json.Unmarshal(this.Ctx.Input.RequestBody, &req)
+	if err != nil {
+		br.Msg = "参数解析异常！"
+		br.ErrMsg = "参数解析失败，Err:" + err.Error()
+		return
+	}
+
+	// 校验缓存是否存在, 存在说明还在刷新中
+	result := true
+	redisKey := excel2.GetExcelEdbBatchRefreshKey(req.Source, req.ReportId, req.ReportChapterId)
+	if redisKey != `` {
+		// 如果找到了key，那么就是还在更新中
+		ok := utils.Rc.IsExist(redisKey)
+		if ok {
+			result = false
+		}
+	}
+
+	resp := struct {
+		RefreshResult bool `description:"刷新结果"`
+	}{
+		RefreshResult: result,
+	}
+	br.Data = resp
+	br.Ret = 200
+	br.Success = true
+}

models/data_manage/excel/excel_info.go

@@ -162,6 +162,14 @@ func GetExcelInfoById(excelInfoId int) (item *ExcelInfo, err error) {
 	return
 }
 
+// GetExcelInfoByUnicode 编码获取表格
+func GetExcelInfoByUnicode(unicode string) (item *ExcelInfo, err error) {
+	o := orm.NewOrmUsingDB("data")
+	sql := ` SELECT * FROM excel_info WHERE unique_code = ? AND is_delete = 0 `
+	err = o.Raw(sql, unicode).QueryRow(&item)
+	return
+}
+
 func GetExcelInfoViewById(excelInfoId int) (item *ExcelInfoView, err error) {
 	o := orm.NewOrmUsingDB("data")
 	sql := ` SELECT * FROM excel_info WHERE excel_info_id=? AND is_delete=0 `
@@ -500,3 +508,11 @@ func SaveExcelInfoAndSheet(excelInfo *ExcelInfo, updateExcelInfoParam []string,
 
 	return
 }
+
+// BatchRefreshExcelReq 批量刷新表格请求
+type BatchRefreshExcelReq struct {
+	ExcelCodes      []string `description:"表格编码"`
+	ReportId        int      `description:"报告ID"`
+	ReportChapterId int      `description:"报告章节ID"`
+	Source          string   `description:"来源，枚举值：report、english_report、smart_report"`
+}

routers/commentsRouter.go

@@ -727,6 +727,24 @@ func init() {
             Filters: nil,
             Params: nil})
 
+    beego.GlobalControllerRouter["eta/eta_api/controllers/data_manage/excel:ExcelInfoController"] = append(beego.GlobalControllerRouter["eta/eta_api/controllers/data_manage/excel:ExcelInfoController"],
+        beego.ControllerComments{
+            Method: "BatchRefresh",
+            Router: `/excel_info/table/batch_refresh`,
+            AllowHTTPMethods: []string{"post"},
+            MethodParams: param.Make(),
+            Filters: nil,
+            Params: nil})
+
+    beego.GlobalControllerRouter["eta/eta_api/controllers/data_manage/excel:ExcelInfoController"] = append(beego.GlobalControllerRouter["eta/eta_api/controllers/data_manage/excel:ExcelInfoController"],
+        beego.ControllerComments{
+            Method: "GetBatchChartRefreshResult",
+            Router: `/excel_info/table/batch_refresh/result`,
+            AllowHTTPMethods: []string{"post"},
+            MethodParams: param.Make(),
+            Filters: nil,
+            Params: nil})
+
     beego.GlobalControllerRouter["eta/eta_api/controllers/data_manage/excel:ExcelInfoController"] = append(beego.GlobalControllerRouter["eta/eta_api/controllers/data_manage/excel:ExcelInfoController"],
         beego.ControllerComments{
             Method: "Calculate",

services/data/edb_info.go

@@ -3057,3 +3057,60 @@ func EdbInfoWsdAdd(item *data_manage.EdbInfo) (edbInfo *data_manage.EdbInfo, err
 	AddOrEditEdbInfoToEs(int(edbInfoId))
 	return
 }
+
+// BatchRefreshEdbByEdbIds 批量刷新指标
+func BatchRefreshEdbByEdbIds(edbIdList []int, redisKey string, refreshKeys []string) (syncing bool, err error) {
+	if len(edbIdList) <= 0 {
+		return
+	}
+
+	// 设置刷新缓存
+	if redisKey != `` {
+		// 设置最多10分钟缓存
+		utils.Rc.SetNX(redisKey, 1, time.Minute*10)
+	}
+
+	// 获取需要刷新的指标列表
+	newBaseEdbInfoArr, newBasePredictEdbInfoArr, newCalculateMap, newPredictCalculateMap, calculateArr, predictCalculateArr := getEdbInfoIdList(edbIdList)
+
+	// 需要刷新的指标数量
+	totalEdbInfo := len(newBaseEdbInfoArr) + len(calculateArr) + len(predictCalculateArr)
+
+	// 关联指标过多的时候, 异步刷新
+	if totalEdbInfo > 20 {
+		syncing = true
+
+		go func() {
+			defer func() {
+				if err != nil {
+					alarm_msg.SendAlarmMsg("BatchRefreshEdbByEdbIds, ErrMsg: "+err.Error(), 3)
+				}
+			}()
+
+			err = edbInfoRefreshAll(false, newBaseEdbInfoArr, newBasePredictEdbInfoArr, newCalculateMap, newPredictCalculateMap, calculateArr, predictCalculateArr)
+
+			if redisKey != `` {
+				_ = utils.Rc.Delete(redisKey)
+			}
+			if len(refreshKeys) > 0 {
+				for _, v := range refreshKeys {
+					_ = utils.Rc.Delete(v)
+				}
+			}
+		}()
+		return
+	}
+
+	// 同步刷新
+	err = edbInfoRefreshAll(false, newBaseEdbInfoArr, newBasePredictEdbInfoArr, newCalculateMap, newPredictCalculateMap, calculateArr, predictCalculateArr)
+
+	if redisKey != `` {
+		_ = utils.Rc.Delete(redisKey)
+	}
+	if len(refreshKeys) > 0 {
+		for _, v := range refreshKeys {
+			_ = utils.Rc.Delete(v)
+		}
+	}
+	return
+}

services/data/excel/excel_info.go

@@ -33,6 +33,27 @@ func GetExcelDetailInfoByExcelInfoId(excelInfoId int) (excelDetail response.Exce
 		return
 	}
 
+	return formatExcelInfo2Detail(excelInfo)
+}
+
+// GetExcelDetailInfoByUnicode 根据表格编码获取表格详情
+func GetExcelDetailInfoByUnicode(unicode string) (excelDetail response.ExcelInfoDetail, errMsg string, err error) {
+	errMsg = `获取失败`
+	// 获取eta表格信息
+	excelInfo, err := excel.GetExcelInfoByUnicode(unicode)
+	if err != nil {
+		err = errors.New("获取ETA表格信息失败，Err:" + err.Error())
+		if err.Error() == utils.ErrNoRow() {
+			errMsg = "ETA表格被删除，请刷新页面"
+			err = errors.New("ETA表格被删除，请刷新页面，Err:" + err.Error())
+		}
+		return
+	}
+
+	return formatExcelInfo2Detail(excelInfo)
+}
+
+func formatExcelInfo2Detail(excelInfo *excel.ExcelInfo) (excelDetail response.ExcelInfoDetail, errMsg string, err error) {
 	excelDetail = response.ExcelInfoDetail{
 		ExcelInfoId:     excelInfo.ExcelInfoId,
 		Source:          excelInfo.Source,
@@ -85,7 +106,6 @@ func GetExcelDetailInfoByExcelInfoId(excelInfoId int) (excelDetail response.Exce
 		result.Data = newData
 		excelDetail.TableData = result
 	}
-
 	return
 }
 
@@ -1262,3 +1282,73 @@ func calculate(calculateFormula string, TagMap map[string]float64) (calVal, errM
 
 	return
 }
+
+// GetEdbIdsFromExcelCodes 获取表格中的指标IDs
+func GetEdbIdsFromExcelCodes(excelCodes []string) (edbIds []int, err error) {
+	edbIds = make([]int, 0)
+	edbIdExist := make(map[int]bool)
+	for _, v := range excelCodes {
+		// 表格详情
+		detail, msg, e := GetExcelDetailInfoByUnicode(v)
+		if e != nil {
+			err = fmt.Errorf("GetExcelDetailInfoByExcelInfoId err: %s, errMsg: %s", e.Error(), msg)
+			return
+		}
+
+		// 自定义表格
+		if detail.Source == utils.TIME_TABLE {
+			jsonByte, e := json.Marshal(detail.TableData)
+			if e != nil {
+				err = fmt.Errorf("JSON格式化自定义表格数据失败, Err: %s", e.Error())
+				return
+			}
+			var tableData request.TableDataReq
+			if e = json.Unmarshal(jsonByte, &tableData); e != nil {
+				err = fmt.Errorf("解析自定义表格数据失败, Err: %s", e.Error())
+				return
+			}
+			for _, tv := range tableData.EdbInfoIdList {
+				if edbIdExist[tv] {
+					continue
+				}
+				edbIdExist[tv] = true
+				edbIds = append(edbIds, tv)
+			}
+		}
+
+		// 混合表格
+		if detail.Source == utils.MIXED_TABLE {
+			jsonByte, e := json.Marshal(detail.TableData)
+			if e != nil {
+				err = fmt.Errorf("JSON格式化混合表格数据失败, Err: %s", e.Error())
+				return
+			}
+			var tableData request.MixedTableReq
+			if e = json.Unmarshal(jsonByte, &tableData); e != nil {
+				err = fmt.Errorf("解析混合表格数据失败, Err: %s", e.Error())
+				return
+			}
+			if len(tableData.Data) > 0 {
+				for _, td := range tableData.Data {
+					for _, tv := range td {
+						if tv.EdbInfoId > 0 && !edbIdExist[tv.EdbInfoId] {
+							edbIdExist[tv.EdbInfoId] = true
+							edbIds = append(edbIds, tv.EdbInfoId)
+						}
+					}
+				}
+			}
+		}
+	}
+
+	return
+}
+
+// GetExcelEdbBatchRefreshKey 获取批量刷新表格指标缓存key
+func GetExcelEdbBatchRefreshKey(source string, reportId, chapterId int) string {
+	if source == `` {
+		return ``
+	}
+
+	return fmt.Sprint("batch_refresh_excel_edb:", source, ":", reportId, ":", chapterId)
+}