初始化添加混合表格

models/data_manage/edb_info_calculate_mapping.go

@@ -158,3 +158,13 @@ func GetEdbInfoCalculateMappingListByEdbInfoId(edbInfoId int) (items []*EdbInfoC
 	_, err = o.Raw(sql, edbInfoId).QueryRows(&items)
 	return
 }
+
+// GetEdbInfoCalculateMappingListByEdbInfoIds 根据生成的指标id获取来源的指标id列表
+func GetEdbInfoCalculateMappingListByEdbInfoIds(edbInfoIds []int) (items []*EdbInfoCalculateMappingInfo, err error) {
+	o := orm.NewOrmUsingDB("data")
+	sql := ` SELECT a.*,b.edb_type as from_edb_type,b.edb_info_type as from_edb_info_type, b.unique_code AS from_unique_code, b.classify_id AS from_classify_id,b.no_update FROM edb_info_calculate_mapping AS a
+			INNER JOIN edb_info AS b ON a.from_edb_info_id=b.edb_info_id
+			WHERE a.edb_info_id in (` + utils.GetOrmInReplace(len(edbInfoIds)) + `) `
+	_, err = o.Raw(sql, edbInfoIds).QueryRows(&items)
+	return
+}

models/data_manage/excel/excel_edb_mapping.go

@@ -0,0 +1,35 @@
+package excel
+
+import (
+	"eta/eta_task/utils"
+	"github.com/beego/beego/v2/client/orm"
+	"time"
+)
+
+// ExcelEdbMapping excel与指标的关系表
+type ExcelEdbMapping struct {
+	ExcelEdbMappingId int       `orm:"column(excel_edb_mapping_id);pk"`
+	ExcelInfoId       int       `description:"excel的id"`
+	Source            int       `description:"表格来源，1：excel插件的表格，2：自定义表格，3：混合表格，4：自定义分析，默认：1"`
+	EdbInfoId         int       `description:"计算指标id"`
+	CreateTime        time.Time `description:"创建时间"`
+	ModifyTime        time.Time `description:"修改时间"`
+}
+
+// GetExcelEdbMappingTotalBySource 根据表格类型获取总数
+func GetExcelEdbMappingTotalBySource(sources []int) (total int, err error) {
+	o := orm.NewOrmUsingDB("data")
+	sql := ` SELECT count(*)  FROM excel_edb_mapping WHERE source in (` + utils.GetOrmInReplace(len(sources)) + `)`
+
+	err = o.Raw(sql, sources).QueryRow(&total)
+	return
+}
+
+// GetExcelEdbMappingListBySource 根据表格类型获取列表
+func GetExcelEdbMappingListBySource(sources []int, pageIndex, pageSize int) (items []*ExcelEdbMapping, err error) {
+	o := orm.NewOrmUsingDB("data")
+	sql := ` SELECT *  FROM excel_edb_mapping WHERE source in (` + utils.GetOrmInReplace(len(sources)) + `) Limit ?,? `
+
+	_, err = o.Raw(sql, sources, pageIndex, pageSize).QueryRows(&items)
+	return
+}

services/edb_relation.go

@@ -4,6 +4,7 @@ import (
 	"encoding/json"
 	"eta/eta_task/models/data_manage"
 	"eta/eta_task/models/data_manage/cross_variety"
+	"eta/eta_task/models/data_manage/excel"
 	"eta/eta_task/models/fe_calendar"
 	"eta/eta_task/models/sandbox"
 	"eta/eta_task/services/alarm_msg"
@@ -16,7 +17,8 @@ func InitEdbRelation() {
 	//InitChartEdbRelation()
 	//InitChartCrossVariety()
 	//initCalendarIndicatorRelation()
-	initSandBoxEdbRelation()
+	//initSandBoxEdbRelation()
+	InitExcelEdbRelation()
 }
 func InitChartEdbRelation() {
 	fmt.Println("开始处理图表中的指标引用")
@@ -155,6 +157,10 @@ func InitChartEdbRelation() {
 						if !ok2 {
 							continue
 						}
+						name1 := fmt.Sprintf("%d-%d", v.ChartInfoId, childEdbMapping.FromEdbInfoId)
+						if _, ok2 := existRelationMap[name1]; !ok2 { //如果已经被直接引用了，则无需添加到间接引用记录中
+							continue
+						}
 						childEdb, ok2 := calculateEdbListMap[childEdbMapping.FromEdbInfoId]
 						if !ok2 {
 							continue
@@ -387,6 +393,10 @@ func InitChartCrossVariety() {
 							if !ok2 {
 								continue
 							}
+							name1 := fmt.Sprintf("%d-%d", item.ChartInfoId, childEdbMapping.FromEdbInfoId)
+							if _, ok2 := existRelationMap[name1]; !ok2 { //如果已经被直接引用了，则无需添加到间接引用记录中
+								continue
+							}
 							childEdb, ok2 := calculateEdbListMap[childEdbMapping.FromEdbInfoId]
 							if !ok2 {
 								continue
@@ -562,6 +572,10 @@ func initCalendarIndicatorRelation() {
 								if !ok2 {
 									continue
 								}
+								name1 := fmt.Sprintf("%d-%d", v.FeCalendarMatterId, childEdbMapping.FromEdbInfoId)
+								if _, ok2 := existRelationMap[name1]; !ok2 { //如果已经被直接引用了，则无需添加到间接引用记录中
+									continue
+								}
 								childEdb, ok2 := calculateEdbListMap[childEdbMapping.FromEdbInfoId]
 								if !ok2 {
 									continue
@@ -612,6 +626,175 @@ func initCalendarIndicatorRelation() {
 	return
 }
 
+// 初始化表格中的指标引用
+func InitExcelEdbRelation() {
+	fmt.Println("开始处理表格中的指标引用")
+	var err error
+	var addNum int
+	defer func() {
+		if err != nil {
+			msg := fmt.Sprintf("初始化指标在表格中的引用失败 InitChartEdbRelation  err: %v", err)
+			utils.FileLog.Info(msg)
+			fmt.Println(msg)
+			go alarm_msg.SendAlarmMsg(msg, 3)
+		}
+	}()
+	//查询表格指标绑定表
+	sources := []int{utils.TIME_TABLE, utils.MIXED_TABLE, utils.BALANCE_TABLE}
+	total, err := excel.GetExcelEdbMappingTotalBySource(sources)
+	if err != nil {
+		err = fmt.Errorf("查询表格关联指标失败 err: %v", err)
+		return
+	}
+	if total == 0 {
+		return
+	}
+	//分页查询，每次处理100条记录
+	pageSize := 100
+	totalPage := (total + pageSize - 1) / pageSize // 使用整数除法，并添加一页以防有余数
+	addList := make([]*data_manage.EdbInfoRelation, 0)
+	//查询表格列表
+	for i := 0; i < totalPage; i += 1 {
+		startSize := i * pageSize
+		list, e := excel.GetExcelEdbMappingListBySource(sources, startSize, pageSize)
+		if e != nil {
+			err = fmt.Errorf("查询表格关联指标列表失败 Err:%s", e)
+			return
+		}
+		if len(list) == 0 {
+			break
+		}
+
+		edbInfoIds := make([]int, 0)
+		for _, v := range list {
+			edbInfoIds = append(edbInfoIds, v.EdbInfoId)
+		}
+		// 查询指标信息表
+		edbInfoList, e := data_manage.GetEdbInfoByIdList(edbInfoIds)
+		if e != nil {
+			err = fmt.Errorf("查询指标信息列表失败 Err:%s", e)
+			return
+		}
+		if len(edbInfoList) == 0 {
+			continue
+		}
+		// 查询计算指标信息，并且建立关联关系
+		// 查询间接引用的指标信息
+		calculateEdbListMap, calculateEdbMappingListMap, calculateEdbMappingIdsMap, e := GetEdbListByEdbInfoId(edbInfoList)
+		if e != nil {
+			err = fmt.Errorf("查询计算指标信息失败，%s", e.Error())
+			return
+		}
+		// 查询指标间接引用
+		edbInfoMap := make(map[int]*data_manage.EdbInfo)
+		for _, v := range edbInfoList {
+			edbInfoMap[v.EdbInfoId] = v
+		}
+
+		// 筛选有用的表格
+		excelIds := make([]int, 0)
+		for _, v := range list {
+			excelIds = append(excelIds, v.ExcelInfoId)
+		}
+
+		//查询引用关系列表，
+		chartEdbRelationList, e := data_manage.GetEdbInfoRelationByReferObjectIds(excelIds, utils.EDB_RELATION_TABLE)
+		if e != nil {
+			err = fmt.Errorf("查询表格引用关系列表失败 Err:%s", e)
+			return
+		}
+		existRelationMap := make(map[string]struct{})
+		for _, v := range chartEdbRelationList {
+			name := fmt.Sprintf("%d-%d", v.ReferObjectId, v.EdbInfoId)
+			existRelationMap[name] = struct{}{}
+		}
+		for _, v := range list {
+			nowTime := time.Now()
+			name := fmt.Sprintf("%d-%d", v.ExcelInfoId, v.EdbInfoId)
+			if _, ok := existRelationMap[name]; !ok {
+				edbInfo, ok2 := edbInfoMap[v.EdbInfoId]
+				if !ok2 {
+					continue
+				}
+				//todo 引用时间
+				tmp := &data_manage.EdbInfoRelation{
+					ReferObjectId:      v.ExcelInfoId,
+					ReferObjectType:    utils.EDB_RELATION_TABLE,
+					ReferObjectSubType: v.Source,
+					EdbInfoId:          v.EdbInfoId,
+					EdbName:            edbInfo.EdbName,
+					Source:             edbInfo.Source,
+					EdbCode:            edbInfo.EdbCode,
+					CreateTime:         nowTime,
+					ModifyTime:         nowTime,
+					RelationTime:       v.CreateTime,
+				}
+				addList = append(addList, tmp)
+				existRelationMap[name] = struct{}{}
+				// 添加间接引用记录
+				if edbInfo.EdbType == 2 {
+					childEdbMappingIds, ok1 := calculateEdbMappingIdsMap[edbInfo.EdbInfoId]
+					if !ok1 {
+						continue
+					}
+					for _, childEdbMappingId := range childEdbMappingIds {
+						childEdbMapping, ok2 := calculateEdbMappingListMap[childEdbMappingId]
+						if !ok2 {
+							continue
+						}
+						name1 := fmt.Sprintf("%d-%d", v.ExcelInfoId, childEdbMapping.FromEdbInfoId)
+						if _, ok2 := existRelationMap[name1]; !ok2 { //如果已经被直接引用了，则无需添加到间接引用记录中
+							continue
+						}
+						childEdb, ok2 := calculateEdbListMap[childEdbMapping.FromEdbInfoId]
+						if !ok2 {
+							continue
+						}
+						tmp1 := &data_manage.EdbInfoRelation{
+							ReferObjectId:      v.ExcelInfoId,
+							ReferObjectType:    utils.EDB_RELATION_TABLE,
+							ReferObjectSubType: v.Source,
+							EdbInfoId:          childEdb.EdbInfoId,
+							EdbName:            childEdb.EdbName,
+							Source:             childEdb.Source,
+							EdbCode:            childEdb.EdbCode,
+							CreateTime:         nowTime,
+							ModifyTime:         nowTime,
+							RelationTime:       v.CreateTime,
+							RelationType:       1,
+							RootEdbInfoId:      edbInfo.EdbInfoId,
+							ChildEdbInfoId:     childEdbMapping.EdbInfoId,
+						}
+						addList = append(addList, tmp1)
+						// todo 防止重复
+					}
+				}
+
+				if len(addList) > pageSize {
+					err = data_manage.AddEdbInfoRelationMulti(addList)
+					if err != nil {
+						err = fmt.Errorf("新增引用记录失败 Err:%s", err)
+						return
+					}
+					addNum += len(addList)
+					addList = make([]*data_manage.EdbInfoRelation, 0)
+				}
+			}
+		}
+	}
+	//拿到500个数据ID，判断相关的引用记录，如果已存在则直接过滤，不存在则新增
+	if len(addList) > 0 {
+		err = data_manage.AddEdbInfoRelationMulti(addList)
+		if err != nil {
+			err = fmt.Errorf("新增引用记录失败 Err:%s", err)
+			return
+		}
+		addNum += len(addList)
+	}
+	fmt.Printf("表格指标引用记录处理完成, 新增%d条记录\n", addNum)
+	return
+}
+
 // 处理逻辑图中的指标引用
 func initSandBoxEdbRelation() {
 	fmt.Println("开始处理逻辑图中的指标引用")
@@ -742,6 +925,10 @@ func initSandBoxEdbRelation() {
 							if !ok2 {
 								continue
 							}
+							name1 := fmt.Sprintf("%d-%d", v.SandboxId, childEdbMapping.FromEdbInfoId)
+							if _, ok2 := existRelationMap[name1]; !ok2 { //如果已经被直接引用了，则无需添加到间接引用记录中
+								continue
+							}
 							childEdb, ok2 := calculateEdbListMap[childEdbMapping.FromEdbInfoId]
 							if !ok2 {
 								continue
@@ -812,7 +999,31 @@ func getSandBoxEdbIdsByContent(content string) (edbInfoIds []int, err error) {
 }
 
 func GetEdbListByEdbInfoId(edbInfoList []*data_manage.EdbInfo) (edbInfoMap map[int]*data_manage.EdbInfo, edbMappingListMap map[int]*data_manage.EdbInfoCalculateMapping, edbInfoMappingRootIdsMap map[int][]int, err error) {
+	if len(edbInfoList) == 0 {
+		return
+	}
+	edbInfoIds := make([]int, 0)
+	for _, v := range edbInfoList {
+		if v.EdbType == 2 {
+			edbInfoIds = append(edbInfoIds, v.EdbInfoId)
+		}
+	}
+	if len(edbInfoIds) == 0 {
+		return
+	}
 	//查询指标信息
+	allEdbMappingMap := make(map[int][]*data_manage.EdbInfoCalculateMappingInfo, 0)
+	allMappingList, e := data_manage.GetEdbInfoCalculateMappingListByEdbInfoIds(edbInfoIds)
+	if e != nil {
+		err = fmt.Errorf("GetEdbInfoCalculateMappingListByEdbInfoIds err: %s", e.Error())
+		return
+	}
+	for _, v := range allMappingList {
+		if _, ok := allEdbMappingMap[v.EdbInfoId]; !ok {
+			allEdbMappingMap[v.EdbInfoId] = make([]*data_manage.EdbInfoCalculateMappingInfo, 0)
+		}
+		allEdbMappingMap[v.EdbInfoId] = append(allEdbMappingMap[v.EdbInfoId], v)
+	}
 	//查询指标映射
 	//查询所有指标数据
 	//查询这个指标相关的mapping信息放到数组里，
@@ -825,7 +1036,7 @@ func GetEdbListByEdbInfoId(edbInfoList []*data_manage.EdbInfo) (edbInfoMap map[i
 	for _, edbInfo := range edbInfoList {
 		if edbInfo.EdbType == 2 {
 			edbInfoId := edbInfo.EdbInfoId
-			edbMappingList, err = getCalculateEdbInfoByEdbInfoId(edbInfoId, hasFindMap, edbInfoIdMap, edbMappingList, edbMappingMap, edbInfoMappingRootIdsMap, edbInfoId)
+			edbMappingList, err = getCalculateEdbInfoByEdbInfoId(allEdbMappingMap, edbInfoId, hasFindMap, edbInfoIdMap, edbMappingList, edbMappingMap, edbInfoMappingRootIdsMap, edbInfoId)
 			if err != nil {
 				err = fmt.Errorf(" GetCalculateEdbInfoByEdbInfoId err: %s", err.Error())
 				return
@@ -863,7 +1074,7 @@ func GetEdbListByEdbInfoId(edbInfoList []*data_manage.EdbInfo) (edbInfoMap map[i
 }
 
 // getCalculateEdbInfoByEdbInfoId 计算指标追溯
-func getCalculateEdbInfoByEdbInfoId(edbInfoId int, hasFindMap map[int]struct{}, edbInfoIdMap map[int]struct{}, edbMappingList []*data_manage.EdbInfoCalculateMapping, edbMappingMap map[int]struct{}, edbInfoMappingRootIdsMap map[int][]int, rootEdbInfoId int) (newEdbMappingList []*data_manage.EdbInfoCalculateMapping, err error) {
+func getCalculateEdbInfoByEdbInfoId(allEdbMappingMap map[int][]*data_manage.EdbInfoCalculateMappingInfo, edbInfoId int, hasFindMap map[int]struct{}, edbInfoIdMap map[int]struct{}, edbMappingList []*data_manage.EdbInfoCalculateMapping, edbMappingMap map[int]struct{}, edbInfoMappingRootIdsMap map[int][]int, rootEdbInfoId int) (newEdbMappingList []*data_manage.EdbInfoCalculateMapping, err error) {
 	newEdbMappingList = edbMappingList
 	_, ok := hasFindMap[edbInfoId]
 	if ok {
@@ -873,13 +1084,16 @@ func getCalculateEdbInfoByEdbInfoId(edbInfoId int, hasFindMap map[int]struct{},
 	if _, ok1 := edbInfoIdMap[edbInfoId]; !ok1 {
 		edbInfoIdMap[edbInfoId] = struct{}{}
 	}
-
-	edbInfoMappingList, e := data_manage.GetEdbInfoCalculateMappingListByEdbInfoId(edbInfoId)
-	if e != nil {
-		err = fmt.Errorf("GetEdbInfoCalculateMappingListByEdbInfoId err: %s", e.Error())
-		return
+	edbInfoMappingList := make([]*data_manage.EdbInfoCalculateMappingInfo, 0)
+	edbInfoMappingList, ok = allEdbMappingMap[edbInfoId]
+	if !ok {
+		edbInfoMappingList, err = data_manage.GetEdbInfoCalculateMappingListByEdbInfoId(edbInfoId)
+		if err != nil {
+			err = fmt.Errorf("GetEdbInfoCalculateMappingListByEdbInfoId err: %s", err.Error())
+			return
+		}
 	}
-
+	hasFindMap[edbInfoId] = struct{}{}
 	if len(edbInfoMappingList) > 0 {
 		fromEdbInfoIdList := make([]int, 0)
 		edbInfoMappingIdList := make([]int, 0)
@@ -914,9 +1128,9 @@ func getCalculateEdbInfoByEdbInfoId(edbInfoId int, hasFindMap map[int]struct{},
 			if edbInfoId != v.FromEdbInfoId && (v.FromEdbType == 2 || v.FromEdbInfoType == 1) {
 				// 查过了就不查了
 				if _, ok2 := hasFindMap[v.FromEdbInfoId]; !ok2 {
-					newEdbMappingList, e = getCalculateEdbInfoByEdbInfoId(v.FromEdbInfoId, hasFindMap, edbInfoIdMap, newEdbMappingList, edbMappingMap, edbInfoMappingRootIdsMap, rootEdbInfoId)
-					if e != nil {
-						err = fmt.Errorf("traceEdbInfoByEdbInfoId err: %s", e.Error())
+					newEdbMappingList, err = getCalculateEdbInfoByEdbInfoId(allEdbMappingMap, v.FromEdbInfoId, hasFindMap, edbInfoIdMap, newEdbMappingList, edbMappingMap, edbInfoMappingRootIdsMap, rootEdbInfoId)
+					if err != nil {
+						err = fmt.Errorf("traceEdbInfoByEdbInfoId err: %s", err.Error())
 						return
 					}
 				}
@@ -926,6 +1140,5 @@ func getCalculateEdbInfoByEdbInfoId(edbInfoId int, hasFindMap map[int]struct{},
 		edbInfoMappingRootIdsMap[rootEdbInfoId] = append(edbInfoMappingRootIdsMap[rootEdbInfoId], edbInfoMappingIdList...)
 	}
 
-	hasFindMap[edbInfoId] = struct{}{}
 	return
 }

utils/constants.go

@@ -179,3 +179,12 @@ const (
 	CHART_SOURCE_CROSS_HEDGING                   = 10 // 跨品种分析图表
 	CHART_SOURCE_BALANCE_EXCEL                   = 11 // 平衡表图表
 )
+
+// ETA表格
+const (
+	EXCEL_DEFAULT         = 1 // 自定义excel
+	TIME_TABLE            = 2 // 时间序列表格
+	MIXED_TABLE           = 3 // 混合表格
+	CUSTOM_ANALYSIS_TABLE = 4 // 自定义分析表格
+	BALANCE_TABLE         = 5 // 平衡表
+)