初始化预测指标

models/data_manage/edb_info_calculate_mapping.go

@@ -168,3 +168,24 @@ func GetEdbInfoCalculateMappingListByEdbInfoIds(edbInfoIds []int) (items []*EdbI
 	_, err = o.Raw(sql, edbInfoIds).QueryRows(&items)
 	return
 }
+
+func GetEdbInfoCalculateMappingCountByCondition(condition string, pars []interface{}) (total int64, err error) {
+	o := orm.NewOrm()
+	sql := ` SELECT count(*) FROM edb_info_calculate_mapping WHERE 1=1 `
+	if condition != "" {
+		sql += condition
+	}
+	err = o.Raw(sql, pars).QueryRow(&total)
+	return
+}
+
+func GetEdbInfoCalculateMappingPageByCondition(condition string, pars []interface{}, startPage, pageSize int) (item []*EdbInfoCalculateMapping, err error) {
+	o := orm.NewOrm()
+	sql := ` SELECT * FROM edb_info_calculate_mapping WHERE 1=1 `
+	if condition != "" {
+		sql += condition
+	}
+	sql += ` LIMIT ?,? `
+	_, err = o.Raw(sql, pars, startPage, pageSize).QueryRows(&item)
+	return
+}

services/edb_refresh.go

@@ -778,9 +778,9 @@ func DisableEdbRefresh(cont context.Context) (err error) {
 		// 查询钢联和wind来源的指标
 		edbEndDate := now.AddDate(0, 0, -rule.EdbStopDays+1).Format(utils.FormatDate)
 
-		condition := ` AND no_update=0 AND source in (?,?) AND create_time < ?`
+		condition := ` AND no_update=0 AND source in (?,?) AND ((create_time < ? and set_update_time is null) or set_update_time < ? )`
 		var pars []interface{}
-		pars = append(pars, utils.DATA_SOURCE_MYSTEEL_CHEMICAL, utils.DATA_SOURCE_WIND, edbEndDate)
+		pars = append(pars, utils.DATA_SOURCE_MYSTEEL_CHEMICAL, utils.DATA_SOURCE_WIND, edbEndDate, edbEndDate)
 		// 查询钢联化工指标和wind指标 分批查询，先查总数，再查列表
 		totalCount, e := data_manage.GetEdbInfoCountByCondition(condition, pars)
 		if e != nil {

services/edb_relation.go

@@ -65,7 +65,7 @@ func InitChartEdbRelation() {
 		}
 		// 查询计算指标信息，并且建立关联关系
 		// 查询间接引用的指标信息
-		calculateEdbMappingListMap, calculateEdbMappingIdsMap, e := GetEdbListByEdbInfoId(edbInfoList)
+		calculateEdbMappingListMap, calculateEdbMappingIdsMap, e := GetEdbListByEdbInfoId(edbInfoList, false)
 		if e != nil {
 			err = fmt.Errorf("查询计算指标信息失败，%s", e.Error())
 			return
@@ -322,7 +322,7 @@ func InitChartCrossVariety() {
 		}
 		// 查询计算指标信息，并且建立关联关系
 		// 查询间接引用的指标信息
-		calculateEdbMappingListMap, calculateEdbMappingIdsMap, e := GetEdbListByEdbInfoId(edbInfoList)
+		calculateEdbMappingListMap, calculateEdbMappingIdsMap, e := GetEdbListByEdbInfoId(edbInfoList, false)
 		if e != nil {
 			err = fmt.Errorf("查询计算指标信息失败，%s", e.Error())
 			return
@@ -529,7 +529,7 @@ func InitCalendarIndicatorRelation() {
 
 		// 查询计算指标信息，并且建立关联关系
 		// 查询间接引用的指标信息
-		calculateEdbMappingListMap, calculateEdbMappingIdsMap, e := GetEdbListByEdbInfoId(edbInfoList)
+		calculateEdbMappingListMap, calculateEdbMappingIdsMap, e := GetEdbListByEdbInfoId(edbInfoList, false)
 		if e != nil {
 			err = fmt.Errorf("查询计算指标信息失败，%s", e.Error())
 			return
@@ -699,7 +699,7 @@ func InitExcelEdbRelation() {
 		}
 		// 查询计算指标信息，并且建立关联关系
 		// 查询间接引用的指标信息
-		calculateEdbMappingListMap, calculateEdbMappingIdsMap, e := GetEdbListByEdbInfoId(edbInfoList)
+		calculateEdbMappingListMap, calculateEdbMappingIdsMap, e := GetEdbListByEdbInfoId(edbInfoList, false)
 		if e != nil {
 			err = fmt.Errorf("查询计算指标信息失败，%s", e.Error())
 			return
@@ -884,7 +884,7 @@ func InitSandBoxEdbRelation() {
 		}
 		// 查询计算指标信息，并且建立关联关系
 		// 查询间接引用的指标信息
-		calculateEdbMappingListMap, calculateEdbMappingIdsMap, e := GetEdbListByEdbInfoId(edbInfoList)
+		calculateEdbMappingListMap, calculateEdbMappingIdsMap, e := GetEdbListByEdbInfoId(edbInfoList, false)
 		if e != nil {
 			err = fmt.Errorf("查询计算指标信息失败，%s", e.Error())
 			return
@@ -1019,13 +1019,13 @@ func getSandBoxEdbIdsByContent(content string) (edbInfoIds []int, err error) {
 	return
 }
 
-func GetEdbListByEdbInfoId(edbInfoList []*data_manage.EdbInfo) (edbMappingListMap map[int]*data_manage.EdbInfoCalculateMapping, edbInfoMappingRootIdsMap map[int][]int, err error) {
+func GetEdbListByEdbInfoId(edbInfoList []*data_manage.EdbInfo, needPredict bool) (edbMappingListMap map[int]*data_manage.EdbInfoCalculateMapping, edbInfoMappingRootIdsMap map[int][]int, err error) {
 	if len(edbInfoList) == 0 {
 		return
 	}
 	edbInfoIds := make([]int, 0)
 	for _, v := range edbInfoList {
-		if v.EdbType == 2 && v.EdbInfoType == 0 { //普通计算指标，排除预算指标
+		if (v.EdbType == 2 && v.EdbInfoType == 0) || (v.EdbType == 1 && v.EdbInfoType == 1 && needPredict) { //普通计算指标，或者是基础预测指标
 			edbInfoIds = append(edbInfoIds, v.EdbInfoId)
 		}
 	}
@@ -1055,7 +1055,7 @@ func GetEdbListByEdbInfoId(edbInfoList []*data_manage.EdbInfo) (edbMappingListMa
 	edbInfoMappingRootIdsMap = make(map[int][]int, 0)
 	edbMappingMap := make(map[int]struct{})
 	for _, edbInfo := range edbInfoList {
-		if edbInfo.EdbType == 2 && edbInfo.EdbInfoType == 0 {
+		if (edbInfo.EdbType == 2 && edbInfo.EdbInfoType == 0) || (edbInfo.EdbType == 1 && edbInfo.EdbInfoType == 1 && needPredict) {
 			edbInfoId := edbInfo.EdbInfoId
 			edbMappingList, err = getCalculateEdbInfoByEdbInfoId(allEdbMappingMap, edbInfoId, hasFindMap, edbInfoIdMap, edbMappingList, edbMappingMap, edbInfoMappingRootIdsMap, edbInfoId)
 			if err != nil {
@@ -1200,3 +1200,171 @@ func GetCalculateEdbByFromEdbInfo(edbInfoIds []int, calculateEdbIds []int, hasFi
 
 	return
 }
+
+// 初始化预测指标中的指标引用
+func InitPredictEdbRelation() {
+	fmt.Println("开始处理预测指标中的指标引用")
+	var err error
+	var addNum int
+	defer func() {
+		if err != nil {
+			msg := fmt.Sprintf("初始化指标在预测指标中的引用失败 InitPredictEdbRelation  err: %v", err)
+			utils.FileLog.Info(msg)
+			fmt.Println(msg)
+			go alarm_msg.SendAlarmMsg(msg, 3)
+		}
+	}()
+	//查询基础预测指标总数
+	condition := ` AND source = 30 `
+	var pars []interface{}
+	totalTmp, err := data_manage.GetEdbInfoCalculateMappingCountByCondition(condition, pars)
+	if err != nil {
+		err = fmt.Errorf("查询表格关联指标失败 err: %v", err)
+		return
+	}
+	total := int(totalTmp)
+	if total == 0 {
+		return
+	}
+	//分页查询，每次处理100条记录
+	pageSize := 100
+	totalPage := (total + pageSize - 1) / pageSize // 使用整数除法，并添加一页以防有余数
+	addList := make([]*data_manage.EdbInfoRelation, 0)
+	//查询表格列表
+	for i := 0; i < totalPage; i += 1 {
+		startSize := i * pageSize
+		list, e := data_manage.GetEdbInfoCalculateMappingPageByCondition(condition, pars, startSize, pageSize)
+		if e != nil {
+			err = fmt.Errorf("查询表格关联指标列表失败 Err:%s", e)
+			return
+		}
+		if len(list) == 0 {
+			break
+		}
+
+		edbInfoIds := make([]int, 0)
+		for _, v := range list {
+			edbInfoIds = append(edbInfoIds, v.FromEdbInfoId)
+		}
+		// 查询指标信息表
+		edbInfoList, e := data_manage.GetEdbInfoByIdList(edbInfoIds)
+		if e != nil {
+			err = fmt.Errorf("查询指标信息列表失败 Err:%s", e)
+			return
+		}
+		if len(edbInfoList) == 0 {
+			continue
+		}
+		// 查询计算指标信息，并且建立关联关系
+		// 查询间接引用的指标信息
+		calculateEdbMappingListMap, calculateEdbMappingIdsMap, e := GetEdbListByEdbInfoId(edbInfoList, true)
+		if e != nil {
+			err = fmt.Errorf("查询计算指标信息失败，%s", e.Error())
+			return
+		}
+		// 查询指标间接引用
+		edbInfoMap := make(map[int]*data_manage.EdbInfo)
+		for _, v := range edbInfoList {
+			edbInfoMap[v.EdbInfoId] = v
+		}
+
+		// 筛选有用的表格
+		predictIds := make([]int, 0)
+		for _, v := range list {
+			predictIds = append(predictIds, v.EdbInfoId)
+		}
+
+		//查询引用关系列表，
+		edbRelationList, e := data_manage.GetEdbInfoRelationByReferObjectIds(predictIds, utils.EDB_RELATION_PREDICT_EDB)
+		if e != nil {
+			err = fmt.Errorf("查询表格引用关系列表失败 Err:%s", e)
+			return
+		}
+		existRelationMap := make(map[string]struct{})
+		for _, v := range edbRelationList {
+			name := fmt.Sprintf("%d-%d", v.ReferObjectId, v.EdbInfoId)
+			existRelationMap[name] = struct{}{}
+		}
+		for _, v := range list {
+			nowTime := time.Now()
+			name := fmt.Sprintf("%d-%d", v.EdbInfoId, v.FromEdbInfoId)
+			if _, ok := existRelationMap[name]; !ok {
+				edbInfo, ok2 := edbInfoMap[v.FromEdbInfoId]
+				if !ok2 {
+					continue
+				}
+				tmp := &data_manage.EdbInfoRelation{
+					ReferObjectId:      v.EdbInfoId,
+					ReferObjectType:    utils.EDB_RELATION_PREDICT_EDB,
+					ReferObjectSubType: 1,
+					EdbInfoId:          v.FromEdbInfoId,
+					EdbName:            edbInfo.EdbName,
+					Source:             edbInfo.Source,
+					EdbCode:            edbInfo.EdbCode,
+					CreateTime:         nowTime,
+					ModifyTime:         nowTime,
+					RelationTime:       v.CreateTime,
+				}
+				tmp.RelationCode = fmt.Sprintf("%d_%d_%d_%d", tmp.EdbInfoId, tmp.ReferObjectId, tmp.ReferObjectType, tmp.ReferObjectSubType)
+				addList = append(addList, tmp)
+				existRelationMap[name] = struct{}{}
+				// 添加间接引用记录
+				if (edbInfo.EdbType == 2 && edbInfo.EdbInfoType == 0) || (edbInfo.EdbType == 1 && edbInfo.EdbInfoType == 1) {
+					childEdbMappingIds, ok1 := calculateEdbMappingIdsMap[edbInfo.EdbInfoId]
+					if !ok1 {
+						continue
+					}
+					for _, childEdbMappingId := range childEdbMappingIds {
+						childEdbMapping, ok2 := calculateEdbMappingListMap[childEdbMappingId]
+						if !ok2 {
+							continue
+						}
+						name1 := fmt.Sprintf("%d-%d", v.EdbInfoId, childEdbMapping.FromEdbInfoId)
+						if _, ok2 := existRelationMap[name1]; ok2 { //如果已经被直接引用了，则无需添加到间接引用记录中
+							continue
+						}
+						tmp1 := &data_manage.EdbInfoRelation{
+							ReferObjectId:      v.EdbInfoId,
+							ReferObjectType:    utils.EDB_RELATION_PREDICT_EDB,
+							ReferObjectSubType: 1,
+							EdbInfoId:          childEdbMapping.FromEdbInfoId,
+							EdbName:            childEdbMapping.FromEdbName,
+							Source:             childEdbMapping.FromSource,
+							EdbCode:            childEdbMapping.FromEdbCode,
+							CreateTime:         nowTime,
+							ModifyTime:         nowTime,
+							RelationTime:       v.CreateTime,
+							RelationType:       1,
+							RootEdbInfoId:      edbInfo.EdbInfoId,
+							ChildEdbInfoId:     childEdbMapping.EdbInfoId,
+							RelationCode:       tmp.RelationCode,
+						}
+						addList = append(addList, tmp1)
+						// todo 防止重复
+					}
+				}
+
+				if len(addList) > pageSize {
+					err = data_manage.AddEdbInfoRelationMulti(addList)
+					if err != nil {
+						err = fmt.Errorf("新增引用记录失败 Err:%s", err)
+						return
+					}
+					addNum += len(addList)
+					addList = make([]*data_manage.EdbInfoRelation, 0)
+				}
+			}
+		}
+	}
+	//拿到500个数据ID，判断相关的引用记录，如果已存在则直接过滤，不存在则新增
+	if len(addList) > 0 {
+		err = data_manage.AddEdbInfoRelationMulti(addList)
+		if err != nil {
+			err = fmt.Errorf("新增引用记录失败 Err:%s", err)
+			return
+		}
+		addNum += len(addList)
+	}
+	fmt.Printf("预测指标引用记录处理完成, 新增%d条记录\n", addNum)
+	return
+}

utils/constants.go

@@ -162,10 +162,11 @@ const CACHE_EDB_UPDATE_LOG_ID = "eta:edb_update_log:id"
 
 // 指标引用对象
 const (
-	EDB_RELATION_CHART    = 1 // 图表
-	EDB_RELATION_SANDBOX  = 2 // ETA逻辑
-	EDB_RELATION_CALENDAR = 3 // 事件日历
-	EDB_RELATION_TABLE    = 4 // 表格
+	EDB_RELATION_CHART       = 1 // 图表
+	EDB_RELATION_SANDBOX     = 2 // ETA逻辑
+	EDB_RELATION_CALENDAR    = 3 // 事件日历
+	EDB_RELATION_TABLE       = 4 // 表格
+	EDB_RELATION_PREDICT_EDB = 5 // 预测指标
 )
 
 // 图表类型