package services import ( "encoding/json" "eta_gn/eta_task/models/data_manage" "eta_gn/eta_task/models/data_manage/cross_variety" "eta_gn/eta_task/models/data_manage/excel" "eta_gn/eta_task/models/fe_calendar" "eta_gn/eta_task/models/sandbox" "eta_gn/eta_task/services/alarm_msg" "eta_gn/eta_task/utils" "fmt" "time" ) func InitChartEdbRelation() { fmt.Println("开始处理图表中的指标引用") var err error var addNum int defer func() { if err != nil { msg := fmt.Sprintf("初始化指标在图表中的引用失败 InitChartEdbRelation err: %v", err) utils.FileLog.Info(msg) fmt.Println(msg) go alarm_msg.SendAlarmMsg(msg, 3) } }() //查询chart_edb_mapping 表 total, err := data_manage.GetChartEdbMappingTotal() if err != nil { err = fmt.Errorf("查询图表关联指标失败 err: %v", err) return } if total == 0 { return } //分页查询，每次处理500条记录 pageSize := 500 totalPage := (total + pageSize - 1) / pageSize // 使用整数除法，并添加一页以防有余数 addList := make([]*data_manage.EdbInfoRelation, 0) //查询图表列表 for i := 0; i < totalPage; i += 1 { startSize := i * pageSize list, e := data_manage.GetChartEdbMappingList(startSize, pageSize) if e != nil { err = fmt.Errorf("查询图表关联指标列表失败 Err:%s", e) return } if len(list) == 0 { break } edbInfoIds := make([]int, 0) for _, v := range list { edbInfoIds = append(edbInfoIds, v.EdbInfoId) } // 查询指标信息表 edbInfoList, e := data_manage.GetEdbInfoByIdList(edbInfoIds) if e != nil { err = fmt.Errorf("查询指标信息列表失败 Err:%s", e) return } if len(edbInfoList) == 0 { continue } // 查询计算指标信息，并且建立关联关系 // 查询间接引用的指标信息 calculateEdbMappingListMap, calculateEdbMappingIdsMap, e := GetEdbListByEdbInfoId(edbInfoList) if e != nil { err = fmt.Errorf("查询计算指标信息失败，%s", e.Error()) return } // 查询指标间接引用 edbInfoMap := make(map[int]*data_manage.EdbInfo) for _, v := range edbInfoList { edbInfoMap[v.EdbInfoId] = v } // 筛选有用的图表 finalList := make([]*data_manage.ChartEdbMapping, 0) chartIds := make([]int, 0) for _, v := range list { if _, ok2 := edbInfoMap[v.EdbInfoId]; !ok2 { continue } finalList = append(finalList, v) chartIds = append(chartIds, v.ChartInfoId) } if len(chartIds) == 0 { continue } // 查询图表信息 chartInfoList, e := data_manage.GetChartInfoByChartInfoIds(chartIds) if e != nil { err = fmt.Errorf("查询图表信息列表失败 Err:%s", e) return } chartInfoMap := make(map[int]*data_manage.ChartInfo) for _, v := range chartInfoList { chartInfoMap[v.ChartInfoId] = v } //查询引用关系列表， chartEdbRelationList, e := data_manage.GetEdbInfoRelationByReferObjectIds(chartIds, utils.EDB_RELATION_CHART) if e != nil { err = fmt.Errorf("查询图表引用关系列表失败 Err:%s", e) return } existRelationMap := make(map[string]struct{}) for _, v := range chartEdbRelationList { name := fmt.Sprintf("%d-%d", v.ReferObjectId, v.EdbInfoId) existRelationMap[name] = struct{}{} } for _, v := range finalList { nowTime := time.Now() name := fmt.Sprintf("%d-%d", v.ChartInfoId, v.EdbInfoId) if _, ok := existRelationMap[name]; !ok { //查询图表信息 chartInfo, ok1 := chartInfoMap[v.ChartInfoId] if !ok1 { continue } if chartInfo.Source == utils.CHART_SOURCE_CROSS_HEDGING { //过滤掉跨品种分析的图表 continue } edbInfo, ok2 := edbInfoMap[v.EdbInfoId] if !ok2 { continue } // 去掉预测指标 if edbInfo.EdbInfoType == 1 { continue } tmp := &data_manage.EdbInfoRelation{ ReferObjectId: v.ChartInfoId, ReferObjectType: utils.EDB_RELATION_CHART, ReferObjectSubType: chartInfo.Source, EdbInfoId: v.EdbInfoId, EdbName: edbInfo.EdbName, Source: edbInfo.Source, EdbCode: edbInfo.EdbCode, CreateTime: nowTime, ModifyTime: nowTime, RelationTime: v.CreateTime, } tmp.RelationCode = fmt.Sprintf("%d_%d_%d_%d", tmp.EdbInfoId, tmp.ReferObjectId, tmp.ReferObjectType, tmp.ReferObjectSubType) addList = append(addList, tmp) existRelationMap[name] = struct{}{} // 添加间接引用记录 if edbInfo.EdbType == 2 && edbInfo.EdbInfoType == 0 { childEdbMappingIds, ok1 := calculateEdbMappingIdsMap[edbInfo.EdbInfoId] if !ok1 { continue } for _, childEdbMappingId := range childEdbMappingIds { childEdbMapping, ok2 := calculateEdbMappingListMap[childEdbMappingId] if !ok2 { continue } name1 := fmt.Sprintf("%d-%d", v.ChartInfoId, childEdbMapping.FromEdbInfoId) if _, ok2 := existRelationMap[name1]; ok2 { //如果已经被直接引用了，则无需添加到间接引用记录中 continue } tmp1 := &data_manage.EdbInfoRelation{ ReferObjectId: v.ChartInfoId, ReferObjectType: utils.EDB_RELATION_CHART, ReferObjectSubType: chartInfo.Source, EdbInfoId: childEdbMapping.FromEdbInfoId, EdbName: childEdbMapping.FromEdbName, Source: childEdbMapping.FromSource, EdbCode: childEdbMapping.FromEdbCode, CreateTime: nowTime, ModifyTime: nowTime, RelationTime: v.CreateTime, RelationType: 1, RootEdbInfoId: edbInfo.EdbInfoId, ChildEdbInfoId: childEdbMapping.EdbInfoId, RelationCode: tmp.RelationCode, } addList = append(addList, tmp1) // todo 防止重复 } } if len(addList) > pageSize { err = data_manage.AddEdbInfoRelationMulti(addList) if err != nil { err = fmt.Errorf("新增引用记录失败 Err:%s", err) return } addNum += len(addList) addList = make([]*data_manage.EdbInfoRelation, 0) } } } } //拿到500个数据ID，判断相关的引用记录，如果已存在则直接过滤，不存在则新增 if len(addList) > 0 { err = data_manage.AddEdbInfoRelationMulti(addList) if err != nil { err = fmt.Errorf("新增引用记录失败 Err:%s", err) return } addNum += len(addList) } fmt.Printf("图表指标引用记录处理完成, 新增%d条记录\n", addNum) return } // InitChartCrossVariety 处理特殊图表，跨品种分析图表 func InitChartCrossVariety() { fmt.Println("开始跨品种分析图表中的指标引用") var addNum int var err error defer func() { if err != nil { msg := fmt.Sprintf("初始化指标在跨品种分析图表中的引用失败 InitChartCrossVariety err: %v", err) utils.FileLog.Info(msg) fmt.Println(msg) go alarm_msg.SendAlarmMsg(msg, 3) } }() total, err := cross_variety.GetChartInfoCrossVarietyTotal() if err != nil { err = fmt.Errorf("查询图表关联指标失败 err: %v", err) return } if total == 0 { return } //分页查询，每次处理500条记录 pageSize := 500 totalPage := (total + pageSize - 1) / pageSize // 使用整数除法，并添加一页以防有余数 addList := make([]*data_manage.EdbInfoRelation, 0) //查询图表列表 for i := 0; i < totalPage; i += 1 { startSize := i * pageSize list, e := cross_variety.GetChartInfoCrossVarietyList(startSize, pageSize) if e != nil { err = fmt.Errorf("查询图表关联指标列表失败 Err:%s", e) return } if len(list) == 0 { break } chartIds := make([]int, 0) chartIdMap := make(map[int]struct{}) tagIds := make([]int, 0) tagIdsMap := make(map[int]struct{}) tagChartMap := make(map[int][]*cross_variety.ChartInfoCrossVariety) for _, v := range list { if _, ok := chartIdMap[v.ChartInfoId]; !ok { chartIdMap[v.ChartInfoId] = struct{}{} chartIds = append(chartIds, v.ChartInfoId) } if _, ok := tagIdsMap[v.ChartXTagId]; !ok { tagIds = append(tagIds, v.ChartXTagId) tagIdsMap[v.ChartXTagId] = struct{}{} } if _, ok := tagIdsMap[v.ChartYTagId]; !ok { tagIds = append(tagIds, v.ChartYTagId) tagIdsMap[v.ChartYTagId] = struct{}{} } if chartCross, ok := tagChartMap[v.ChartXTagId]; ok { chartCross = append(chartCross, v) tagChartMap[v.ChartXTagId] = chartCross } else { chartCross = make([]*cross_variety.ChartInfoCrossVariety, 0) chartCross = append(chartCross, v) tagChartMap[v.ChartXTagId] = chartCross } if chartCross, ok := tagChartMap[v.ChartYTagId]; ok { chartCross = append(chartCross, v) tagChartMap[v.ChartYTagId] = chartCross } else { chartCross = make([]*cross_variety.ChartInfoCrossVariety, 0) chartCross = append(chartCross, v) tagChartMap[v.ChartYTagId] = chartCross } } chartInfoMap := make(map[int]*data_manage.ChartInfo) if len(chartIds) > 0 { // 查询图表信息 chartInfoList, e := data_manage.GetChartInfoByChartInfoIds(chartIds) if e != nil { err = fmt.Errorf("查询图表信息列表失败 Err:%s", e) return } for _, v := range chartInfoList { chartInfoMap[v.ChartInfoId] = v } } chartTagVarietyList, e := cross_variety.GetChartTagVarietyEdbInfoIdsByTagIds(tagIds) if e != nil { err = fmt.Errorf("查询指标信息列表失败 Err:%s", e) return } edbInfoIds := make([]int, 0) chartTagVarietyMap := make(map[int][]*cross_variety.ChartTagVariety) for _, v := range chartTagVarietyList { if tagList, ok := chartTagVarietyMap[v.EdbInfoId]; ok { tagList = append(tagList, v) chartTagVarietyMap[v.EdbInfoId] = tagList } else { tagList = make([]*cross_variety.ChartTagVariety, 0) tagList = append(tagList, v) chartTagVarietyMap[v.EdbInfoId] = tagList } edbInfoIds = append(edbInfoIds, v.EdbInfoId) } // 查询指标信息表 edbInfoList, e := data_manage.GetEdbInfoByIdList(edbInfoIds) if e != nil { err = fmt.Errorf("查询指标信息列表失败 Err:%s", e) return } if len(edbInfoList) == 0 { continue } // 查询计算指标信息，并且建立关联关系 // 查询间接引用的指标信息 calculateEdbMappingListMap, calculateEdbMappingIdsMap, e := GetEdbListByEdbInfoId(edbInfoList) if e != nil { err = fmt.Errorf("查询计算指标信息失败，%s", e.Error()) return } edbInfoMap := make(map[int]*data_manage.EdbInfo) chartInfoCrossMap := make(map[int]struct{}) chartInfoCrossList := make([]*cross_variety.ChartInfoCrossVariety, 0) edbCrossMap := make(map[int][]*cross_variety.ChartInfoCrossVariety) for _, v := range edbInfoList { edbInfoMap[v.EdbInfoId] = v if tagList, ok := chartTagVarietyMap[v.EdbInfoId]; ok { for _, tag := range tagList { if chartCross, ok2 := tagChartMap[tag.ChartTagId]; ok2 { for _, crossItem := range chartCross { if _, ok3 := chartInfoCrossMap[crossItem.ChartInfoId]; !ok3 { chartInfo, chartOk := chartInfoMap[crossItem.ChartInfoId] if !chartOk { //表示图表不存在 continue } // 查询真正有用的指标ID var config cross_variety.ChartConfigReq e := json.Unmarshal([]byte(chartInfo.ExtraConfig), &config) if e != nil { continue } if utils.InArrayByInt(config.VarietyList, tag.ChartVarietyId) { chartInfoCrossMap[crossItem.ChartInfoId] = struct{}{} chartInfoCrossList = append(chartInfoCrossList, crossItem) } } } } } } edbCrossMap[v.EdbInfoId] = chartInfoCrossList chartInfoCrossMap = make(map[int]struct{}) chartInfoCrossList = make([]*cross_variety.ChartInfoCrossVariety, 0) } //查询引用关系列表， chartEdbRelationList, e := data_manage.GetEdbInfoRelationByReferObjectIds(chartIds, utils.EDB_RELATION_CHART) if e != nil { err = fmt.Errorf("查询图表引用关系列表失败 Err:%s", e) return } existRelationMap := make(map[string]struct{}) for _, v := range chartEdbRelationList { name := fmt.Sprintf("%d-%d", v.ReferObjectId, v.EdbInfoId) existRelationMap[name] = struct{}{} } for edbInfoId, chartCrossList := range edbCrossMap { nowTime := time.Now() for _, item := range chartCrossList { name := fmt.Sprintf("%d-%d", item.ChartInfoId, edbInfoId) if _, ok1 := existRelationMap[name]; !ok1 { edbInfo, ok2 := edbInfoMap[edbInfoId] if !ok2 { continue } // 去掉预测指标 if edbInfo.EdbInfoType == 1 { continue } tmp := &data_manage.EdbInfoRelation{ ReferObjectId: item.ChartInfoId, ReferObjectType: utils.EDB_RELATION_CHART, ReferObjectSubType: utils.CHART_SOURCE_CROSS_HEDGING, EdbInfoId: edbInfoId, EdbName: edbInfo.EdbName, Source: edbInfo.Source, EdbCode: edbInfo.EdbCode, CreateTime: nowTime, ModifyTime: nowTime, RelationTime: item.CreateTime, } tmp.RelationCode = fmt.Sprintf("%d_%d_%d_%d", tmp.EdbInfoId, tmp.ReferObjectId, tmp.ReferObjectType, tmp.ReferObjectSubType) addList = append(addList, tmp) existRelationMap[name] = struct{}{} // 添加间接引用记录 if edbInfo.EdbType == 2 && edbInfo.EdbInfoType == 0 { childEdbMappingIds, ok1 := calculateEdbMappingIdsMap[edbInfo.EdbInfoId] if !ok1 { continue } for _, childEdbMappingId := range childEdbMappingIds { childEdbMapping, ok2 := calculateEdbMappingListMap[childEdbMappingId] if !ok2 { continue } name1 := fmt.Sprintf("%d-%d", item.ChartInfoId, childEdbMapping.FromEdbInfoId) if _, ok2 := existRelationMap[name1]; ok2 { //如果已经被直接引用了，则无需添加到间接引用记录中 continue } tmp1 := &data_manage.EdbInfoRelation{ ReferObjectId: item.ChartInfoId, ReferObjectType: utils.EDB_RELATION_CHART, ReferObjectSubType: utils.CHART_SOURCE_CROSS_HEDGING, EdbInfoId: childEdbMapping.FromEdbInfoId, EdbName: childEdbMapping.FromEdbName, Source: childEdbMapping.FromSource, EdbCode: childEdbMapping.FromEdbCode, CreateTime: nowTime, ModifyTime: nowTime, RelationTime: item.CreateTime, RelationType: 1, RootEdbInfoId: edbInfo.EdbInfoId, ChildEdbInfoId: childEdbMapping.EdbInfoId, RelationCode: tmp.RelationCode, } addList = append(addList, tmp1) // todo 防止重复 } } if len(addList) > pageSize { err = data_manage.AddEdbInfoRelationMulti(addList) if err != nil { err = fmt.Errorf("新增引用记录失败 Err:%s", err) return } addNum += len(addList) addList = make([]*data_manage.EdbInfoRelation, 0) } } } } } //拿到500个数据ID，判断相关的引用记录，如果已存在则直接过滤，不存在则新增 if len(addList) > 0 { err = data_manage.AddEdbInfoRelationMulti(addList) if err != nil { err = fmt.Errorf("新增引用记录失败 Err:%s", err) return } addNum += len(addList) } fmt.Printf("跨品种分析图表指标引用记录处理完成, 新增%d条记录\n", addNum) return } // 初始化事件日历中的指标引用 func InitCalendarIndicatorRelation() { fmt.Println("开始处理事件日历中的指标引用") var addNum int var err error defer func() { if err != nil { msg := fmt.Sprintf("初始化指标在事件日历中的引用失败 initCalendarIndicatorRelation err: %v", err) utils.FileLog.Info(msg) fmt.Println(msg) go alarm_msg.SendAlarmMsg(msg, 3) } }() //查询chart_edb_mapping 表 obj := new(fe_calendar.FeCalendarMatter) condition := " AND edb_info_id > 0" total, err := obj.GetCountByCondition(condition, []interface{}{}) if err != nil { err = fmt.Errorf("查询事件日历关联指标失败 err: %v", err) return } if total == 0 { return } //分页查询，每次处理500条记录 pageSize := 500 totalPage := (total + pageSize - 1) / pageSize // 使用整数除法，并添加一页以防有余数 addList := make([]*data_manage.EdbInfoRelation, 0) //查询图表列表 for i := 0; i < totalPage; i += 1 { startSize := i * pageSize list, e := obj.GetPageItemsByCondition(condition, []interface{}{}, []string{}, "", startSize, pageSize) if e != nil { err = fmt.Errorf("查询事件日历关联指标列表失败 Err:%s", e) return } if len(list) == 0 { break } edbInfoIds := make([]int, 0) edbInfoMatterMap := make(map[int][]*fe_calendar.FeCalendarMatter) for _, v := range list { edbInfoIds = append(edbInfoIds, v.EdbInfoId) items, ok := edbInfoMatterMap[v.EdbInfoId] if ok { items = append(items, v) edbInfoMatterMap[v.EdbInfoId] = items } else { items = make([]*fe_calendar.FeCalendarMatter, 0) items = append(items, v) edbInfoMatterMap[v.EdbInfoId] = items } } // 查询指标信息表 edbInfoList, e := data_manage.GetEdbInfoByIdList(edbInfoIds) if e != nil { err = fmt.Errorf("查询指标信息列表失败 Err:%s", e) return } if len(edbInfoList) == 0 { continue } // 查询计算指标信息，并且建立关联关系 // 查询间接引用的指标信息 calculateEdbMappingListMap, calculateEdbMappingIdsMap, e := GetEdbListByEdbInfoId(edbInfoList) if e != nil { err = fmt.Errorf("查询计算指标信息失败，%s", e.Error()) return } edbInfoMap := make(map[int]*data_manage.EdbInfo) matterIds := make([]int, 0) for _, v := range edbInfoList { edbInfoMap[v.EdbInfoId] = v items, ok := edbInfoMatterMap[v.EdbInfoId] if ok { for _, item := range items { matterIds = append(matterIds, item.FeCalendarMatterId) } } } //查询引用关系列表， chartEdbRelationList, e := data_manage.GetEdbInfoRelationByReferObjectIds(matterIds, utils.EDB_RELATION_CALENDAR) if e != nil { err = fmt.Errorf("查询图表引用关系列表失败 Err:%s", e) return } existRelationMap := make(map[string]struct{}) for _, v := range chartEdbRelationList { name := fmt.Sprintf("%d-%d", v.ReferObjectId, v.EdbInfoId) existRelationMap[name] = struct{}{} } for edbInfoId, edbInfo := range edbInfoMap { // 去掉预测指标 if edbInfo.EdbInfoType == 1 { continue } nowTime := time.Now() items, ok := edbInfoMatterMap[edbInfoId] if ok { for _, v := range items { name := fmt.Sprintf("%d-%d", v.FeCalendarMatterId, v.EdbInfoId) if _, ok1 := existRelationMap[name]; !ok1 { //todo 引用时间 tmp := &data_manage.EdbInfoRelation{ ReferObjectId: v.FeCalendarMatterId, ReferObjectType: utils.EDB_RELATION_CALENDAR, EdbInfoId: v.EdbInfoId, EdbName: edbInfo.EdbName, Source: edbInfo.Source, EdbCode: edbInfo.EdbCode, CreateTime: nowTime, ModifyTime: nowTime, RelationTime: v.CreateTime, } tmp.RelationCode = fmt.Sprintf("%d_%d_%d_%d", tmp.EdbInfoId, tmp.ReferObjectId, tmp.ReferObjectType, tmp.ReferObjectSubType) addList = append(addList, tmp) existRelationMap[name] = struct{}{} // 添加间接引用记录 if edbInfo.EdbType == 2 && edbInfo.EdbInfoType == 0 { childEdbMappingIds, ok1 := calculateEdbMappingIdsMap[edbInfo.EdbInfoId] if !ok1 { continue } for _, childEdbMappingId := range childEdbMappingIds { childEdbMapping, ok2 := calculateEdbMappingListMap[childEdbMappingId] if !ok2 { continue } name1 := fmt.Sprintf("%d-%d", v.FeCalendarMatterId, childEdbMapping.FromEdbInfoId) if _, ok2 := existRelationMap[name1]; ok2 { //如果已经被直接引用了，则无需添加到间接引用记录中 continue } tmp1 := &data_manage.EdbInfoRelation{ ReferObjectId: v.FeCalendarMatterId, ReferObjectType: utils.EDB_RELATION_CALENDAR, EdbInfoId: childEdbMapping.FromEdbInfoId, EdbName: childEdbMapping.FromEdbName, Source: childEdbMapping.FromSource, EdbCode: childEdbMapping.FromEdbCode, CreateTime: nowTime, ModifyTime: nowTime, RelationTime: v.CreateTime, RelationType: 1, RootEdbInfoId: edbInfo.EdbInfoId, ChildEdbInfoId: childEdbMapping.EdbInfoId, RelationCode: tmp.RelationCode, } addList = append(addList, tmp1) // todo 防止重复 } } if len(addList) > pageSize { err = data_manage.AddEdbInfoRelationMulti(addList) if err != nil { err = fmt.Errorf("新增引用记录失败 Err:%s", err) return } addNum += len(addList) addList = make([]*data_manage.EdbInfoRelation, 0) } } } } } } //拿到500个数据ID，判断相关的引用记录，如果已存在则直接过滤，不存在则新增 if len(addList) > 0 { err = data_manage.AddEdbInfoRelationMulti(addList) if err != nil { err = fmt.Errorf("新增引用记录失败 Err:%s", err) return } addNum += len(addList) } fmt.Printf("事件日历指标引用记录处理完成, 新增%d条记录\n", addNum) return } // 初始化表格中的指标引用 func InitExcelEdbRelation() { fmt.Println("开始处理表格中的指标引用") var err error var addNum int defer func() { if err != nil { msg := fmt.Sprintf("初始化指标在表格中的引用失败 InitChartEdbRelation err: %v", err) utils.FileLog.Info(msg) fmt.Println(msg) go alarm_msg.SendAlarmMsg(msg, 3) } }() //查询表格指标绑定表 sources := []int{utils.TIME_TABLE, utils.MIXED_TABLE, utils.BALANCE_TABLE} total, err := excel.GetExcelEdbMappingTotalBySource(sources) if err != nil { err = fmt.Errorf("查询表格关联指标失败 err: %v", err) return } if total == 0 { return } //分页查询，每次处理100条记录 pageSize := 100 totalPage := (total + pageSize - 1) / pageSize // 使用整数除法，并添加一页以防有余数 addList := make([]*data_manage.EdbInfoRelation, 0) //查询表格列表 for i := 0; i < totalPage; i += 1 { startSize := i * pageSize list, e := excel.GetExcelEdbMappingListBySource(sources, startSize, pageSize) if e != nil { err = fmt.Errorf("查询表格关联指标列表失败 Err:%s", e) return } if len(list) == 0 { break } edbInfoIds := make([]int, 0) for _, v := range list { edbInfoIds = append(edbInfoIds, v.EdbInfoId) } // 查询指标信息表 edbInfoList, e := data_manage.GetEdbInfoByIdList(edbInfoIds) if e != nil { err = fmt.Errorf("查询指标信息列表失败 Err:%s", e) return } if len(edbInfoList) == 0 { continue } // 查询计算指标信息，并且建立关联关系 // 查询间接引用的指标信息 calculateEdbMappingListMap, calculateEdbMappingIdsMap, e := GetEdbListByEdbInfoId(edbInfoList) if e != nil { err = fmt.Errorf("查询计算指标信息失败，%s", e.Error()) return } // 查询指标间接引用 edbInfoMap := make(map[int]*data_manage.EdbInfo) for _, v := range edbInfoList { edbInfoMap[v.EdbInfoId] = v } // 筛选有用的表格 excelIds := make([]int, 0) for _, v := range list { excelIds = append(excelIds, v.ExcelInfoId) } //查询引用关系列表， chartEdbRelationList, e := data_manage.GetEdbInfoRelationByReferObjectIds(excelIds, utils.EDB_RELATION_TABLE) if e != nil { err = fmt.Errorf("查询表格引用关系列表失败 Err:%s", e) return } existRelationMap := make(map[string]struct{}) for _, v := range chartEdbRelationList { name := fmt.Sprintf("%d-%d", v.ReferObjectId, v.EdbInfoId) existRelationMap[name] = struct{}{} } for _, v := range list { nowTime := time.Now() name := fmt.Sprintf("%d-%d", v.ExcelInfoId, v.EdbInfoId) if _, ok := existRelationMap[name]; !ok { edbInfo, ok2 := edbInfoMap[v.EdbInfoId] if !ok2 { continue } // 去掉预测指标 if edbInfo.EdbInfoType == 1 { continue } tmp := &data_manage.EdbInfoRelation{ ReferObjectId: v.ExcelInfoId, ReferObjectType: utils.EDB_RELATION_TABLE, ReferObjectSubType: v.Source, EdbInfoId: v.EdbInfoId, EdbName: edbInfo.EdbName, Source: edbInfo.Source, EdbCode: edbInfo.EdbCode, CreateTime: nowTime, ModifyTime: nowTime, RelationTime: v.CreateTime, } tmp.RelationCode = fmt.Sprintf("%d_%d_%d_%d", tmp.EdbInfoId, tmp.ReferObjectId, tmp.ReferObjectType, tmp.ReferObjectSubType) addList = append(addList, tmp) existRelationMap[name] = struct{}{} // 添加间接引用记录 if edbInfo.EdbType == 2 && edbInfo.EdbInfoType == 0 { childEdbMappingIds, ok1 := calculateEdbMappingIdsMap[edbInfo.EdbInfoId] if !ok1 { continue } for _, childEdbMappingId := range childEdbMappingIds { childEdbMapping, ok2 := calculateEdbMappingListMap[childEdbMappingId] if !ok2 { continue } name1 := fmt.Sprintf("%d-%d", v.ExcelInfoId, childEdbMapping.FromEdbInfoId) if _, ok2 := existRelationMap[name1]; ok2 { //如果已经被直接引用了，则无需添加到间接引用记录中 continue } tmp1 := &data_manage.EdbInfoRelation{ ReferObjectId: v.ExcelInfoId, ReferObjectType: utils.EDB_RELATION_TABLE, ReferObjectSubType: v.Source, EdbInfoId: childEdbMapping.FromEdbInfoId, EdbName: childEdbMapping.FromEdbName, Source: childEdbMapping.FromSource, EdbCode: childEdbMapping.FromEdbCode, CreateTime: nowTime, ModifyTime: nowTime, RelationTime: v.CreateTime, RelationType: 1, RootEdbInfoId: edbInfo.EdbInfoId, ChildEdbInfoId: childEdbMapping.EdbInfoId, RelationCode: tmp.RelationCode, } addList = append(addList, tmp1) // todo 防止重复 } } if len(addList) > pageSize { err = data_manage.AddEdbInfoRelationMulti(addList) if err != nil { err = fmt.Errorf("新增引用记录失败 Err:%s", err) return } addNum += len(addList) addList = make([]*data_manage.EdbInfoRelation, 0) } } } } //拿到500个数据ID，判断相关的引用记录，如果已存在则直接过滤，不存在则新增 if len(addList) > 0 { err = data_manage.AddEdbInfoRelationMulti(addList) if err != nil { err = fmt.Errorf("新增引用记录失败 Err:%s", err) return } addNum += len(addList) } fmt.Printf("表格指标引用记录处理完成, 新增%d条记录\n", addNum) return } // 处理逻辑图中的指标引用 func InitSandBoxEdbRelation() { fmt.Println("开始处理逻辑图中的指标引用") var err error var addNum int defer func() { if err != nil { msg := fmt.Sprintf("初始化指标在逻辑图中的引用失败 initSandBoxEdbRelation err: %v", err) utils.FileLog.Info(msg) fmt.Println(msg) go alarm_msg.SendAlarmMsg(msg, 3) } }() condition := " AND is_delete = 0" total, err := sandbox.GetSandboxListCountByCondition(condition, []interface{}{}) if err != nil { err = fmt.Errorf("查询逻辑图总数失败 err: %v", err) return } if total == 0 { return } //分页查询，每次处理500条记录 pageSize := 100 totalPage := (total + pageSize - 1) / pageSize // 使用整数除法，并添加一页以防有余数 addList := make([]*data_manage.EdbInfoRelation, 0) //查询图表列表 for i := 0; i < totalPage; i += 1 { startSize := i * pageSize list, e := sandbox.GetSandboxListByCondition(condition, []interface{}{}, startSize, pageSize) if e != nil { err = fmt.Errorf("查询逻辑图列表失败 Err:%s", e) return } if len(list) == 0 { break } edbInfoIds := make([]int, 0) edbSandboxMap := make(map[int][]*sandbox.Sandbox) for _, v := range list { if v.Content == "" { continue } edbInfoIdsTmp, e := getSandBoxEdbIdsByContent(v.Content) if e != nil { continue //err = fmt.Errorf("查询逻辑图关联的指标Id失败 Err:%s", e) //return } for _, edbId := range edbInfoIdsTmp { edbInfoIds = append(edbInfoIds, edbId) edbSandboxMap[edbId] = append(edbSandboxMap[edbId], v) } } if len(edbInfoIds) <= 0 { continue } // 查询指标信息表 edbInfoList, e := data_manage.GetEdbInfoByIdList(edbInfoIds) if e != nil { err = fmt.Errorf("查询指标信息列表失败 Err:%s", e) return } if len(edbInfoList) == 0 { continue } // 查询计算指标信息，并且建立关联关系 // 查询间接引用的指标信息 calculateEdbMappingListMap, calculateEdbMappingIdsMap, e := GetEdbListByEdbInfoId(edbInfoList) if e != nil { err = fmt.Errorf("查询计算指标信息失败，%s", e.Error()) return } edbInfoMap := make(map[int]*data_manage.EdbInfo) sandboxIds := make([]int, 0) for _, v := range edbInfoList { edbInfoMap[v.EdbInfoId] = v if items, ok := edbSandboxMap[v.EdbInfoId]; ok { for _, item := range items { sandboxIds = append(sandboxIds, item.SandboxId) } } } //查询引用关系列表， chartEdbRelationList, e := data_manage.GetEdbInfoRelationByReferObjectIds(sandboxIds, utils.EDB_RELATION_SANDBOX) if e != nil { err = fmt.Errorf("查询逻辑图引用关系列表失败 Err:%s", e) return } existRelationMap := make(map[string]struct{}) for _, v := range chartEdbRelationList { name := fmt.Sprintf("%d-%d", v.ReferObjectId, v.EdbInfoId) existRelationMap[name] = struct{}{} } for edbInfoId, sandboxList := range edbSandboxMap { nowTime := time.Now() for _, v := range sandboxList { name := fmt.Sprintf("%d-%d", v.SandboxId, edbInfoId) if _, ok := existRelationMap[name]; !ok { edbInfo, ok2 := edbInfoMap[edbInfoId] if !ok2 { continue } // 去掉预测指标 if edbInfo.EdbInfoType == 1 { continue } tmp := &data_manage.EdbInfoRelation{ ReferObjectId: v.SandboxId, ReferObjectType: utils.EDB_RELATION_SANDBOX, EdbInfoId: edbInfoId, EdbName: edbInfo.EdbName, Source: edbInfo.Source, EdbCode: edbInfo.EdbCode, CreateTime: nowTime, ModifyTime: nowTime, RelationTime: v.CreateTime, } tmp.RelationCode = fmt.Sprintf("%d_%d_%d_%d", tmp.EdbInfoId, tmp.ReferObjectId, tmp.ReferObjectType, tmp.ReferObjectSubType) addList = append(addList, tmp) existRelationMap[name] = struct{}{} // 添加间接引用记录 if edbInfo.EdbType == 2 && edbInfo.EdbInfoType == 0 { childEdbMappingIds, ok1 := calculateEdbMappingIdsMap[edbInfo.EdbInfoId] if !ok1 { continue } for _, childEdbMappingId := range childEdbMappingIds { childEdbMapping, ok2 := calculateEdbMappingListMap[childEdbMappingId] if !ok2 { continue } name1 := fmt.Sprintf("%d-%d", v.SandboxId, childEdbMapping.FromEdbInfoId) if _, ok2 := existRelationMap[name1]; ok2 { //如果已经被直接引用了，则无需添加到间接引用记录中 continue } tmp1 := &data_manage.EdbInfoRelation{ ReferObjectId: v.SandboxId, ReferObjectType: utils.EDB_RELATION_SANDBOX, EdbInfoId: childEdbMapping.FromEdbInfoId, EdbName: childEdbMapping.FromEdbName, Source: childEdbMapping.FromSource, EdbCode: childEdbMapping.FromEdbCode, CreateTime: nowTime, ModifyTime: nowTime, RelationTime: v.CreateTime, RelationType: 1, RootEdbInfoId: edbInfo.EdbInfoId, ChildEdbInfoId: childEdbMapping.EdbInfoId, RelationCode: tmp.RelationCode, } addList = append(addList, tmp1) // todo 防止重复 } } if len(addList) > pageSize { err = data_manage.AddEdbInfoRelationMulti(addList) if err != nil { err = fmt.Errorf("新增引用记录失败 Err:%s", err) return } addNum += len(addList) addList = make([]*data_manage.EdbInfoRelation, 0) } } } } } //拿到500个数据ID，判断相关的引用记录，如果已存在则直接过滤，不存在则新增 if len(addList) > 0 { err = data_manage.AddEdbInfoRelationMulti(addList) if err != nil { err = fmt.Errorf("新增引用记录失败 Err:%s", err) return } addNum += len(addList) } fmt.Printf("逻辑图指标引用记录处理完成, 新增%d条记录\n", addNum) return } func getSandBoxEdbIdsByContent(content string) (edbInfoIds []int, err error) { var contentInfo sandbox.ContentDataStruct err = json.Unmarshal([]byte(content), &contentInfo) if err != nil { err = fmt.Errorf("json.Unmarshal err:%s", err.Error()) return } // 遍历所有节点 for _, node := range contentInfo.Cells { if node.Data == nil { continue } for _, v := range node.Data.LinkData { if v.Type == 1 { edbInfoIds = append(edbInfoIds, v.Id) } } } return } func GetEdbListByEdbInfoId(edbInfoList []*data_manage.EdbInfo) (edbMappingListMap map[int]*data_manage.EdbInfoCalculateMapping, edbInfoMappingRootIdsMap map[int][]int, err error) { if len(edbInfoList) == 0 { return } edbInfoIds := make([]int, 0) for _, v := range edbInfoList { if v.EdbType == 2 && v.EdbInfoType == 0 { //普通计算指标，排除预算指标 edbInfoIds = append(edbInfoIds, v.EdbInfoId) } } if len(edbInfoIds) == 0 { return } //查询指标信息 allEdbMappingMap := make(map[int][]*data_manage.EdbInfoCalculateMappingInfo, 0) allMappingList, e := data_manage.GetEdbInfoCalculateMappingListByEdbInfoIds(edbInfoIds) if e != nil { err = fmt.Errorf("GetEdbInfoCalculateMappingListByEdbInfoIds err: %s", e.Error()) return } for _, v := range allMappingList { if _, ok := allEdbMappingMap[v.EdbInfoId]; !ok { allEdbMappingMap[v.EdbInfoId] = make([]*data_manage.EdbInfoCalculateMappingInfo, 0) } allEdbMappingMap[v.EdbInfoId] = append(allEdbMappingMap[v.EdbInfoId], v) } //查询指标映射 //查询所有指标数据 //查询这个指标相关的mapping信息放到数组里， //将得到的指标ID信息放到数组里 hasFindMap := make(map[int]struct{}) edbInfoIdMap := make(map[int]struct{}) edbMappingList := make([]*data_manage.EdbInfoCalculateMapping, 0) edbInfoMappingRootIdsMap = make(map[int][]int, 0) edbMappingMap := make(map[int]struct{}) for _, edbInfo := range edbInfoList { if edbInfo.EdbType == 2 && edbInfo.EdbInfoType == 0 { edbInfoId := edbInfo.EdbInfoId edbMappingList, err = getCalculateEdbInfoByEdbInfoId(allEdbMappingMap, edbInfoId, hasFindMap, edbInfoIdMap, edbMappingList, edbMappingMap, edbInfoMappingRootIdsMap, edbInfoId) if err != nil { err = fmt.Errorf(" GetCalculateEdbInfoByEdbInfoId err: %s", err.Error()) return } } } if len(edbMappingList) == 0 { return } // 查询指标信息 // 指标信息map edbInfoIdList := make([]int, 0) for k, _ := range edbInfoIdMap { edbInfoIdList = append(edbInfoIdList, k) } edbMappingListMap = make(map[int]*data_manage.EdbInfoCalculateMapping) if len(edbMappingList) > 0 { for _, v := range edbMappingList { edbMappingListMap[v.EdbInfoCalculateMappingId] = v } } return } // getCalculateEdbInfoByEdbInfoId 计算指标追溯 func getCalculateEdbInfoByEdbInfoId(allEdbMappingMap map[int][]*data_manage.EdbInfoCalculateMappingInfo, edbInfoId int, hasFindMap map[int]struct{}, edbInfoIdMap map[int]struct{}, edbMappingList []*data_manage.EdbInfoCalculateMapping, edbMappingMap map[int]struct{}, edbInfoMappingRootIdsMap map[int][]int, rootEdbInfoId int) (newEdbMappingList []*data_manage.EdbInfoCalculateMapping, err error) { newEdbMappingList = edbMappingList _, ok := hasFindMap[edbInfoId] if ok { return } if _, ok1 := edbInfoIdMap[edbInfoId]; !ok1 { edbInfoIdMap[edbInfoId] = struct{}{} } edbInfoMappingList := make([]*data_manage.EdbInfoCalculateMappingInfo, 0) edbInfoMappingList, ok = allEdbMappingMap[edbInfoId] if !ok { edbInfoMappingList, err = data_manage.GetEdbInfoCalculateMappingListByEdbInfoId(edbInfoId) if err != nil { err = fmt.Errorf("GetEdbInfoCalculateMappingListByEdbInfoId err: %s", err.Error()) return } } hasFindMap[edbInfoId] = struct{}{} if len(edbInfoMappingList) > 0 { fromEdbInfoIdList := make([]int, 0) edbInfoMappingIdList := make([]int, 0) for _, v := range edbInfoMappingList { fromEdbInfoIdList = append(fromEdbInfoIdList, v.FromEdbInfoId) edbInfoMappingIdList = append(edbInfoMappingIdList, v.EdbInfoCalculateMappingId) if _, ok1 := edbInfoIdMap[v.FromEdbInfoId]; !ok1 { edbInfoIdMap[v.FromEdbInfoId] = struct{}{} } if _, ok2 := edbMappingMap[v.EdbInfoCalculateMappingId]; !ok2 { edbMappingMap[v.EdbInfoCalculateMappingId] = struct{}{} tmp := &data_manage.EdbInfoCalculateMapping{ EdbInfoCalculateMappingId: v.EdbInfoCalculateMappingId, EdbInfoId: v.EdbInfoId, Source: v.Source, SourceName: v.SourceName, EdbCode: v.EdbCode, FromEdbInfoId: v.FromEdbInfoId, FromEdbCode: v.FromEdbCode, FromEdbName: v.FromEdbName, FromSource: v.FromSource, FromSourceName: v.FromSourceName, FromTag: v.FromTag, Sort: v.Sort, CreateTime: v.CreateTime, ModifyTime: v.ModifyTime, } newEdbMappingList = append(newEdbMappingList, tmp) } if edbInfoId != v.FromEdbInfoId && (v.FromEdbType == 2 || v.FromEdbInfoType == 1) { // 查过了就不查了 if _, ok2 := hasFindMap[v.FromEdbInfoId]; !ok2 { newEdbMappingList, err = getCalculateEdbInfoByEdbInfoId(allEdbMappingMap, v.FromEdbInfoId, hasFindMap, edbInfoIdMap, newEdbMappingList, edbMappingMap, edbInfoMappingRootIdsMap, rootEdbInfoId) if err != nil { err = fmt.Errorf("traceEdbInfoByEdbInfoId err: %s", err.Error()) return } } } hasFindMap[v.FromEdbInfoId] = struct{}{} } edbInfoMappingRootIdsMap[rootEdbInfoId] = append(edbInfoMappingRootIdsMap[rootEdbInfoId], edbInfoMappingIdList...) } return }