|
|
@@ -0,0 +1,1202 @@
|
|
|
+package services
|
|
|
+
|
|
|
+import (
|
|
|
+ "encoding/json"
|
|
|
+ "eta/eta_task/models/data_manage"
|
|
|
+ "eta/eta_task/models/data_manage/cross_variety"
|
|
|
+ "eta/eta_task/models/data_manage/excel"
|
|
|
+ "eta/eta_task/models/fe_calendar"
|
|
|
+ "eta/eta_task/models/sandbox"
|
|
|
+ "eta/eta_task/services/alarm_msg"
|
|
|
+ "eta/eta_task/utils"
|
|
|
+ "fmt"
|
|
|
+ "time"
|
|
|
+)
|
|
|
+
|
|
|
+func InitChartEdbRelation() {
|
|
|
+ fmt.Println("开始处理图表中的指标引用")
|
|
|
+ var err error
|
|
|
+ var addNum int
|
|
|
+ defer func() {
|
|
|
+ if err != nil {
|
|
|
+ msg := fmt.Sprintf("初始化指标在图表中的引用失败 InitChartEdbRelation err: %v", err)
|
|
|
+ utils.FileLog.Info(msg)
|
|
|
+ fmt.Println(msg)
|
|
|
+ go alarm_msg.SendAlarmMsg(msg, 3)
|
|
|
+ }
|
|
|
+ }()
|
|
|
+ //查询chart_edb_mapping 表
|
|
|
+ total, err := data_manage.GetChartEdbMappingTotal()
|
|
|
+ if err != nil {
|
|
|
+ err = fmt.Errorf("查询图表关联指标失败 err: %v", err)
|
|
|
+ return
|
|
|
+ }
|
|
|
+ if total == 0 {
|
|
|
+ return
|
|
|
+ }
|
|
|
+ //分页查询，每次处理500条记录
|
|
|
+ pageSize := 500
|
|
|
+ totalPage := (total + pageSize - 1) / pageSize // 使用整数除法，并添加一页以防有余数
|
|
|
+ addList := make([]*data_manage.EdbInfoRelation, 0)
|
|
|
+ //查询图表列表
|
|
|
+ for i := 0; i < totalPage; i += 1 {
|
|
|
+ startSize := i * pageSize
|
|
|
+ list, e := data_manage.GetChartEdbMappingList(startSize, pageSize)
|
|
|
+ if e != nil {
|
|
|
+ err = fmt.Errorf("查询图表关联指标列表失败 Err:%s", e)
|
|
|
+ return
|
|
|
+ }
|
|
|
+ if len(list) == 0 {
|
|
|
+ break
|
|
|
+ }
|
|
|
+
|
|
|
+ edbInfoIds := make([]int, 0)
|
|
|
+ for _, v := range list {
|
|
|
+ edbInfoIds = append(edbInfoIds, v.EdbInfoId)
|
|
|
+ }
|
|
|
+ // 查询指标信息表
|
|
|
+ edbInfoList, e := data_manage.GetEdbInfoByIdList(edbInfoIds)
|
|
|
+ if e != nil {
|
|
|
+ err = fmt.Errorf("查询指标信息列表失败 Err:%s", e)
|
|
|
+ return
|
|
|
+ }
|
|
|
+ if len(edbInfoList) == 0 {
|
|
|
+ continue
|
|
|
+ }
|
|
|
+ // 查询计算指标信息，并且建立关联关系
|
|
|
+ // 查询间接引用的指标信息
|
|
|
+ calculateEdbMappingListMap, calculateEdbMappingIdsMap, e := GetEdbListByEdbInfoId(edbInfoList)
|
|
|
+ if e != nil {
|
|
|
+ err = fmt.Errorf("查询计算指标信息失败，%s", e.Error())
|
|
|
+ return
|
|
|
+ }
|
|
|
+ // 查询指标间接引用
|
|
|
+ edbInfoMap := make(map[int]*data_manage.EdbInfo)
|
|
|
+ for _, v := range edbInfoList {
|
|
|
+ edbInfoMap[v.EdbInfoId] = v
|
|
|
+ }
|
|
|
+
|
|
|
+ // 筛选有用的图表
|
|
|
+ finalList := make([]*data_manage.ChartEdbMapping, 0)
|
|
|
+ chartIds := make([]int, 0)
|
|
|
+ for _, v := range list {
|
|
|
+ if _, ok2 := edbInfoMap[v.EdbInfoId]; !ok2 {
|
|
|
+ continue
|
|
|
+ }
|
|
|
+ finalList = append(finalList, v)
|
|
|
+ chartIds = append(chartIds, v.ChartInfoId)
|
|
|
+ }
|
|
|
+ if len(chartIds) == 0 {
|
|
|
+ continue
|
|
|
+ }
|
|
|
+ // 查询图表信息
|
|
|
+ chartInfoList, e := data_manage.GetChartInfoByChartInfoIds(chartIds)
|
|
|
+ if e != nil {
|
|
|
+ err = fmt.Errorf("查询图表信息列表失败 Err:%s", e)
|
|
|
+ return
|
|
|
+ }
|
|
|
+ chartInfoMap := make(map[int]*data_manage.ChartInfo)
|
|
|
+ for _, v := range chartInfoList {
|
|
|
+ chartInfoMap[v.ChartInfoId] = v
|
|
|
+ }
|
|
|
+
|
|
|
+ //查询引用关系列表，
|
|
|
+ chartEdbRelationList, e := data_manage.GetEdbInfoRelationByReferObjectIds(chartIds, utils.EDB_RELATION_CHART)
|
|
|
+ if e != nil {
|
|
|
+ err = fmt.Errorf("查询图表引用关系列表失败 Err:%s", e)
|
|
|
+ return
|
|
|
+ }
|
|
|
+ existRelationMap := make(map[string]struct{})
|
|
|
+ for _, v := range chartEdbRelationList {
|
|
|
+ name := fmt.Sprintf("%d-%d", v.ReferObjectId, v.EdbInfoId)
|
|
|
+ existRelationMap[name] = struct{}{}
|
|
|
+ }
|
|
|
+ for _, v := range finalList {
|
|
|
+ nowTime := time.Now()
|
|
|
+ name := fmt.Sprintf("%d-%d", v.ChartInfoId, v.EdbInfoId)
|
|
|
+ if _, ok := existRelationMap[name]; !ok {
|
|
|
+ //查询图表信息
|
|
|
+ chartInfo, ok1 := chartInfoMap[v.ChartInfoId]
|
|
|
+ if !ok1 {
|
|
|
+ continue
|
|
|
+ }
|
|
|
+ if chartInfo.Source == utils.CHART_SOURCE_CROSS_HEDGING { //过滤掉跨品种分析的图表
|
|
|
+ continue
|
|
|
+ }
|
|
|
+ edbInfo, ok2 := edbInfoMap[v.EdbInfoId]
|
|
|
+ if !ok2 {
|
|
|
+ continue
|
|
|
+ }
|
|
|
+ // 去掉预测指标
|
|
|
+ if edbInfo.EdbInfoType == 1 {
|
|
|
+ continue
|
|
|
+ }
|
|
|
+
|
|
|
+ tmp := &data_manage.EdbInfoRelation{
|
|
|
+ ReferObjectId: v.ChartInfoId,
|
|
|
+ ReferObjectType: utils.EDB_RELATION_CHART,
|
|
|
+ ReferObjectSubType: chartInfo.Source,
|
|
|
+ EdbInfoId: v.EdbInfoId,
|
|
|
+ EdbName: edbInfo.EdbName,
|
|
|
+ Source: edbInfo.Source,
|
|
|
+ EdbCode: edbInfo.EdbCode,
|
|
|
+ CreateTime: nowTime,
|
|
|
+ ModifyTime: nowTime,
|
|
|
+ RelationTime: v.CreateTime,
|
|
|
+ }
|
|
|
+ tmp.RelationCode = fmt.Sprintf("%d_%d_%d_%d", tmp.EdbInfoId, tmp.ReferObjectId, tmp.ReferObjectType, tmp.ReferObjectSubType)
|
|
|
+ addList = append(addList, tmp)
|
|
|
+ existRelationMap[name] = struct{}{}
|
|
|
+ // 添加间接引用记录
|
|
|
+ if edbInfo.EdbType == 2 && edbInfo.EdbInfoType == 0 {
|
|
|
+ childEdbMappingIds, ok1 := calculateEdbMappingIdsMap[edbInfo.EdbInfoId]
|
|
|
+ if !ok1 {
|
|
|
+ continue
|
|
|
+ }
|
|
|
+ for _, childEdbMappingId := range childEdbMappingIds {
|
|
|
+ childEdbMapping, ok2 := calculateEdbMappingListMap[childEdbMappingId]
|
|
|
+ if !ok2 {
|
|
|
+ continue
|
|
|
+ }
|
|
|
+ name1 := fmt.Sprintf("%d-%d", v.ChartInfoId, childEdbMapping.FromEdbInfoId)
|
|
|
+ if _, ok2 := existRelationMap[name1]; ok2 { //如果已经被直接引用了，则无需添加到间接引用记录中
|
|
|
+ continue
|
|
|
+ }
|
|
|
+ tmp1 := &data_manage.EdbInfoRelation{
|
|
|
+ ReferObjectId: v.ChartInfoId,
|
|
|
+ ReferObjectType: utils.EDB_RELATION_CHART,
|
|
|
+ ReferObjectSubType: chartInfo.Source,
|
|
|
+ EdbInfoId: childEdbMapping.FromEdbInfoId,
|
|
|
+ EdbName: childEdbMapping.FromEdbName,
|
|
|
+ Source: childEdbMapping.FromSource,
|
|
|
+ EdbCode: childEdbMapping.FromEdbCode,
|
|
|
+ CreateTime: nowTime,
|
|
|
+ ModifyTime: nowTime,
|
|
|
+ RelationTime: v.CreateTime,
|
|
|
+ RelationType: 1,
|
|
|
+ RootEdbInfoId: edbInfo.EdbInfoId,
|
|
|
+ ChildEdbInfoId: childEdbMapping.EdbInfoId,
|
|
|
+ RelationCode: tmp.RelationCode,
|
|
|
+ }
|
|
|
+ addList = append(addList, tmp1)
|
|
|
+ // todo 防止重复
|
|
|
+ }
|
|
|
+ }
|
|
|
+
|
|
|
+ if len(addList) > pageSize {
|
|
|
+ err = data_manage.AddEdbInfoRelationMulti(addList)
|
|
|
+ if err != nil {
|
|
|
+ err = fmt.Errorf("新增引用记录失败 Err:%s", err)
|
|
|
+ return
|
|
|
+ }
|
|
|
+ addNum += len(addList)
|
|
|
+ addList = make([]*data_manage.EdbInfoRelation, 0)
|
|
|
+ }
|
|
|
+ }
|
|
|
+ }
|
|
|
+ }
|
|
|
+ //拿到500个数据ID，判断相关的引用记录，如果已存在则直接过滤，不存在则新增
|
|
|
+ if len(addList) > 0 {
|
|
|
+ err = data_manage.AddEdbInfoRelationMulti(addList)
|
|
|
+ if err != nil {
|
|
|
+ err = fmt.Errorf("新增引用记录失败 Err:%s", err)
|
|
|
+ return
|
|
|
+ }
|
|
|
+ addNum += len(addList)
|
|
|
+ }
|
|
|
+ fmt.Printf("图表指标引用记录处理完成, 新增%d条记录\n", addNum)
|
|
|
+ return
|
|
|
+}
|
|
|
+
|
|
|
+// InitChartCrossVariety 处理特殊图表，跨品种分析图表
|
|
|
+func InitChartCrossVariety() {
|
|
|
+ fmt.Println("开始跨品种分析图表中的指标引用")
|
|
|
+ var addNum int
|
|
|
+ var err error
|
|
|
+ defer func() {
|
|
|
+ if err != nil {
|
|
|
+ msg := fmt.Sprintf("初始化指标在跨品种分析图表中的引用失败 InitChartCrossVariety err: %v", err)
|
|
|
+ utils.FileLog.Info(msg)
|
|
|
+ fmt.Println(msg)
|
|
|
+ go alarm_msg.SendAlarmMsg(msg, 3)
|
|
|
+ }
|
|
|
+ }()
|
|
|
+ total, err := cross_variety.GetChartInfoCrossVarietyTotal()
|
|
|
+ if err != nil {
|
|
|
+ err = fmt.Errorf("查询图表关联指标失败 err: %v", err)
|
|
|
+ return
|
|
|
+ }
|
|
|
+ if total == 0 {
|
|
|
+ return
|
|
|
+ }
|
|
|
+ //分页查询，每次处理500条记录
|
|
|
+ pageSize := 500
|
|
|
+ totalPage := (total + pageSize - 1) / pageSize // 使用整数除法，并添加一页以防有余数
|
|
|
+ addList := make([]*data_manage.EdbInfoRelation, 0)
|
|
|
+ //查询图表列表
|
|
|
+ for i := 0; i < totalPage; i += 1 {
|
|
|
+ startSize := i * pageSize
|
|
|
+ list, e := cross_variety.GetChartInfoCrossVarietyList(startSize, pageSize)
|
|
|
+ if e != nil {
|
|
|
+ err = fmt.Errorf("查询图表关联指标列表失败 Err:%s", e)
|
|
|
+ return
|
|
|
+ }
|
|
|
+ if len(list) == 0 {
|
|
|
+ break
|
|
|
+ }
|
|
|
+ chartIds := make([]int, 0)
|
|
|
+ chartIdMap := make(map[int]struct{})
|
|
|
+ tagIds := make([]int, 0)
|
|
|
+ tagIdsMap := make(map[int]struct{})
|
|
|
+ tagChartMap := make(map[int][]*cross_variety.ChartInfoCrossVariety)
|
|
|
+ for _, v := range list {
|
|
|
+ if _, ok := chartIdMap[v.ChartInfoId]; !ok {
|
|
|
+ chartIdMap[v.ChartInfoId] = struct{}{}
|
|
|
+ chartIds = append(chartIds, v.ChartInfoId)
|
|
|
+ }
|
|
|
+ if _, ok := tagIdsMap[v.ChartXTagId]; !ok {
|
|
|
+ tagIds = append(tagIds, v.ChartXTagId)
|
|
|
+ tagIdsMap[v.ChartXTagId] = struct{}{}
|
|
|
+ }
|
|
|
+ if _, ok := tagIdsMap[v.ChartYTagId]; !ok {
|
|
|
+ tagIds = append(tagIds, v.ChartYTagId)
|
|
|
+ tagIdsMap[v.ChartYTagId] = struct{}{}
|
|
|
+ }
|
|
|
+ if chartCross, ok := tagChartMap[v.ChartXTagId]; ok {
|
|
|
+ chartCross = append(chartCross, v)
|
|
|
+ tagChartMap[v.ChartXTagId] = chartCross
|
|
|
+ } else {
|
|
|
+ chartCross = make([]*cross_variety.ChartInfoCrossVariety, 0)
|
|
|
+ chartCross = append(chartCross, v)
|
|
|
+ tagChartMap[v.ChartXTagId] = chartCross
|
|
|
+ }
|
|
|
+ if chartCross, ok := tagChartMap[v.ChartYTagId]; ok {
|
|
|
+ chartCross = append(chartCross, v)
|
|
|
+ tagChartMap[v.ChartYTagId] = chartCross
|
|
|
+ } else {
|
|
|
+ chartCross = make([]*cross_variety.ChartInfoCrossVariety, 0)
|
|
|
+ chartCross = append(chartCross, v)
|
|
|
+ tagChartMap[v.ChartYTagId] = chartCross
|
|
|
+ }
|
|
|
+ }
|
|
|
+ chartInfoMap := make(map[int]*data_manage.ChartInfo)
|
|
|
+ if len(chartIds) > 0 {
|
|
|
+ // 查询图表信息
|
|
|
+ chartInfoList, e := data_manage.GetChartInfoByChartInfoIds(chartIds)
|
|
|
+ if e != nil {
|
|
|
+ err = fmt.Errorf("查询图表信息列表失败 Err:%s", e)
|
|
|
+ return
|
|
|
+ }
|
|
|
+ for _, v := range chartInfoList {
|
|
|
+ chartInfoMap[v.ChartInfoId] = v
|
|
|
+ }
|
|
|
+ }
|
|
|
+ chartTagVarietyList, e := cross_variety.GetChartTagVarietyEdbInfoIdsByTagIds(tagIds)
|
|
|
+ if e != nil {
|
|
|
+ err = fmt.Errorf("查询指标信息列表失败 Err:%s", e)
|
|
|
+ return
|
|
|
+ }
|
|
|
+ edbInfoIds := make([]int, 0)
|
|
|
+ chartTagVarietyMap := make(map[int][]*cross_variety.ChartTagVariety)
|
|
|
+ for _, v := range chartTagVarietyList {
|
|
|
+ if tagList, ok := chartTagVarietyMap[v.EdbInfoId]; ok {
|
|
|
+ tagList = append(tagList, v)
|
|
|
+ chartTagVarietyMap[v.EdbInfoId] = tagList
|
|
|
+ } else {
|
|
|
+ tagList = make([]*cross_variety.ChartTagVariety, 0)
|
|
|
+ tagList = append(tagList, v)
|
|
|
+ chartTagVarietyMap[v.EdbInfoId] = tagList
|
|
|
+ }
|
|
|
+
|
|
|
+ edbInfoIds = append(edbInfoIds, v.EdbInfoId)
|
|
|
+ }
|
|
|
+
|
|
|
+ // 查询指标信息表
|
|
|
+ edbInfoList, e := data_manage.GetEdbInfoByIdList(edbInfoIds)
|
|
|
+ if e != nil {
|
|
|
+ err = fmt.Errorf("查询指标信息列表失败 Err:%s", e)
|
|
|
+ return
|
|
|
+ }
|
|
|
+ if len(edbInfoList) == 0 {
|
|
|
+ continue
|
|
|
+ }
|
|
|
+ // 查询计算指标信息，并且建立关联关系
|
|
|
+ // 查询间接引用的指标信息
|
|
|
+ calculateEdbMappingListMap, calculateEdbMappingIdsMap, e := GetEdbListByEdbInfoId(edbInfoList)
|
|
|
+ if e != nil {
|
|
|
+ err = fmt.Errorf("查询计算指标信息失败，%s", e.Error())
|
|
|
+ return
|
|
|
+ }
|
|
|
+ edbInfoMap := make(map[int]*data_manage.EdbInfo)
|
|
|
+ chartInfoCrossMap := make(map[int]struct{})
|
|
|
+ chartInfoCrossList := make([]*cross_variety.ChartInfoCrossVariety, 0)
|
|
|
+ edbCrossMap := make(map[int][]*cross_variety.ChartInfoCrossVariety)
|
|
|
+ for _, v := range edbInfoList {
|
|
|
+ edbInfoMap[v.EdbInfoId] = v
|
|
|
+ if tagList, ok := chartTagVarietyMap[v.EdbInfoId]; ok {
|
|
|
+ for _, tag := range tagList {
|
|
|
+ if chartCross, ok2 := tagChartMap[tag.ChartTagId]; ok2 {
|
|
|
+ for _, crossItem := range chartCross {
|
|
|
+ if _, ok3 := chartInfoCrossMap[crossItem.ChartInfoId]; !ok3 {
|
|
|
+ chartInfo, chartOk := chartInfoMap[crossItem.ChartInfoId]
|
|
|
+ if !chartOk { //表示图表不存在
|
|
|
+ continue
|
|
|
+ }
|
|
|
+ // 查询真正有用的指标ID
|
|
|
+ var config cross_variety.ChartConfigReq
|
|
|
+ e := json.Unmarshal([]byte(chartInfo.ExtraConfig), &config)
|
|
|
+ if e != nil {
|
|
|
+ continue
|
|
|
+ }
|
|
|
+ if utils.InArrayByInt(config.VarietyList, tag.ChartVarietyId) {
|
|
|
+ chartInfoCrossMap[crossItem.ChartInfoId] = struct{}{}
|
|
|
+ chartInfoCrossList = append(chartInfoCrossList, crossItem)
|
|
|
+ }
|
|
|
+ }
|
|
|
+ }
|
|
|
+ }
|
|
|
+ }
|
|
|
+ }
|
|
|
+ edbCrossMap[v.EdbInfoId] = chartInfoCrossList
|
|
|
+ chartInfoCrossMap = make(map[int]struct{})
|
|
|
+ chartInfoCrossList = make([]*cross_variety.ChartInfoCrossVariety, 0)
|
|
|
+ }
|
|
|
+
|
|
|
+ //查询引用关系列表，
|
|
|
+ chartEdbRelationList, e := data_manage.GetEdbInfoRelationByReferObjectIds(chartIds, utils.EDB_RELATION_CHART)
|
|
|
+ if e != nil {
|
|
|
+ err = fmt.Errorf("查询图表引用关系列表失败 Err:%s", e)
|
|
|
+ return
|
|
|
+ }
|
|
|
+ existRelationMap := make(map[string]struct{})
|
|
|
+ for _, v := range chartEdbRelationList {
|
|
|
+ name := fmt.Sprintf("%d-%d", v.ReferObjectId, v.EdbInfoId)
|
|
|
+ existRelationMap[name] = struct{}{}
|
|
|
+ }
|
|
|
+ for edbInfoId, chartCrossList := range edbCrossMap {
|
|
|
+ nowTime := time.Now()
|
|
|
+ for _, item := range chartCrossList {
|
|
|
+ name := fmt.Sprintf("%d-%d", item.ChartInfoId, edbInfoId)
|
|
|
+ if _, ok1 := existRelationMap[name]; !ok1 {
|
|
|
+ edbInfo, ok2 := edbInfoMap[edbInfoId]
|
|
|
+ if !ok2 {
|
|
|
+ continue
|
|
|
+ }
|
|
|
+ // 去掉预测指标
|
|
|
+ if edbInfo.EdbInfoType == 1 {
|
|
|
+ continue
|
|
|
+ }
|
|
|
+ tmp := &data_manage.EdbInfoRelation{
|
|
|
+ ReferObjectId: item.ChartInfoId,
|
|
|
+ ReferObjectType: utils.EDB_RELATION_CHART,
|
|
|
+ ReferObjectSubType: utils.CHART_SOURCE_CROSS_HEDGING,
|
|
|
+ EdbInfoId: edbInfoId,
|
|
|
+ EdbName: edbInfo.EdbName,
|
|
|
+ Source: edbInfo.Source,
|
|
|
+ EdbCode: edbInfo.EdbCode,
|
|
|
+ CreateTime: nowTime,
|
|
|
+ ModifyTime: nowTime,
|
|
|
+ RelationTime: item.CreateTime,
|
|
|
+ }
|
|
|
+ tmp.RelationCode = fmt.Sprintf("%d_%d_%d_%d", tmp.EdbInfoId, tmp.ReferObjectId, tmp.ReferObjectType, tmp.ReferObjectSubType)
|
|
|
+ addList = append(addList, tmp)
|
|
|
+ existRelationMap[name] = struct{}{}
|
|
|
+ // 添加间接引用记录
|
|
|
+ if edbInfo.EdbType == 2 && edbInfo.EdbInfoType == 0 {
|
|
|
+ childEdbMappingIds, ok1 := calculateEdbMappingIdsMap[edbInfo.EdbInfoId]
|
|
|
+ if !ok1 {
|
|
|
+ continue
|
|
|
+ }
|
|
|
+ for _, childEdbMappingId := range childEdbMappingIds {
|
|
|
+ childEdbMapping, ok2 := calculateEdbMappingListMap[childEdbMappingId]
|
|
|
+ if !ok2 {
|
|
|
+ continue
|
|
|
+ }
|
|
|
+ name1 := fmt.Sprintf("%d-%d", item.ChartInfoId, childEdbMapping.FromEdbInfoId)
|
|
|
+ if _, ok2 := existRelationMap[name1]; ok2 { //如果已经被直接引用了，则无需添加到间接引用记录中
|
|
|
+ continue
|
|
|
+ }
|
|
|
+
|
|
|
+ tmp1 := &data_manage.EdbInfoRelation{
|
|
|
+ ReferObjectId: item.ChartInfoId,
|
|
|
+ ReferObjectType: utils.EDB_RELATION_CHART,
|
|
|
+ ReferObjectSubType: utils.CHART_SOURCE_CROSS_HEDGING,
|
|
|
+ EdbInfoId: childEdbMapping.FromEdbInfoId,
|
|
|
+ EdbName: childEdbMapping.FromEdbName,
|
|
|
+ Source: childEdbMapping.FromSource,
|
|
|
+ EdbCode: childEdbMapping.FromEdbCode,
|
|
|
+ CreateTime: nowTime,
|
|
|
+ ModifyTime: nowTime,
|
|
|
+ RelationTime: item.CreateTime,
|
|
|
+ RelationType: 1,
|
|
|
+ RootEdbInfoId: edbInfo.EdbInfoId,
|
|
|
+ ChildEdbInfoId: childEdbMapping.EdbInfoId,
|
|
|
+ RelationCode: tmp.RelationCode,
|
|
|
+ }
|
|
|
+ addList = append(addList, tmp1)
|
|
|
+ // todo 防止重复
|
|
|
+ }
|
|
|
+ }
|
|
|
+ if len(addList) > pageSize {
|
|
|
+ err = data_manage.AddEdbInfoRelationMulti(addList)
|
|
|
+ if err != nil {
|
|
|
+ err = fmt.Errorf("新增引用记录失败 Err:%s", err)
|
|
|
+ return
|
|
|
+ }
|
|
|
+ addNum += len(addList)
|
|
|
+ addList = make([]*data_manage.EdbInfoRelation, 0)
|
|
|
+ }
|
|
|
+ }
|
|
|
+ }
|
|
|
+ }
|
|
|
+ }
|
|
|
+ //拿到500个数据ID，判断相关的引用记录，如果已存在则直接过滤，不存在则新增
|
|
|
+ if len(addList) > 0 {
|
|
|
+ err = data_manage.AddEdbInfoRelationMulti(addList)
|
|
|
+ if err != nil {
|
|
|
+ err = fmt.Errorf("新增引用记录失败 Err:%s", err)
|
|
|
+ return
|
|
|
+ }
|
|
|
+ addNum += len(addList)
|
|
|
+ }
|
|
|
+ fmt.Printf("跨品种分析图表指标引用记录处理完成, 新增%d条记录\n", addNum)
|
|
|
+ return
|
|
|
+}
|
|
|
+
|
|
|
+// 初始化事件日历中的指标引用
|
|
|
+func InitCalendarIndicatorRelation() {
|
|
|
+ fmt.Println("开始处理事件日历中的指标引用")
|
|
|
+ var addNum int
|
|
|
+ var err error
|
|
|
+ defer func() {
|
|
|
+ if err != nil {
|
|
|
+ msg := fmt.Sprintf("初始化指标在事件日历中的引用失败 initCalendarIndicatorRelation err: %v", err)
|
|
|
+ utils.FileLog.Info(msg)
|
|
|
+ fmt.Println(msg)
|
|
|
+ go alarm_msg.SendAlarmMsg(msg, 3)
|
|
|
+ }
|
|
|
+ }()
|
|
|
+ //查询chart_edb_mapping 表
|
|
|
+ obj := new(fe_calendar.FeCalendarMatter)
|
|
|
+ condition := " AND edb_info_id > 0"
|
|
|
+ total, err := obj.GetCountByCondition(condition, []interface{}{})
|
|
|
+ if err != nil {
|
|
|
+ err = fmt.Errorf("查询事件日历关联指标失败 err: %v", err)
|
|
|
+ return
|
|
|
+ }
|
|
|
+ if total == 0 {
|
|
|
+ return
|
|
|
+ }
|
|
|
+ //分页查询，每次处理500条记录
|
|
|
+ pageSize := 500
|
|
|
+ totalPage := (total + pageSize - 1) / pageSize // 使用整数除法，并添加一页以防有余数
|
|
|
+ addList := make([]*data_manage.EdbInfoRelation, 0)
|
|
|
+ //查询图表列表
|
|
|
+ for i := 0; i < totalPage; i += 1 {
|
|
|
+ startSize := i * pageSize
|
|
|
+ list, e := obj.GetPageItemsByCondition(condition, []interface{}{}, []string{}, "", startSize, pageSize)
|
|
|
+ if e != nil {
|
|
|
+ err = fmt.Errorf("查询事件日历关联指标列表失败 Err:%s", e)
|
|
|
+ return
|
|
|
+ }
|
|
|
+ if len(list) == 0 {
|
|
|
+ break
|
|
|
+ }
|
|
|
+
|
|
|
+ edbInfoIds := make([]int, 0)
|
|
|
+ edbInfoMatterMap := make(map[int][]*fe_calendar.FeCalendarMatter)
|
|
|
+ for _, v := range list {
|
|
|
+ edbInfoIds = append(edbInfoIds, v.EdbInfoId)
|
|
|
+ items, ok := edbInfoMatterMap[v.EdbInfoId]
|
|
|
+ if ok {
|
|
|
+ items = append(items, v)
|
|
|
+ edbInfoMatterMap[v.EdbInfoId] = items
|
|
|
+ } else {
|
|
|
+ items = make([]*fe_calendar.FeCalendarMatter, 0)
|
|
|
+ items = append(items, v)
|
|
|
+ edbInfoMatterMap[v.EdbInfoId] = items
|
|
|
+ }
|
|
|
+ }
|
|
|
+ // 查询指标信息表
|
|
|
+ edbInfoList, e := data_manage.GetEdbInfoByIdList(edbInfoIds)
|
|
|
+ if e != nil {
|
|
|
+ err = fmt.Errorf("查询指标信息列表失败 Err:%s", e)
|
|
|
+ return
|
|
|
+ }
|
|
|
+ if len(edbInfoList) == 0 {
|
|
|
+ continue
|
|
|
+ }
|
|
|
+
|
|
|
+ // 查询计算指标信息，并且建立关联关系
|
|
|
+ // 查询间接引用的指标信息
|
|
|
+ calculateEdbMappingListMap, calculateEdbMappingIdsMap, e := GetEdbListByEdbInfoId(edbInfoList)
|
|
|
+ if e != nil {
|
|
|
+ err = fmt.Errorf("查询计算指标信息失败，%s", e.Error())
|
|
|
+ return
|
|
|
+ }
|
|
|
+ edbInfoMap := make(map[int]*data_manage.EdbInfo)
|
|
|
+ matterIds := make([]int, 0)
|
|
|
+ for _, v := range edbInfoList {
|
|
|
+ edbInfoMap[v.EdbInfoId] = v
|
|
|
+ items, ok := edbInfoMatterMap[v.EdbInfoId]
|
|
|
+ if ok {
|
|
|
+ for _, item := range items {
|
|
|
+ matterIds = append(matterIds, item.FeCalendarMatterId)
|
|
|
+ }
|
|
|
+ }
|
|
|
+ }
|
|
|
+
|
|
|
+ //查询引用关系列表，
|
|
|
+ chartEdbRelationList, e := data_manage.GetEdbInfoRelationByReferObjectIds(matterIds, utils.EDB_RELATION_CALENDAR)
|
|
|
+ if e != nil {
|
|
|
+ err = fmt.Errorf("查询图表引用关系列表失败 Err:%s", e)
|
|
|
+ return
|
|
|
+ }
|
|
|
+ existRelationMap := make(map[string]struct{})
|
|
|
+ for _, v := range chartEdbRelationList {
|
|
|
+ name := fmt.Sprintf("%d-%d", v.ReferObjectId, v.EdbInfoId)
|
|
|
+ existRelationMap[name] = struct{}{}
|
|
|
+ }
|
|
|
+ for edbInfoId, edbInfo := range edbInfoMap {
|
|
|
+ // 去掉预测指标
|
|
|
+ if edbInfo.EdbInfoType == 1 {
|
|
|
+ continue
|
|
|
+ }
|
|
|
+ nowTime := time.Now()
|
|
|
+ items, ok := edbInfoMatterMap[edbInfoId]
|
|
|
+ if ok {
|
|
|
+ for _, v := range items {
|
|
|
+ name := fmt.Sprintf("%d-%d", v.FeCalendarMatterId, v.EdbInfoId)
|
|
|
+ if _, ok1 := existRelationMap[name]; !ok1 {
|
|
|
+ //todo 引用时间
|
|
|
+ tmp := &data_manage.EdbInfoRelation{
|
|
|
+ ReferObjectId: v.FeCalendarMatterId,
|
|
|
+ ReferObjectType: utils.EDB_RELATION_CALENDAR,
|
|
|
+ EdbInfoId: v.EdbInfoId,
|
|
|
+ EdbName: edbInfo.EdbName,
|
|
|
+ Source: edbInfo.Source,
|
|
|
+ EdbCode: edbInfo.EdbCode,
|
|
|
+ CreateTime: nowTime,
|
|
|
+ ModifyTime: nowTime,
|
|
|
+ RelationTime: v.CreateTime,
|
|
|
+ }
|
|
|
+ tmp.RelationCode = fmt.Sprintf("%d_%d_%d_%d", tmp.EdbInfoId, tmp.ReferObjectId, tmp.ReferObjectType, tmp.ReferObjectSubType)
|
|
|
+ addList = append(addList, tmp)
|
|
|
+ existRelationMap[name] = struct{}{}
|
|
|
+ // 添加间接引用记录
|
|
|
+ if edbInfo.EdbType == 2 && edbInfo.EdbInfoType == 0 {
|
|
|
+ childEdbMappingIds, ok1 := calculateEdbMappingIdsMap[edbInfo.EdbInfoId]
|
|
|
+ if !ok1 {
|
|
|
+ continue
|
|
|
+ }
|
|
|
+ for _, childEdbMappingId := range childEdbMappingIds {
|
|
|
+ childEdbMapping, ok2 := calculateEdbMappingListMap[childEdbMappingId]
|
|
|
+ if !ok2 {
|
|
|
+ continue
|
|
|
+ }
|
|
|
+ name1 := fmt.Sprintf("%d-%d", v.FeCalendarMatterId, childEdbMapping.FromEdbInfoId)
|
|
|
+ if _, ok2 := existRelationMap[name1]; ok2 { //如果已经被直接引用了，则无需添加到间接引用记录中
|
|
|
+ continue
|
|
|
+ }
|
|
|
+ tmp1 := &data_manage.EdbInfoRelation{
|
|
|
+ ReferObjectId: v.FeCalendarMatterId,
|
|
|
+ ReferObjectType: utils.EDB_RELATION_CALENDAR,
|
|
|
+ EdbInfoId: childEdbMapping.FromEdbInfoId,
|
|
|
+ EdbName: childEdbMapping.FromEdbName,
|
|
|
+ Source: childEdbMapping.FromSource,
|
|
|
+ EdbCode: childEdbMapping.FromEdbCode,
|
|
|
+ CreateTime: nowTime,
|
|
|
+ ModifyTime: nowTime,
|
|
|
+ RelationTime: v.CreateTime,
|
|
|
+ RelationType: 1,
|
|
|
+ RootEdbInfoId: edbInfo.EdbInfoId,
|
|
|
+ ChildEdbInfoId: childEdbMapping.EdbInfoId,
|
|
|
+ RelationCode: tmp.RelationCode,
|
|
|
+ }
|
|
|
+ addList = append(addList, tmp1)
|
|
|
+ // todo 防止重复
|
|
|
+ }
|
|
|
+ }
|
|
|
+ if len(addList) > pageSize {
|
|
|
+ err = data_manage.AddEdbInfoRelationMulti(addList)
|
|
|
+ if err != nil {
|
|
|
+ err = fmt.Errorf("新增引用记录失败 Err:%s", err)
|
|
|
+ return
|
|
|
+ }
|
|
|
+ addNum += len(addList)
|
|
|
+ addList = make([]*data_manage.EdbInfoRelation, 0)
|
|
|
+ }
|
|
|
+ }
|
|
|
+ }
|
|
|
+ }
|
|
|
+
|
|
|
+ }
|
|
|
+ }
|
|
|
+ //拿到500个数据ID，判断相关的引用记录，如果已存在则直接过滤，不存在则新增
|
|
|
+ if len(addList) > 0 {
|
|
|
+ err = data_manage.AddEdbInfoRelationMulti(addList)
|
|
|
+ if err != nil {
|
|
|
+ err = fmt.Errorf("新增引用记录失败 Err:%s", err)
|
|
|
+ return
|
|
|
+ }
|
|
|
+ addNum += len(addList)
|
|
|
+ }
|
|
|
+ fmt.Printf("事件日历指标引用记录处理完成, 新增%d条记录\n", addNum)
|
|
|
+ return
|
|
|
+}
|
|
|
+
|
|
|
+// 初始化表格中的指标引用
|
|
|
+func InitExcelEdbRelation() {
|
|
|
+ fmt.Println("开始处理表格中的指标引用")
|
|
|
+ var err error
|
|
|
+ var addNum int
|
|
|
+ defer func() {
|
|
|
+ if err != nil {
|
|
|
+ msg := fmt.Sprintf("初始化指标在表格中的引用失败 InitChartEdbRelation err: %v", err)
|
|
|
+ utils.FileLog.Info(msg)
|
|
|
+ fmt.Println(msg)
|
|
|
+ go alarm_msg.SendAlarmMsg(msg, 3)
|
|
|
+ }
|
|
|
+ }()
|
|
|
+ //查询表格指标绑定表
|
|
|
+ sources := []int{utils.TIME_TABLE, utils.MIXED_TABLE, utils.BALANCE_TABLE}
|
|
|
+ total, err := excel.GetExcelEdbMappingTotalBySource(sources)
|
|
|
+ if err != nil {
|
|
|
+ err = fmt.Errorf("查询表格关联指标失败 err: %v", err)
|
|
|
+ return
|
|
|
+ }
|
|
|
+ if total == 0 {
|
|
|
+ return
|
|
|
+ }
|
|
|
+ //分页查询，每次处理100条记录
|
|
|
+ pageSize := 100
|
|
|
+ totalPage := (total + pageSize - 1) / pageSize // 使用整数除法，并添加一页以防有余数
|
|
|
+ addList := make([]*data_manage.EdbInfoRelation, 0)
|
|
|
+ //查询表格列表
|
|
|
+ for i := 0; i < totalPage; i += 1 {
|
|
|
+ startSize := i * pageSize
|
|
|
+ list, e := excel.GetExcelEdbMappingListBySource(sources, startSize, pageSize)
|
|
|
+ if e != nil {
|
|
|
+ err = fmt.Errorf("查询表格关联指标列表失败 Err:%s", e)
|
|
|
+ return
|
|
|
+ }
|
|
|
+ if len(list) == 0 {
|
|
|
+ break
|
|
|
+ }
|
|
|
+
|
|
|
+ edbInfoIds := make([]int, 0)
|
|
|
+ for _, v := range list {
|
|
|
+ edbInfoIds = append(edbInfoIds, v.EdbInfoId)
|
|
|
+ }
|
|
|
+ // 查询指标信息表
|
|
|
+ edbInfoList, e := data_manage.GetEdbInfoByIdList(edbInfoIds)
|
|
|
+ if e != nil {
|
|
|
+ err = fmt.Errorf("查询指标信息列表失败 Err:%s", e)
|
|
|
+ return
|
|
|
+ }
|
|
|
+ if len(edbInfoList) == 0 {
|
|
|
+ continue
|
|
|
+ }
|
|
|
+ // 查询计算指标信息，并且建立关联关系
|
|
|
+ // 查询间接引用的指标信息
|
|
|
+ calculateEdbMappingListMap, calculateEdbMappingIdsMap, e := GetEdbListByEdbInfoId(edbInfoList)
|
|
|
+ if e != nil {
|
|
|
+ err = fmt.Errorf("查询计算指标信息失败，%s", e.Error())
|
|
|
+ return
|
|
|
+ }
|
|
|
+ // 查询指标间接引用
|
|
|
+ edbInfoMap := make(map[int]*data_manage.EdbInfo)
|
|
|
+ for _, v := range edbInfoList {
|
|
|
+ edbInfoMap[v.EdbInfoId] = v
|
|
|
+ }
|
|
|
+
|
|
|
+ // 筛选有用的表格
|
|
|
+ excelIds := make([]int, 0)
|
|
|
+ for _, v := range list {
|
|
|
+ excelIds = append(excelIds, v.ExcelInfoId)
|
|
|
+ }
|
|
|
+
|
|
|
+ //查询引用关系列表，
|
|
|
+ chartEdbRelationList, e := data_manage.GetEdbInfoRelationByReferObjectIds(excelIds, utils.EDB_RELATION_TABLE)
|
|
|
+ if e != nil {
|
|
|
+ err = fmt.Errorf("查询表格引用关系列表失败 Err:%s", e)
|
|
|
+ return
|
|
|
+ }
|
|
|
+ existRelationMap := make(map[string]struct{})
|
|
|
+ for _, v := range chartEdbRelationList {
|
|
|
+ name := fmt.Sprintf("%d-%d", v.ReferObjectId, v.EdbInfoId)
|
|
|
+ existRelationMap[name] = struct{}{}
|
|
|
+ }
|
|
|
+ for _, v := range list {
|
|
|
+ nowTime := time.Now()
|
|
|
+ name := fmt.Sprintf("%d-%d", v.ExcelInfoId, v.EdbInfoId)
|
|
|
+ if _, ok := existRelationMap[name]; !ok {
|
|
|
+ edbInfo, ok2 := edbInfoMap[v.EdbInfoId]
|
|
|
+ if !ok2 {
|
|
|
+ continue
|
|
|
+ }
|
|
|
+ // 去掉预测指标
|
|
|
+ if edbInfo.EdbInfoType == 1 {
|
|
|
+ continue
|
|
|
+ }
|
|
|
+ tmp := &data_manage.EdbInfoRelation{
|
|
|
+ ReferObjectId: v.ExcelInfoId,
|
|
|
+ ReferObjectType: utils.EDB_RELATION_TABLE,
|
|
|
+ ReferObjectSubType: v.Source,
|
|
|
+ EdbInfoId: v.EdbInfoId,
|
|
|
+ EdbName: edbInfo.EdbName,
|
|
|
+ Source: edbInfo.Source,
|
|
|
+ EdbCode: edbInfo.EdbCode,
|
|
|
+ CreateTime: nowTime,
|
|
|
+ ModifyTime: nowTime,
|
|
|
+ RelationTime: v.CreateTime,
|
|
|
+ }
|
|
|
+ tmp.RelationCode = fmt.Sprintf("%d_%d_%d_%d", tmp.EdbInfoId, tmp.ReferObjectId, tmp.ReferObjectType, tmp.ReferObjectSubType)
|
|
|
+ addList = append(addList, tmp)
|
|
|
+ existRelationMap[name] = struct{}{}
|
|
|
+ // 添加间接引用记录
|
|
|
+ if edbInfo.EdbType == 2 && edbInfo.EdbInfoType == 0 {
|
|
|
+ childEdbMappingIds, ok1 := calculateEdbMappingIdsMap[edbInfo.EdbInfoId]
|
|
|
+ if !ok1 {
|
|
|
+ continue
|
|
|
+ }
|
|
|
+ for _, childEdbMappingId := range childEdbMappingIds {
|
|
|
+ childEdbMapping, ok2 := calculateEdbMappingListMap[childEdbMappingId]
|
|
|
+ if !ok2 {
|
|
|
+ continue
|
|
|
+ }
|
|
|
+ name1 := fmt.Sprintf("%d-%d", v.ExcelInfoId, childEdbMapping.FromEdbInfoId)
|
|
|
+ if _, ok2 := existRelationMap[name1]; ok2 { //如果已经被直接引用了，则无需添加到间接引用记录中
|
|
|
+ continue
|
|
|
+ }
|
|
|
+ tmp1 := &data_manage.EdbInfoRelation{
|
|
|
+ ReferObjectId: v.ExcelInfoId,
|
|
|
+ ReferObjectType: utils.EDB_RELATION_TABLE,
|
|
|
+ ReferObjectSubType: v.Source,
|
|
|
+ EdbInfoId: childEdbMapping.FromEdbInfoId,
|
|
|
+ EdbName: childEdbMapping.FromEdbName,
|
|
|
+ Source: childEdbMapping.FromSource,
|
|
|
+ EdbCode: childEdbMapping.FromEdbCode,
|
|
|
+ CreateTime: nowTime,
|
|
|
+ ModifyTime: nowTime,
|
|
|
+ RelationTime: v.CreateTime,
|
|
|
+ RelationType: 1,
|
|
|
+ RootEdbInfoId: edbInfo.EdbInfoId,
|
|
|
+ ChildEdbInfoId: childEdbMapping.EdbInfoId,
|
|
|
+ RelationCode: tmp.RelationCode,
|
|
|
+ }
|
|
|
+ addList = append(addList, tmp1)
|
|
|
+ // todo 防止重复
|
|
|
+ }
|
|
|
+ }
|
|
|
+
|
|
|
+ if len(addList) > pageSize {
|
|
|
+ err = data_manage.AddEdbInfoRelationMulti(addList)
|
|
|
+ if err != nil {
|
|
|
+ err = fmt.Errorf("新增引用记录失败 Err:%s", err)
|
|
|
+ return
|
|
|
+ }
|
|
|
+ addNum += len(addList)
|
|
|
+ addList = make([]*data_manage.EdbInfoRelation, 0)
|
|
|
+ }
|
|
|
+ }
|
|
|
+ }
|
|
|
+ }
|
|
|
+ //拿到500个数据ID，判断相关的引用记录，如果已存在则直接过滤，不存在则新增
|
|
|
+ if len(addList) > 0 {
|
|
|
+ err = data_manage.AddEdbInfoRelationMulti(addList)
|
|
|
+ if err != nil {
|
|
|
+ err = fmt.Errorf("新增引用记录失败 Err:%s", err)
|
|
|
+ return
|
|
|
+ }
|
|
|
+ addNum += len(addList)
|
|
|
+ }
|
|
|
+ fmt.Printf("表格指标引用记录处理完成, 新增%d条记录\n", addNum)
|
|
|
+ return
|
|
|
+}
|
|
|
+
|
|
|
+// 处理逻辑图中的指标引用
|
|
|
+func InitSandBoxEdbRelation() {
|
|
|
+ fmt.Println("开始处理逻辑图中的指标引用")
|
|
|
+ var err error
|
|
|
+ var addNum int
|
|
|
+ defer func() {
|
|
|
+ if err != nil {
|
|
|
+ msg := fmt.Sprintf("初始化指标在逻辑图中的引用失败 initSandBoxEdbRelation err: %v", err)
|
|
|
+ utils.FileLog.Info(msg)
|
|
|
+ fmt.Println(msg)
|
|
|
+ go alarm_msg.SendAlarmMsg(msg, 3)
|
|
|
+ }
|
|
|
+ }()
|
|
|
+ condition := " AND is_delete = 0"
|
|
|
+ total, err := sandbox.GetSandboxListCountByCondition(condition, []interface{}{})
|
|
|
+ if err != nil {
|
|
|
+ err = fmt.Errorf("查询逻辑图总数失败 err: %v", err)
|
|
|
+ return
|
|
|
+ }
|
|
|
+ if total == 0 {
|
|
|
+ return
|
|
|
+ }
|
|
|
+ //分页查询，每次处理500条记录
|
|
|
+ pageSize := 100
|
|
|
+ totalPage := (total + pageSize - 1) / pageSize // 使用整数除法，并添加一页以防有余数
|
|
|
+ addList := make([]*data_manage.EdbInfoRelation, 0)
|
|
|
+ //查询图表列表
|
|
|
+ for i := 0; i < totalPage; i += 1 {
|
|
|
+ startSize := i * pageSize
|
|
|
+ list, e := sandbox.GetSandboxListByCondition(condition, []interface{}{}, startSize, pageSize)
|
|
|
+ if e != nil {
|
|
|
+ err = fmt.Errorf("查询逻辑图列表失败 Err:%s", e)
|
|
|
+ return
|
|
|
+ }
|
|
|
+ if len(list) == 0 {
|
|
|
+ break
|
|
|
+ }
|
|
|
+
|
|
|
+ edbInfoIds := make([]int, 0)
|
|
|
+ edbSandboxMap := make(map[int][]*sandbox.Sandbox)
|
|
|
+ for _, v := range list {
|
|
|
+ if v.Content == "" {
|
|
|
+ continue
|
|
|
+ }
|
|
|
+ edbInfoIdsTmp, e := getSandBoxEdbIdsByContent(v.Content)
|
|
|
+ if e != nil {
|
|
|
+ continue
|
|
|
+ //err = fmt.Errorf("查询逻辑图关联的指标Id失败 Err:%s", e)
|
|
|
+ //return
|
|
|
+ }
|
|
|
+ for _, edbId := range edbInfoIdsTmp {
|
|
|
+ edbInfoIds = append(edbInfoIds, edbId)
|
|
|
+ edbSandboxMap[edbId] = append(edbSandboxMap[edbId], v)
|
|
|
+ }
|
|
|
+ }
|
|
|
+ if len(edbInfoIds) <= 0 {
|
|
|
+ continue
|
|
|
+ }
|
|
|
+ // 查询指标信息表
|
|
|
+ edbInfoList, e := data_manage.GetEdbInfoByIdList(edbInfoIds)
|
|
|
+ if e != nil {
|
|
|
+ err = fmt.Errorf("查询指标信息列表失败 Err:%s", e)
|
|
|
+ return
|
|
|
+ }
|
|
|
+ if len(edbInfoList) == 0 {
|
|
|
+ continue
|
|
|
+ }
|
|
|
+ // 查询计算指标信息，并且建立关联关系
|
|
|
+ // 查询间接引用的指标信息
|
|
|
+ calculateEdbMappingListMap, calculateEdbMappingIdsMap, e := GetEdbListByEdbInfoId(edbInfoList)
|
|
|
+ if e != nil {
|
|
|
+ err = fmt.Errorf("查询计算指标信息失败，%s", e.Error())
|
|
|
+ return
|
|
|
+ }
|
|
|
+ edbInfoMap := make(map[int]*data_manage.EdbInfo)
|
|
|
+ sandboxIds := make([]int, 0)
|
|
|
+ for _, v := range edbInfoList {
|
|
|
+ edbInfoMap[v.EdbInfoId] = v
|
|
|
+ if items, ok := edbSandboxMap[v.EdbInfoId]; ok {
|
|
|
+ for _, item := range items {
|
|
|
+ sandboxIds = append(sandboxIds, item.SandboxId)
|
|
|
+ }
|
|
|
+ }
|
|
|
+ }
|
|
|
+
|
|
|
+ //查询引用关系列表，
|
|
|
+ chartEdbRelationList, e := data_manage.GetEdbInfoRelationByReferObjectIds(sandboxIds, utils.EDB_RELATION_SANDBOX)
|
|
|
+ if e != nil {
|
|
|
+ err = fmt.Errorf("查询逻辑图引用关系列表失败 Err:%s", e)
|
|
|
+ return
|
|
|
+ }
|
|
|
+ existRelationMap := make(map[string]struct{})
|
|
|
+ for _, v := range chartEdbRelationList {
|
|
|
+ name := fmt.Sprintf("%d-%d", v.ReferObjectId, v.EdbInfoId)
|
|
|
+ existRelationMap[name] = struct{}{}
|
|
|
+ }
|
|
|
+ for edbInfoId, sandboxList := range edbSandboxMap {
|
|
|
+ nowTime := time.Now()
|
|
|
+ for _, v := range sandboxList {
|
|
|
+ name := fmt.Sprintf("%d-%d", v.SandboxId, edbInfoId)
|
|
|
+ if _, ok := existRelationMap[name]; !ok {
|
|
|
+ edbInfo, ok2 := edbInfoMap[edbInfoId]
|
|
|
+ if !ok2 {
|
|
|
+ continue
|
|
|
+ }
|
|
|
+ // 去掉预测指标
|
|
|
+ if edbInfo.EdbInfoType == 1 {
|
|
|
+ continue
|
|
|
+ }
|
|
|
+ tmp := &data_manage.EdbInfoRelation{
|
|
|
+ ReferObjectId: v.SandboxId,
|
|
|
+ ReferObjectType: utils.EDB_RELATION_SANDBOX,
|
|
|
+ EdbInfoId: edbInfoId,
|
|
|
+ EdbName: edbInfo.EdbName,
|
|
|
+ Source: edbInfo.Source,
|
|
|
+ EdbCode: edbInfo.EdbCode,
|
|
|
+ CreateTime: nowTime,
|
|
|
+ ModifyTime: nowTime,
|
|
|
+ RelationTime: v.CreateTime,
|
|
|
+ }
|
|
|
+ tmp.RelationCode = fmt.Sprintf("%d_%d_%d_%d", tmp.EdbInfoId, tmp.ReferObjectId, tmp.ReferObjectType, tmp.ReferObjectSubType)
|
|
|
+ addList = append(addList, tmp)
|
|
|
+ existRelationMap[name] = struct{}{}
|
|
|
+ // 添加间接引用记录
|
|
|
+ if edbInfo.EdbType == 2 && edbInfo.EdbInfoType == 0 {
|
|
|
+ childEdbMappingIds, ok1 := calculateEdbMappingIdsMap[edbInfo.EdbInfoId]
|
|
|
+ if !ok1 {
|
|
|
+ continue
|
|
|
+ }
|
|
|
+ for _, childEdbMappingId := range childEdbMappingIds {
|
|
|
+ childEdbMapping, ok2 := calculateEdbMappingListMap[childEdbMappingId]
|
|
|
+ if !ok2 {
|
|
|
+ continue
|
|
|
+ }
|
|
|
+ name1 := fmt.Sprintf("%d-%d", v.SandboxId, childEdbMapping.FromEdbInfoId)
|
|
|
+ if _, ok2 := existRelationMap[name1]; ok2 { //如果已经被直接引用了，则无需添加到间接引用记录中
|
|
|
+ continue
|
|
|
+ }
|
|
|
+ tmp1 := &data_manage.EdbInfoRelation{
|
|
|
+ ReferObjectId: v.SandboxId,
|
|
|
+ ReferObjectType: utils.EDB_RELATION_SANDBOX,
|
|
|
+ EdbInfoId: childEdbMapping.FromEdbInfoId,
|
|
|
+ EdbName: childEdbMapping.FromEdbName,
|
|
|
+ Source: childEdbMapping.FromSource,
|
|
|
+ EdbCode: childEdbMapping.FromEdbCode,
|
|
|
+ CreateTime: nowTime,
|
|
|
+ ModifyTime: nowTime,
|
|
|
+ RelationTime: v.CreateTime,
|
|
|
+ RelationType: 1,
|
|
|
+ RootEdbInfoId: edbInfo.EdbInfoId,
|
|
|
+ ChildEdbInfoId: childEdbMapping.EdbInfoId,
|
|
|
+ RelationCode: tmp.RelationCode,
|
|
|
+ }
|
|
|
+ addList = append(addList, tmp1)
|
|
|
+ // todo 防止重复
|
|
|
+ }
|
|
|
+ }
|
|
|
+ if len(addList) > pageSize {
|
|
|
+ err = data_manage.AddEdbInfoRelationMulti(addList)
|
|
|
+ if err != nil {
|
|
|
+ err = fmt.Errorf("新增引用记录失败 Err:%s", err)
|
|
|
+ return
|
|
|
+ }
|
|
|
+ addNum += len(addList)
|
|
|
+ addList = make([]*data_manage.EdbInfoRelation, 0)
|
|
|
+ }
|
|
|
+ }
|
|
|
+ }
|
|
|
+ }
|
|
|
+ }
|
|
|
+ //拿到500个数据ID，判断相关的引用记录，如果已存在则直接过滤，不存在则新增
|
|
|
+ if len(addList) > 0 {
|
|
|
+ err = data_manage.AddEdbInfoRelationMulti(addList)
|
|
|
+ if err != nil {
|
|
|
+ err = fmt.Errorf("新增引用记录失败 Err:%s", err)
|
|
|
+ return
|
|
|
+ }
|
|
|
+ addNum += len(addList)
|
|
|
+ }
|
|
|
+ fmt.Printf("逻辑图指标引用记录处理完成, 新增%d条记录\n", addNum)
|
|
|
+ return
|
|
|
+}
|
|
|
+
|
|
|
+func getSandBoxEdbIdsByContent(content string) (edbInfoIds []int, err error) {
|
|
|
+ var contentInfo sandbox.ContentDataStruct
|
|
|
+ err = json.Unmarshal([]byte(content), &contentInfo)
|
|
|
+ if err != nil {
|
|
|
+ err = fmt.Errorf("json.Unmarshal err:%s", err.Error())
|
|
|
+ return
|
|
|
+ }
|
|
|
+ // 遍历所有节点
|
|
|
+ for _, node := range contentInfo.Cells {
|
|
|
+ if node.Data == nil {
|
|
|
+ continue
|
|
|
+ }
|
|
|
+ for _, v := range node.Data.LinkData {
|
|
|
+ if v.Type == 1 {
|
|
|
+ edbInfoIds = append(edbInfoIds, v.Id)
|
|
|
+ }
|
|
|
+ }
|
|
|
+ }
|
|
|
+ return
|
|
|
+}
|
|
|
+
|
|
|
+func GetEdbListByEdbInfoId(edbInfoList []*data_manage.EdbInfo) (edbMappingListMap map[int]*data_manage.EdbInfoCalculateMapping, edbInfoMappingRootIdsMap map[int][]int, err error) {
|
|
|
+ if len(edbInfoList) == 0 {
|
|
|
+ return
|
|
|
+ }
|
|
|
+ edbInfoIds := make([]int, 0)
|
|
|
+ for _, v := range edbInfoList {
|
|
|
+ if v.EdbType == 2 && v.EdbInfoType == 0 { //普通计算指标，排除预算指标
|
|
|
+ edbInfoIds = append(edbInfoIds, v.EdbInfoId)
|
|
|
+ }
|
|
|
+ }
|
|
|
+ if len(edbInfoIds) == 0 {
|
|
|
+ return
|
|
|
+ }
|
|
|
+ //查询指标信息
|
|
|
+ allEdbMappingMap := make(map[int][]*data_manage.EdbInfoCalculateMappingInfo, 0)
|
|
|
+ allMappingList, e := data_manage.GetEdbInfoCalculateMappingListByEdbInfoIds(edbInfoIds)
|
|
|
+ if e != nil {
|
|
|
+ err = fmt.Errorf("GetEdbInfoCalculateMappingListByEdbInfoIds err: %s", e.Error())
|
|
|
+ return
|
|
|
+ }
|
|
|
+ for _, v := range allMappingList {
|
|
|
+ if _, ok := allEdbMappingMap[v.EdbInfoId]; !ok {
|
|
|
+ allEdbMappingMap[v.EdbInfoId] = make([]*data_manage.EdbInfoCalculateMappingInfo, 0)
|
|
|
+ }
|
|
|
+ allEdbMappingMap[v.EdbInfoId] = append(allEdbMappingMap[v.EdbInfoId], v)
|
|
|
+ }
|
|
|
+ //查询指标映射
|
|
|
+ //查询所有指标数据
|
|
|
+ //查询这个指标相关的mapping信息放到数组里，
|
|
|
+ //将得到的指标ID信息放到数组里
|
|
|
+ hasFindMap := make(map[int]struct{})
|
|
|
+ edbInfoIdMap := make(map[int]struct{})
|
|
|
+ edbMappingList := make([]*data_manage.EdbInfoCalculateMapping, 0)
|
|
|
+ edbInfoMappingRootIdsMap = make(map[int][]int, 0)
|
|
|
+ edbMappingMap := make(map[int]struct{})
|
|
|
+ for _, edbInfo := range edbInfoList {
|
|
|
+ if edbInfo.EdbType == 2 && edbInfo.EdbInfoType == 0 {
|
|
|
+ edbInfoId := edbInfo.EdbInfoId
|
|
|
+ edbMappingList, err = getCalculateEdbInfoByEdbInfoId(allEdbMappingMap, edbInfoId, hasFindMap, edbInfoIdMap, edbMappingList, edbMappingMap, edbInfoMappingRootIdsMap, edbInfoId)
|
|
|
+ if err != nil {
|
|
|
+ err = fmt.Errorf(" GetCalculateEdbInfoByEdbInfoId err: %s", err.Error())
|
|
|
+ return
|
|
|
+ }
|
|
|
+ }
|
|
|
+ }
|
|
|
+ if len(edbMappingList) == 0 {
|
|
|
+ return
|
|
|
+ }
|
|
|
+ // 查询指标信息
|
|
|
+ // 指标信息map
|
|
|
+ edbInfoIdList := make([]int, 0)
|
|
|
+ for k, _ := range edbInfoIdMap {
|
|
|
+ edbInfoIdList = append(edbInfoIdList, k)
|
|
|
+ }
|
|
|
+ edbMappingListMap = make(map[int]*data_manage.EdbInfoCalculateMapping)
|
|
|
+
|
|
|
+ if len(edbMappingList) > 0 {
|
|
|
+ for _, v := range edbMappingList {
|
|
|
+ edbMappingListMap[v.EdbInfoCalculateMappingId] = v
|
|
|
+ }
|
|
|
+ }
|
|
|
+ return
|
|
|
+}
|
|
|
+
|
|
|
+// getCalculateEdbInfoByEdbInfoId 计算指标追溯
|
|
|
+func getCalculateEdbInfoByEdbInfoId(allEdbMappingMap map[int][]*data_manage.EdbInfoCalculateMappingInfo, edbInfoId int, hasFindMap map[int]struct{}, edbInfoIdMap map[int]struct{}, edbMappingList []*data_manage.EdbInfoCalculateMapping, edbMappingMap map[int]struct{}, edbInfoMappingRootIdsMap map[int][]int, rootEdbInfoId int) (newEdbMappingList []*data_manage.EdbInfoCalculateMapping, err error) {
|
|
|
+ newEdbMappingList = edbMappingList
|
|
|
+ _, ok := hasFindMap[edbInfoId]
|
|
|
+ if ok {
|
|
|
+ return
|
|
|
+ }
|
|
|
+
|
|
|
+ if _, ok1 := edbInfoIdMap[edbInfoId]; !ok1 {
|
|
|
+ edbInfoIdMap[edbInfoId] = struct{}{}
|
|
|
+ }
|
|
|
+ edbInfoMappingList := make([]*data_manage.EdbInfoCalculateMappingInfo, 0)
|
|
|
+ edbInfoMappingList, ok = allEdbMappingMap[edbInfoId]
|
|
|
+ if !ok {
|
|
|
+ edbInfoMappingList, err = data_manage.GetEdbInfoCalculateMappingListByEdbInfoId(edbInfoId)
|
|
|
+ if err != nil {
|
|
|
+ err = fmt.Errorf("GetEdbInfoCalculateMappingListByEdbInfoId err: %s", err.Error())
|
|
|
+ return
|
|
|
+ }
|
|
|
+ }
|
|
|
+ hasFindMap[edbInfoId] = struct{}{}
|
|
|
+ if len(edbInfoMappingList) > 0 {
|
|
|
+ fromEdbInfoIdList := make([]int, 0)
|
|
|
+ edbInfoMappingIdList := make([]int, 0)
|
|
|
+ for _, v := range edbInfoMappingList {
|
|
|
+ fromEdbInfoIdList = append(fromEdbInfoIdList, v.FromEdbInfoId)
|
|
|
+ edbInfoMappingIdList = append(edbInfoMappingIdList, v.EdbInfoCalculateMappingId)
|
|
|
+ if _, ok1 := edbInfoIdMap[v.FromEdbInfoId]; !ok1 {
|
|
|
+ edbInfoIdMap[v.FromEdbInfoId] = struct{}{}
|
|
|
+ }
|
|
|
+ if _, ok2 := edbMappingMap[v.EdbInfoCalculateMappingId]; !ok2 {
|
|
|
+ edbMappingMap[v.EdbInfoCalculateMappingId] = struct{}{}
|
|
|
+ tmp := &data_manage.EdbInfoCalculateMapping{
|
|
|
+ EdbInfoCalculateMappingId: v.EdbInfoCalculateMappingId,
|
|
|
+ EdbInfoId: v.EdbInfoId,
|
|
|
+ Source: v.Source,
|
|
|
+ SourceName: v.SourceName,
|
|
|
+ EdbCode: v.EdbCode,
|
|
|
+ FromEdbInfoId: v.FromEdbInfoId,
|
|
|
+ FromEdbCode: v.FromEdbCode,
|
|
|
+ FromEdbName: v.FromEdbName,
|
|
|
+ FromSource: v.FromSource,
|
|
|
+ FromSourceName: v.FromSourceName,
|
|
|
+ FromTag: v.FromTag,
|
|
|
+ Sort: v.Sort,
|
|
|
+ CreateTime: v.CreateTime,
|
|
|
+ ModifyTime: v.ModifyTime,
|
|
|
+ }
|
|
|
+ newEdbMappingList = append(newEdbMappingList, tmp)
|
|
|
+
|
|
|
+ }
|
|
|
+
|
|
|
+ if edbInfoId != v.FromEdbInfoId && (v.FromEdbType == 2 || v.FromEdbInfoType == 1) {
|
|
|
+ // 查过了就不查了
|
|
|
+ if _, ok2 := hasFindMap[v.FromEdbInfoId]; !ok2 {
|
|
|
+ newEdbMappingList, err = getCalculateEdbInfoByEdbInfoId(allEdbMappingMap, v.FromEdbInfoId, hasFindMap, edbInfoIdMap, newEdbMappingList, edbMappingMap, edbInfoMappingRootIdsMap, rootEdbInfoId)
|
|
|
+ if err != nil {
|
|
|
+ err = fmt.Errorf("traceEdbInfoByEdbInfoId err: %s", err.Error())
|
|
|
+ return
|
|
|
+ }
|
|
|
+ }
|
|
|
+ }
|
|
|
+ hasFindMap[v.FromEdbInfoId] = struct{}{}
|
|
|
+ }
|
|
|
+ edbInfoMappingRootIdsMap[rootEdbInfoId] = append(edbInfoMappingRootIdsMap[rootEdbInfoId], edbInfoMappingIdList...)
|
|
|
+ }
|
|
|
+
|
|
|
+ return
|
|
|
+}
|
|
|
+
|
|
|
+// GetCalculateEdbByFromEdbInfo 找到依赖于该基础指标的所有计算指标
|
|
|
+func GetCalculateEdbByFromEdbInfo(edbInfoIds []int, calculateEdbIds []int, hasFind map[int]struct{}) (newCalculateEdbIds []int, err error) {
|
|
|
+ if len(edbInfoIds) == 0 {
|
|
|
+ return
|
|
|
+ }
|
|
|
+ newCalculateEdbIds = calculateEdbIds
|
|
|
+ newEdbInfoIds := make([]int, 0)
|
|
|
+ for _, v := range edbInfoIds {
|
|
|
+ if _, ok := hasFind[v]; ok {
|
|
|
+ continue
|
|
|
+ }
|
|
|
+ newEdbInfoIds = append(newEdbInfoIds, v)
|
|
|
+ }
|
|
|
+ if len(newEdbInfoIds) == 0 {
|
|
|
+ return
|
|
|
+ }
|
|
|
+ var condition string
|
|
|
+ var pars []interface{}
|
|
|
+ // 关联指标
|
|
|
+ condition += ` AND b.from_edb_info_id in (` + utils.GetOrmInReplace(len(newEdbInfoIds)) + `)`
|
|
|
+ pars = append(pars, newEdbInfoIds)
|
|
|
+
|
|
|
+ //获取关联图表列表
|
|
|
+ list, err := data_manage.GetRelationEdbInfoListMappingByCondition(condition, pars)
|
|
|
+ if err != nil && err.Error() != utils.ErrNoRow() {
|
|
|
+ err = fmt.Errorf("获取关联指标信息失败,Err:%s", err.Error())
|
|
|
+ return
|
|
|
+ }
|
|
|
+ calculateEdbIdsTmp := make([]int, 0)
|
|
|
+ for _, mapping := range list {
|
|
|
+ if mapping.EdbType == 2 && mapping.EdbInfoType == 0 { // 如果指标库里的计算指标，则加入，否则继续找
|
|
|
+ newCalculateEdbIds = append(newCalculateEdbIds, mapping.EdbInfoId)
|
|
|
+ calculateEdbIdsTmp = append(calculateEdbIdsTmp, mapping.EdbInfoId)
|
|
|
+ }
|
|
|
+ }
|
|
|
+ for _, v := range newEdbInfoIds {
|
|
|
+ hasFind[v] = struct{}{}
|
|
|
+ }
|
|
|
+ if len(calculateEdbIdsTmp) > 0 {
|
|
|
+ newCalculateEdbIds, err = GetCalculateEdbByFromEdbInfo(calculateEdbIdsTmp, newCalculateEdbIds, hasFind)
|
|
|
+ if err != nil {
|
|
|
+ return
|
|
|
+ }
|
|
|
+ }
|
|
|
+
|
|
|
+ return
|
|
|
+}
|
zqbao