package controllers import ( "eta_gn/eta_api/models/data_manage" "eta_gn/eta_api/services/data" "eta_gn/eta_api/utils" "fmt" "sort" "strconv" "strings" "time" ) // init // @Description: 数据修复 // @author: Roc // @datetime 2024-12-02 09:29:56 func init() { // ### 1、先修复指标分类完整路径 //initFixEdbClassifyPath() // ### 2、再修复个人分类、删除不存在指标的分类 // ### 2.1 数据加工 //initFixSelfEdbClassify(utils.EdbClassifyTypeCalculate) // //// ### 2.2 衍生指标 //initFixSelfEdbClassify(utils.EdbClassifyTypePredict) // // 开始图表数据了 // ### 3、修复图表分类完整路径 //initFixChartClassifyPath() // ### 4、修复图表分类完整路径 //initFixSelfChartClassify() // ### 5、修复ES数据 //initEs() //fmt.Println("修复完成") } // initFixEdbClassifyPath // @Description: 修复分类完整路径 // @author: Roc // @datetime 2024-11-26 15:40:57 func initFixEdbClassifyPath() { allList := make([]*data_manage.EdbClassifyItems, 0) allList, e := data_manage.GetEdbClassifyByClassifyTypes([]int{utils.EdbClassifyTypeBase, utils.EdbClassifyTypePredict, utils.EdbClassifyTypeCalculate}, 0) if e != nil { fmt.Println("获取分类失败", e) return } var sortList data_manage.EdbClassifyItemList allList = data.GetClassifyTreeRecursive(allList, 0) //根据sort值排序 sortList = allList sort.Sort(sortList) for _, v := range sortList { updateEdbPath(v, "", "") } fmt.Println("修复分类路径完成") } func updateEdbPath(item *data_manage.EdbClassifyItems, classifyNamePath, classifyIdPath string) { if classifyNamePath == `` { classifyNamePath = item.ClassifyName classifyIdPath = fmt.Sprint(item.ClassifyId) } else { classifyNamePath = fmt.Sprint(classifyNamePath, "|", item.ClassifyName) classifyIdPath = fmt.Sprint(classifyIdPath, ",", item.ClassifyId) } err := data_manage.UpdateEdbClassify(item.ClassifyId, classifyNamePath, classifyIdPath) if err != nil { fmt.Println(item.ClassifyId, ";更新失败", err) } if item.Children != nil { for _, v := range item.Children { updateEdbPath(v, classifyNamePath, classifyIdPath) } } } // initFixSelfEdbClassify // @Description: initFixUserClassify // @author: Roc // @datetime 2024-11-29 16:57:54 func initFixSelfEdbClassify(classifyType int) { // 指标数据明细 condition := ` AND edb_info_type = 0 and edb_type = 2 ` if classifyType == utils.EdbClassifyTypePredict { condition = ` AND edb_info_type = 1 ` } pars := make([]interface{}, 0) edbList, err := data_manage.GetEdbInfoListByCond(condition, pars) if err != nil { fmt.Println("查找指标加工数据失败", err.Error()) return } edbClassifyIdMap := make(map[int]bool) classifyIdList := make([]int, 0) edbUserMap := make(map[int]string) for _, v := range edbList { if _, ok := edbClassifyIdMap[v.ClassifyId]; !ok { classifyIdList = append(classifyIdList, v.ClassifyId) edbClassifyIdMap[v.ClassifyId] = true } edbUserMap[v.SysUserId] = v.SysUserRealName } // 获取所有数据加工的分类 classifyList, err := data_manage.GetAllEdbClassifyByType(classifyType, 0) if err != nil { fmt.Println("获取数据加工分类失败", err.Error()) return } hasClassifyIdStrList := make([]string, 0) for _, v := range classifyList { // 判断当前分类id是否加到指标中,如果加进去了,那么才处理 _, ok := edbClassifyIdMap[v.ClassifyId] if !ok { continue } hasClassifyIdStrList = append(hasClassifyIdStrList, v.ClassifyIdPath) } classifyIdStrList := make([]string, 0) if len(hasClassifyIdStrList) > 0 { classifyIdStrList = strings.Split(strings.Join(hasClassifyIdStrList, ","), ",") } classifyIdMap := make(map[string]bool) for _, v := range classifyIdStrList { classifyIdMap[v] = true } newClassifyList := make([]*data_manage.EdbClassifyItems, 0) newClassifyMap := make(map[string]bool) for _, v := range classifyList { classifyIdStr := fmt.Sprint(v.ClassifyId) // 是否在关联指标的map里面,如果不在的话,那么就过滤 _, ok := classifyIdMap[classifyIdStr] if !ok { continue } // 是否已经加到需要关联指标分类的map里面,如果已经在了,那么也过滤 _, ok = newClassifyMap[classifyIdStr] if ok { continue } newClassifyMap[classifyIdStr] = true newClassifyList = append(newClassifyList, v) } rootList, err := data_manage.GetEdbClassifyByParentId(0, int8(classifyType), 0) nodeAll := make([]*data_manage.EdbClassifyItems, 0) for k := range rootList { rootNode := rootList[k] data.EdbClassifyItemsMakeTree(newClassifyList, rootNode) nodeAll = append(nodeAll, rootNode) } // 创建新的分类 // [用户id][原分类id][新分类id] userClassifyIdMap := make(map[int]map[int]int) for userId, sysUserName := range edbUserMap { classifyRelationMap := make(map[int]int) for _, v := range nodeAll { createNewEdbClassify(userId, 0, 0, 0, sysUserName, ``, ``, v, classifyRelationMap) } userClassifyIdMap[userId] = classifyRelationMap } // ##### 第二步,指标数据迁移 // 指标数据迁移 for _, v := range edbList { userClassifyMap, ok := userClassifyIdMap[v.SysUserId] if !ok { fmt.Println("找不到该用户所属的分类,userId:", v.SysUserId) continue } newClassifyId, ok := userClassifyMap[v.ClassifyId] if !ok { fmt.Println("找不到该用户所属的分类,classifyId:", v.ClassifyId) continue } // 开始迁移指标数据 err = data_manage.UpdateClassifyIdByEdbInfoId(v.EdbInfoId, newClassifyId) if err != nil { fmt.Println("指标数据迁移失败", err.Error()) } } // ##### 3、开始删除没有指标的分类 deleteNullEdbClassify(classifyType) return } func deleteNullEdbClassify(classifyType int) { // 指标列表数据 condition := ` AND edb_info_type = 0 and edb_type = 2 ` if classifyType == utils.EdbClassifyTypePredict { condition = ` AND edb_info_type = 1 ` } pars := make([]interface{}, 0) edbList, err := data_manage.GetEdbInfoListByCond(condition, pars) if err != nil { fmt.Println("查找指标加工数据失败", err.Error()) return } edbClassifyIdMap := make(map[int]bool) classifyIdList := make([]int, 0) for _, v := range edbList { if _, ok := edbClassifyIdMap[v.ClassifyId]; !ok { classifyIdList = append(classifyIdList, v.ClassifyId) edbClassifyIdMap[v.ClassifyId] = true } } // 获取所有数据加工的分类 classifyList, err := data_manage.GetAllEdbClassifyByType(classifyType, 0) if err != nil { fmt.Println("获取数据加工分类失败", err.Error()) return } // 存在指标的分类 hasEdbClassifyList := make([]*data_manage.EdbClassifyItems, 0) for _, v := range classifyList { if _, ok := edbClassifyIdMap[v.ClassifyId]; ok { hasEdbClassifyList = append(hasEdbClassifyList, v) } } // 待删除的分类id集合 deleteClassifyIdMap := make(map[int]int) // 全部的分类id集合 for _, v := range classifyList { deleteClassifyIdMap[v.ClassifyId] = v.ClassifyId } // 存在指标的的分类id集合 hasClassifyIdStrList := make([]string, 0) for _, v := range hasEdbClassifyList { hasClassifyIdStrList = append(hasClassifyIdStrList, v.ClassifyIdPath) } classifyIdStrList := make([]string, 0) if len(hasClassifyIdStrList) > 0 { classifyIdStrList = strings.Split(strings.Join(hasClassifyIdStrList, ","), ",") } for _, v := range classifyIdStrList { classifyId, err := strconv.Atoi(v) if err != nil { fmt.Println("分类字符串转int失败,", err.Error()) return } delete(deleteClassifyIdMap, classifyId) } // 删除空分类 if len(deleteClassifyIdMap) > 0 { deleteClassifyIdList := make([]int, 0) for _, v := range deleteClassifyIdMap { deleteClassifyIdList = append(deleteClassifyIdList, v) } err = data_manage.DelEdbClassifyByIdList(deleteClassifyIdList) if err != nil { fmt.Println("删除空分类失败", err.Error()) return } } } // createNewEdbClassify // @Description: 创建分类 // @author: Roc // @datetime 2024-11-29 17:57:47 // @param userId int // @param parentId int // @param rootId int // @param level int // @param sysUserName string // @param parentClassifyNamePath string // @param parentClassifyIdPath string // @param node *data_manage.EdbClassifyItems // @param classifyRelationMap map[int]int func createNewEdbClassify(userId, parentId, rootId, level int, sysUserName, parentClassifyNamePath, parentClassifyIdPath string, node *data_manage.EdbClassifyItems, classifyRelationMap map[int]int) { var classifyNamePath, classifyIdPath string if parentId > 0 { classifyNamePath = fmt.Sprint(parentClassifyNamePath, "|", node.ClassifyName) classifyIdPath = fmt.Sprint(parentClassifyIdPath, ",") } else { classifyNamePath = node.ClassifyName } timestamp := strconv.FormatInt(time.Now().UnixNano(), 10) classifyInfo := &data_manage.EdbClassify{ //ClassifyId: 0, ClassifyType: node.ClassifyType, ClassifyName: node.ClassifyName, ClassifyNameEn: node.ClassifyNameEn, ParentId: parentId, RootId: rootId, HasData: 1, CreateTime: time.Now(), ModifyTime: time.Now(), SysUserId: userId, SysUserRealName: sysUserName, Level: level + 1, UniqueCode: utils.MD5(utils.DATA_PREFIX + "_" + timestamp), Sort: node.Sort, ClassifyNamePath: classifyNamePath, } err := data_manage.AddEdbClassify(classifyInfo) if err != nil { fmt.Println("创建分类失败:", err.Error()) return } updateCols := []string{"ClassifyIdPath"} // 更改分类id完整路径path classifyInfo.ClassifyIdPath = fmt.Sprint(classifyIdPath, classifyInfo.ClassifyId) if rootId <= 0 { rootId = classifyInfo.ClassifyId classifyInfo.RootId = classifyInfo.ClassifyId updateCols = append(updateCols, "RootId") } _ = classifyInfo.Update(updateCols) classifyRelationMap[node.ClassifyId] = classifyInfo.ClassifyId level = classifyInfo.Level if node.Children == nil { return } if len(node.Children) <= 0 { return } for _, child := range node.Children { createNewEdbClassify(userId, classifyInfo.ClassifyId, rootId, level, sysUserName, classifyNamePath, classifyInfo.ClassifyIdPath, child, classifyRelationMap) } return } // initFixChartClassifyPath // @Description: 修复图表分类完整路径 // @author: Roc // @datetime 2024-11-26 15:40:57 func initFixChartClassifyPath() { fmt.Println("开始修复图表分类路径,请稍等...") allList := make([]*data_manage.ChartClassifyItems, 0) allList, e := data_manage.GetAllChartClassify() if e != nil { fmt.Println("获取分类失败", e) return } allList = data.GetChartClassifyTreeRecursive(allList, 0) //根据sort值排序 for _, v := range allList { updateChartPath(v, "", "") } fmt.Println("修复图表分类路径完成") } // initFixSelfChartClassify // @Description: 修复用户图表分类信息 // @author: Roc // @datetime 2024-11-29 16:57:54 func initFixSelfChartClassify() { fmt.Println("修复用户图表分类信息开始,请稍后...") // 图表数据明细 condition := `` pars := make([]interface{}, 0) chartList, err := data_manage.GetChartListByCondition(condition, pars, 0, 1000000) if err != nil { fmt.Println("查找用户图表列表数据失败", err.Error()) return } chartClassifyIdMap := make(map[int]bool) classifyIdList := make([]int, 0) edbUserMap := make(map[int]string) for _, v := range chartList { if _, ok := chartClassifyIdMap[v.ChartClassifyId]; !ok { classifyIdList = append(classifyIdList, v.ChartClassifyId) chartClassifyIdMap[v.ChartClassifyId] = true } edbUserMap[v.SysUserId] = v.SysUserRealName } // 获取所有图表的分类 classifyList, err := data_manage.GetAllChartClassify() if err != nil { fmt.Println("获取图表分类失败", err.Error()) return } hasClassifyIdStrList := make([]string, 0) for _, v := range classifyList { // 判断当前分类id是否加到指标中,如果加进去了,那么才处理 _, ok := chartClassifyIdMap[v.ChartClassifyId] if !ok { continue } hasClassifyIdStrList = append(hasClassifyIdStrList, v.ChartClassifyIdPath) } classifyIdStrList := make([]string, 0) if len(hasClassifyIdStrList) > 0 { classifyIdStrList = strings.Split(strings.Join(hasClassifyIdStrList, ","), ",") } classifyIdMap := make(map[string]bool) for _, v := range classifyIdStrList { classifyIdMap[v] = true } newClassifyList := make([]*data_manage.ChartClassifyItems, 0) newClassifyMap := make(map[string]bool) for _, v := range classifyList { classifyIdStr := fmt.Sprint(v.ChartClassifyId) // 是否在关联指标的map里面,如果不在的话,那么就过滤 _, ok := classifyIdMap[classifyIdStr] if !ok { continue } // 是否已经加到需要关联指标分类的map里面,如果已经在了,那么也过滤 _, ok = newClassifyMap[classifyIdStr] if ok { continue } newClassifyMap[classifyIdStr] = true newClassifyList = append(newClassifyList, v) } rootList, err := data_manage.GetAllChartClassifyByParentId(0) nodeAll := make([]*data_manage.ChartClassifyItems, 0) for k := range rootList { rootNode := rootList[k] data.ChartClassifyItemsMakeTreeV2(newClassifyList, rootNode) nodeAll = append(nodeAll, rootNode) } // 创建新的分类 // [用户id][原分类id][新分类id] userClassifyIdMap := make(map[int]map[int]int) for userId, sysUserName := range edbUserMap { classifyRelationMap := make(map[int]int) for _, v := range nodeAll { createNewChartClassify(userId, 0, 0, 0, sysUserName, ``, ``, v, classifyRelationMap) } userClassifyIdMap[userId] = classifyRelationMap } // ##### 第二步,图表数据迁移 // 指标数据迁移 for _, v := range chartList { userClassifyMap, ok := userClassifyIdMap[v.SysUserId] if !ok { fmt.Println("找不到该用户所属的分类,userId:", v.SysUserId) continue } newClassifyId, ok := userClassifyMap[v.ChartClassifyId] if !ok { fmt.Println("找不到该用户所属的图表分类,classifyId:", v.ChartClassifyId) continue } // 开始迁移指标数据 err = data_manage.UpdateClassifyIdByChartInfoId(v.ChartInfoId, newClassifyId) if err != nil { fmt.Println("图表数据迁移失败", err.Error()) } } // ##### 3、开始删除没有图表的分类 deleteNullChartClassify() fmt.Println("修复用户图表分类信息完成") return } func deleteNullChartClassify() { // 图表数据明细 condition := `` pars := make([]interface{}, 0) chartList, err := data_manage.GetChartListByCondition(condition, pars, 0, 1000000) if err != nil { fmt.Println("查找指标加工数据失败", err.Error()) return } chartClassifyIdMap := make(map[int]bool) classifyIdList := make([]int, 0) for _, v := range chartList { if _, ok := chartClassifyIdMap[v.ChartClassifyId]; !ok { classifyIdList = append(classifyIdList, v.ChartClassifyId) chartClassifyIdMap[v.ChartClassifyId] = true } } // 获取所有数据加工的分类 classifyList, err := data_manage.GetAllChartClassify() if err != nil { fmt.Println("获取数据加工分类失败", err.Error()) return } // 存在图表的分类 hasEdbClassifyList := make([]*data_manage.ChartClassifyItems, 0) for _, v := range classifyList { if _, ok := chartClassifyIdMap[v.ChartClassifyId]; ok { hasEdbClassifyList = append(hasEdbClassifyList, v) } } // 待删除的分类id集合 deleteClassifyIdMap := make(map[int]int) // 全部的分类id集合 for _, v := range classifyList { deleteClassifyIdMap[v.ChartClassifyId] = v.ChartClassifyId } // 存在指标的的分类id集合 hasClassifyIdStrList := make([]string, 0) for _, v := range hasEdbClassifyList { hasClassifyIdStrList = append(hasClassifyIdStrList, v.ChartClassifyIdPath) } classifyIdStrList := make([]string, 0) if len(hasClassifyIdStrList) > 0 { classifyIdStrList = strings.Split(strings.Join(hasClassifyIdStrList, ","), ",") } for _, v := range classifyIdStrList { classifyId, err := strconv.Atoi(v) if err != nil { fmt.Println("分类字符串转int失败,", err.Error()) return } delete(deleteClassifyIdMap, classifyId) } // 删除空分类 if len(deleteClassifyIdMap) > 0 { deleteClassifyIdList := make([]int, 0) for _, v := range deleteClassifyIdMap { deleteClassifyIdList = append(deleteClassifyIdList, v) } err = data_manage.DelChartClassifyByIdList(deleteClassifyIdList) if err != nil { fmt.Println("删除空分类失败", err.Error()) return } } } // createNewChartClassify // @Description: 创建图表分类 // @author: Roc // @datetime 2024-11-29 17:57:47 // @param userId int // @param parentId int // @param rootId int // @param level int // @param sysUserName string // @param parentClassifyNamePath string // @param parentClassifyIdPath string // @param node *data_manage.EdbClassifyItems // @param classifyRelationMap map[int]int func createNewChartClassify(userId, parentId, rootId, level int, sysUserName, parentClassifyNamePath, parentClassifyIdPath string, node *data_manage.ChartClassifyItems, classifyRelationMap map[int]int) { var classifyNamePath, classifyIdPath string if parentId > 0 { classifyNamePath = fmt.Sprint(parentClassifyNamePath, "|", node.ChartClassifyName) classifyIdPath = fmt.Sprint(parentClassifyIdPath, ",") } else { classifyNamePath = node.ChartClassifyName } timestamp := strconv.FormatInt(time.Now().UnixNano(), 10) classifyInfo := &data_manage.ChartClassify{ ChartClassifyId: 0, ChartClassifyName: node.ChartClassifyName, ParentId: parentId, HasData: 1, CreateTime: time.Now(), ModifyTime: time.Now(), SysUserId: userId, SysUserRealName: sysUserName, Level: level + 1, UniqueCode: utils.MD5(utils.CHART_PREFIX + "_" + timestamp), Sort: node.Sort, Source: node.Source, //IsJoinPermission: 0, ChartClassifyNameEn: node.ChartClassifyNameEn, RootId: rootId, ChartClassifyNamePath: classifyNamePath, ChartClassifyIdPath: "", } err := data_manage.AddChartClassify(classifyInfo) if err != nil { fmt.Println("创建分类失败:", err.Error()) return } updateCols := []string{"ChartClassifyIdPath"} // 更改分类id完整路径path classifyInfo.ChartClassifyIdPath = fmt.Sprint(classifyIdPath, classifyInfo.ChartClassifyId) if rootId <= 0 { rootId = classifyInfo.ChartClassifyId classifyInfo.RootId = classifyInfo.ChartClassifyId updateCols = append(updateCols, "RootId") } _ = classifyInfo.Update(updateCols) classifyRelationMap[node.ChartClassifyId] = classifyInfo.ChartClassifyId level = classifyInfo.Level if node.Children == nil { return } if len(node.Children) <= 0 { return } for _, child := range node.Children { createNewChartClassify(userId, classifyInfo.ChartClassifyId, rootId, level, sysUserName, classifyNamePath, classifyInfo.ChartClassifyIdPath, child, classifyRelationMap) } return } // updateChartPath // @Description: 更新图表分类的完整路径 // @author: Roc // @datetime 2024-12-09 10:30:16 // @param item *data_manage.ChartClassifyItems // @param classifyNamePath string // @param classifyIdPath string func updateChartPath(item *data_manage.ChartClassifyItems, classifyNamePath, classifyIdPath string) { if classifyNamePath == `` { classifyNamePath = item.ChartClassifyName classifyIdPath = fmt.Sprint(item.ChartClassifyId) } else { classifyNamePath = fmt.Sprint(classifyNamePath, "|", item.ChartClassifyName) classifyIdPath = fmt.Sprint(classifyIdPath, ",", item.ChartClassifyId) } err := data_manage.UpdateChartClassify(item.ChartClassifyId, classifyNamePath, classifyIdPath) if err != nil { fmt.Println(item.ChartClassifyId, ";更新失败", err) } if item.Children != nil { for _, v := range item.Children { updateChartPath(v, classifyNamePath, classifyIdPath) } } } // initEs // @Description: 修复ES中的指标和图表数据 // @author: Roc // @datetime 2024-11-29 16:18:02 func initEs() { // 更新ES中的指标数据 //data.AddOrEditAllEdbInfoToEs() // 更新es中的图表数据 data.AddAllChartInfo() fmt.Println("全部es数据修复完成") }