eta_api/controllers/fix.go

package controllers

import (
	"eta_gn/eta_api/models/data_manage"
	"eta_gn/eta_api/services/data"
	"eta_gn/eta_api/utils"
	"fmt"
	"sort"
	"strconv"
	"strings"
	"time"
)

// init
// @Description: 数据修复
// @author: Roc
// @datetime 2024-12-02 09:29:56
func init() {
	// ### 1、先修复指标分类完整路径
	//initFixEdbClassifyPath()

	// ### 2、再修复个人分类、删除不存在指标的分类
	// ### 2.1 数据加工
	//initFixSelfEdbClassify(utils.EdbClassifyTypeCalculate)
	//
	//// ### 2.2 衍生指标
	//initFixSelfEdbClassify(utils.EdbClassifyTypePredict)
	//

	// 开始图表数据了

	// ### 3、修复图表分类完整路径
	//initFixChartClassifyPath()

	// ### 4、修复图表分类完整路径
	//initFixSelfChartClassify()

	// ### 5、修复ES数据
	//initEs()

	//fmt.Println("修复完成")
}

// initFixEdbClassifyPath
// @Description: 修复分类完整路径
// @author: Roc
// @datetime 2024-11-26 15:40:57
func initFixEdbClassifyPath() {
	allList := make([]*data_manage.EdbClassifyItems, 0)
	allList, e := data_manage.GetEdbClassifyByClassifyTypes([]int{utils.EdbClassifyTypeBase, utils.EdbClassifyTypePredict, utils.EdbClassifyTypeCalculate}, 0)
	if e != nil {
		fmt.Println("获取分类失败", e)
		return
	}

	var sortList data_manage.EdbClassifyItemList

	allList = data.GetClassifyTreeRecursive(allList, 0)
	//根据sort值排序
	sortList = allList
	sort.Sort(sortList)

	for _, v := range sortList {
		updateEdbPath(v, "", "")
	}
	fmt.Println("修复分类路径完成")
}

func updateEdbPath(item *data_manage.EdbClassifyItems, classifyNamePath, classifyIdPath string) {
	if classifyNamePath == `` {
		classifyNamePath = item.ClassifyName
		classifyIdPath = fmt.Sprint(item.ClassifyId)
	} else {
		classifyNamePath = fmt.Sprint(classifyNamePath, "|", item.ClassifyName)
		classifyIdPath = fmt.Sprint(classifyIdPath, ",", item.ClassifyId)
	}
	err := data_manage.UpdateEdbClassify(item.ClassifyId, classifyNamePath, classifyIdPath)
	if err != nil {
		fmt.Println(item.ClassifyId, ";更新失败", err)
	}
	if item.Children != nil {
		for _, v := range item.Children {
			updateEdbPath(v, classifyNamePath, classifyIdPath)
		}
	}
}

// initFixSelfEdbClassify
// @Description: initFixUserClassify
// @author: Roc
// @datetime 2024-11-29 16:57:54
func initFixSelfEdbClassify(classifyType int) {
	// 指标数据明细
	condition := ` AND edb_info_type = 0 and edb_type = 2 `
	if classifyType == utils.EdbClassifyTypePredict {
		condition = ` AND edb_info_type = 1 `
	}

	pars := make([]interface{}, 0)
	edbList, err := data_manage.GetEdbInfoListByCond(condition, pars)
	if err != nil {
		fmt.Println("查找指标加工数据失败", err.Error())
		return
	}

	edbClassifyIdMap := make(map[int]bool)
	classifyIdList := make([]int, 0)

	edbUserMap := make(map[int]string)

	for _, v := range edbList {
		if _, ok := edbClassifyIdMap[v.ClassifyId]; !ok {
			classifyIdList = append(classifyIdList, v.ClassifyId)
			edbClassifyIdMap[v.ClassifyId] = true
		}

		edbUserMap[v.SysUserId] = v.SysUserRealName
	}

	// 获取所有数据加工的分类
	classifyList, err := data_manage.GetAllEdbClassifyByType(classifyType, 0)
	if err != nil {
		fmt.Println("获取数据加工分类失败", err.Error())
		return
	}

	hasClassifyIdStrList := make([]string, 0)
	for _, v := range classifyList {
		// 判断当前分类id是否加到指标中，如果加进去了，那么才处理
		_, ok := edbClassifyIdMap[v.ClassifyId]
		if !ok {
			continue
		}
		hasClassifyIdStrList = append(hasClassifyIdStrList, v.ClassifyIdPath)
	}
	classifyIdStrList := strings.Split(strings.Join(hasClassifyIdStrList, ","), ",")
	classifyIdMap := make(map[string]bool)
	for _, v := range classifyIdStrList {
		classifyIdMap[v] = true
	}
	newClassifyList := make([]*data_manage.EdbClassifyItems, 0)
	newClassifyMap := make(map[string]bool)
	for _, v := range classifyList {
		classifyIdStr := fmt.Sprint(v.ClassifyId)

		// 是否在关联指标的map里面，如果不在的话，那么就过滤
		_, ok := classifyIdMap[classifyIdStr]
		if !ok {
			continue
		}

		// 是否已经加到需要关联指标分类的map里面，如果已经在了，那么也过滤
		_, ok = newClassifyMap[classifyIdStr]
		if ok {
			continue
		}

		newClassifyMap[classifyIdStr] = true
		newClassifyList = append(newClassifyList, v)
	}

	rootList, err := data_manage.GetEdbClassifyByParentId(0, int8(classifyType), 0)
	nodeAll := make([]*data_manage.EdbClassifyItems, 0)
	for k := range rootList {
		rootNode := rootList[k]
		data.EdbClassifyItemsMakeTree(newClassifyList, rootNode)
		nodeAll = append(nodeAll, rootNode)
	}

	// 创建新的分类
	// [用户id][原分类id][新分类id]
	userClassifyIdMap := make(map[int]map[int]int)
	for userId, sysUserName := range edbUserMap {
		classifyRelationMap := make(map[int]int)
		for _, v := range nodeAll {
			createNewEdbClassify(userId, 0, 0, 0, sysUserName, ``, ``, v, classifyRelationMap)
		}
		userClassifyIdMap[userId] = classifyRelationMap
	}

	// ##### 第二步，指标数据迁移

	// 指标数据迁移
	for _, v := range edbList {
		userClassifyMap, ok := userClassifyIdMap[v.SysUserId]
		if !ok {
			fmt.Println("找不到该用户所属的分类，userId:", v.SysUserId)
			continue
		}

		newClassifyId, ok := userClassifyMap[v.ClassifyId]
		if !ok {
			fmt.Println("找不到该用户所属的分类，classifyId:", v.ClassifyId)
			continue
		}
		// 开始迁移指标数据
		err = data_manage.UpdateClassifyIdByEdbInfoId(v.EdbInfoId, newClassifyId)
		if err != nil {
			fmt.Println("指标数据迁移失败", err.Error())
		}
	}

	// ##### 3、开始删除没有指标的分类
	deleteNullEdbClassify(classifyType)
	return
}

func deleteNullEdbClassify(classifyType int) {
	// 指标列表数据
	condition := ` AND edb_info_type = 0 and edb_type = 2 `
	if classifyType == utils.EdbClassifyTypePredict {
		condition = ` AND edb_info_type = 1 `
	}

	pars := make([]interface{}, 0)
	edbList, err := data_manage.GetEdbInfoListByCond(condition, pars)
	if err != nil {
		fmt.Println("查找指标加工数据失败", err.Error())
		return
	}

	edbClassifyIdMap := make(map[int]bool)
	classifyIdList := make([]int, 0)

	for _, v := range edbList {
		if _, ok := edbClassifyIdMap[v.ClassifyId]; !ok {
			classifyIdList = append(classifyIdList, v.ClassifyId)
			edbClassifyIdMap[v.ClassifyId] = true
		}
	}

	// 获取所有数据加工的分类
	classifyList, err := data_manage.GetAllEdbClassifyByType(classifyType, 0)
	if err != nil {
		fmt.Println("获取数据加工分类失败", err.Error())
		return
	}

	// 存在指标的分类
	hasEdbClassifyList := make([]*data_manage.EdbClassifyItems, 0)
	for _, v := range classifyList {
		if _, ok := edbClassifyIdMap[v.ClassifyId]; ok {
			hasEdbClassifyList = append(hasEdbClassifyList, v)
		}
	}

	// 待删除的分类id集合
	deleteClassifyIdMap := make(map[int]int)
	// 全部的分类id集合
	for _, v := range classifyList {
		deleteClassifyIdMap[v.ClassifyId] = v.ClassifyId
	}

	// 存在指标的的分类id集合
	hasClassifyIdStrList := make([]string, 0)
	for _, v := range hasEdbClassifyList {
		hasClassifyIdStrList = append(hasClassifyIdStrList, v.ClassifyIdPath)
	}
	classifyIdStrList := strings.Split(strings.Join(hasClassifyIdStrList, ","), ",")
	for _, v := range classifyIdStrList {
		classifyId, err := strconv.Atoi(v)
		if err != nil {
			fmt.Println("分类字符串转int失败，", err.Error())
			return
		}
		delete(deleteClassifyIdMap, classifyId)
	}

	// 删除空分类
	if len(deleteClassifyIdMap) > 0 {
		deleteClassifyIdList := make([]int, 0)
		for _, v := range deleteClassifyIdMap {
			deleteClassifyIdList = append(deleteClassifyIdList, v)
		}

		err = data_manage.DelEdbClassifyByIdList(deleteClassifyIdList)
		if err != nil {
			fmt.Println("删除空分类失败", err.Error())
			return
		}
	}
}

// createNewEdbClassify
// @Description: 创建分类
// @author: Roc
// @datetime 2024-11-29 17:57:47
// @param userId int
// @param parentId int
// @param rootId int
// @param level int
// @param sysUserName string
// @param parentClassifyNamePath string
// @param parentClassifyIdPath string
// @param node *data_manage.EdbClassifyItems
// @param classifyRelationMap map[int]int
func createNewEdbClassify(userId, parentId, rootId, level int, sysUserName, parentClassifyNamePath, parentClassifyIdPath string, node *data_manage.EdbClassifyItems, classifyRelationMap map[int]int) {
	var classifyNamePath, classifyIdPath string

	if parentId > 0 {
		classifyNamePath = fmt.Sprint(parentClassifyNamePath, "|", node.ClassifyName)
		classifyIdPath = fmt.Sprint(parentClassifyIdPath, ",")
	} else {
		classifyNamePath = node.ClassifyName
	}

	timestamp := strconv.FormatInt(time.Now().UnixNano(), 10)
	classifyInfo := &data_manage.EdbClassify{
		//ClassifyId:      0,
		ClassifyType:     node.ClassifyType,
		ClassifyName:     node.ClassifyName,
		ClassifyNameEn:   node.ClassifyNameEn,
		ParentId:         parentId,
		RootId:           rootId,
		HasData:          1,
		CreateTime:       time.Now(),
		ModifyTime:       time.Now(),
		SysUserId:        userId,
		SysUserRealName:  sysUserName,
		Level:            level + 1,
		UniqueCode:       utils.MD5(utils.DATA_PREFIX + "_" + timestamp),
		Sort:             node.Sort,
		ClassifyNamePath: classifyNamePath,
	}
	err := data_manage.AddEdbClassify(classifyInfo)
	if err != nil {
		fmt.Println("创建分类失败：", err.Error())
		return
	}

	updateCols := []string{"ClassifyIdPath"}
	// 更改分类id完整路径path
	classifyInfo.ClassifyIdPath = fmt.Sprint(classifyIdPath, classifyInfo.ClassifyId)

	if rootId <= 0 {
		rootId = classifyInfo.ClassifyId
		classifyInfo.RootId = classifyInfo.ClassifyId
		updateCols = append(updateCols, "RootId")
	}

	_ = classifyInfo.Update(updateCols)

	classifyRelationMap[node.ClassifyId] = classifyInfo.ClassifyId

	level = classifyInfo.Level

	if node.Children == nil {
		return
	}
	if len(node.Children) <= 0 {
		return
	}
	for _, child := range node.Children {
		createNewEdbClassify(userId, classifyInfo.ClassifyId, rootId, level, sysUserName, classifyNamePath, classifyInfo.ClassifyIdPath, child, classifyRelationMap)
	}

	return
}

// initFixChartClassifyPath
// @Description: 修复图表分类完整路径
// @author: Roc
// @datetime 2024-11-26 15:40:57
func initFixChartClassifyPath() {
	fmt.Println("开始修复图表分类路径，请稍等...")
	allList := make([]*data_manage.ChartClassifyItems, 0)
	allList, e := data_manage.GetAllChartClassify()
	if e != nil {
		fmt.Println("获取分类失败", e)
		return
	}

	allList = data.GetChartClassifyTreeRecursive(allList, 0)
	//根据sort值排序

	for _, v := range allList {
		updateChartPath(v, "", "")
	}
	fmt.Println("修复图表分类路径完成")
}

// initFixSelfChartClassify
// @Description: 修复用户图表分类信息
// @author: Roc
// @datetime 2024-11-29 16:57:54
func initFixSelfChartClassify() {
	fmt.Println("修复用户图表分类信息开始，请稍后...")
	// 图表数据明细
	condition := ``

	pars := make([]interface{}, 0)
	chartList, err := data_manage.GetChartListByCondition(condition, pars, 0, 1000000)
	if err != nil {
		fmt.Println("查找用户图表列表数据失败", err.Error())
		return
	}

	chartClassifyIdMap := make(map[int]bool)
	classifyIdList := make([]int, 0)

	edbUserMap := make(map[int]string)

	for _, v := range chartList {
		if _, ok := chartClassifyIdMap[v.ChartClassifyId]; !ok {
			classifyIdList = append(classifyIdList, v.ChartClassifyId)
			chartClassifyIdMap[v.ChartClassifyId] = true
		}

		edbUserMap[v.SysUserId] = v.SysUserRealName
	}

	// 获取所有图表的分类
	classifyList, err := data_manage.GetAllChartClassify()
	if err != nil {
		fmt.Println("获取图表分类失败", err.Error())
		return
	}

	hasClassifyIdStrList := make([]string, 0)
	for _, v := range classifyList {
		// 判断当前分类id是否加到指标中，如果加进去了，那么才处理
		_, ok := chartClassifyIdMap[v.ChartClassifyId]
		if !ok {
			continue
		}
		hasClassifyIdStrList = append(hasClassifyIdStrList, v.ChartClassifyIdPath)
	}
	classifyIdStrList := strings.Split(strings.Join(hasClassifyIdStrList, ","), ",")
	classifyIdMap := make(map[string]bool)
	for _, v := range classifyIdStrList {
		classifyIdMap[v] = true
	}
	newClassifyList := make([]*data_manage.ChartClassifyItems, 0)
	newClassifyMap := make(map[string]bool)
	for _, v := range classifyList {
		classifyIdStr := fmt.Sprint(v.ChartClassifyId)

		// 是否在关联指标的map里面，如果不在的话，那么就过滤
		_, ok := classifyIdMap[classifyIdStr]
		if !ok {
			continue
		}

		// 是否已经加到需要关联指标分类的map里面，如果已经在了，那么也过滤
		_, ok = newClassifyMap[classifyIdStr]
		if ok {
			continue
		}

		newClassifyMap[classifyIdStr] = true
		newClassifyList = append(newClassifyList, v)
	}

	rootList, err := data_manage.GetAllChartClassifyByParentId(0)
	nodeAll := make([]*data_manage.ChartClassifyItems, 0)
	for k := range rootList {
		rootNode := rootList[k]
		data.ChartClassifyItemsMakeTreeV2(newClassifyList, rootNode)
		nodeAll = append(nodeAll, rootNode)
	}

	// 创建新的分类
	// [用户id][原分类id][新分类id]
	userClassifyIdMap := make(map[int]map[int]int)
	for userId, sysUserName := range edbUserMap {
		classifyRelationMap := make(map[int]int)
		for _, v := range nodeAll {
			createNewChartClassify(userId, 0, 0, 0, sysUserName, ``, ``, v, classifyRelationMap)
		}
		userClassifyIdMap[userId] = classifyRelationMap
	}

	// ##### 第二步，图表数据迁移

	// 指标数据迁移
	for _, v := range chartList {
		userClassifyMap, ok := userClassifyIdMap[v.SysUserId]
		if !ok {
			fmt.Println("找不到该用户所属的分类，userId:", v.SysUserId)
			continue
		}

		newClassifyId, ok := userClassifyMap[v.ChartClassifyId]
		if !ok {
			fmt.Println("找不到该用户所属的图表分类，classifyId:", v.ChartClassifyId)
			continue
		}
		// 开始迁移指标数据
		err = data_manage.UpdateClassifyIdByChartInfoId(v.ChartInfoId, newClassifyId)
		if err != nil {
			fmt.Println("图表数据迁移失败", err.Error())
		}
	}

	// ##### 3、开始删除没有图表的分类
	deleteNullChartClassify()

	fmt.Println("修复用户图表分类信息完成")
	return
}

func deleteNullChartClassify() {
	// 图表数据明细
	condition := ``

	pars := make([]interface{}, 0)
	chartList, err := data_manage.GetChartListByCondition(condition, pars, 0, 1000000)
	if err != nil {
		fmt.Println("查找指标加工数据失败", err.Error())
		return
	}

	chartClassifyIdMap := make(map[int]bool)
	classifyIdList := make([]int, 0)

	for _, v := range chartList {
		if _, ok := chartClassifyIdMap[v.ChartClassifyId]; !ok {
			classifyIdList = append(classifyIdList, v.ChartClassifyId)
			chartClassifyIdMap[v.ChartClassifyId] = true
		}
	}

	// 获取所有数据加工的分类
	classifyList, err := data_manage.GetAllChartClassify()
	if err != nil {
		fmt.Println("获取数据加工分类失败", err.Error())
		return
	}

	// 存在图表的分类
	hasEdbClassifyList := make([]*data_manage.ChartClassifyItems, 0)
	for _, v := range classifyList {
		if _, ok := chartClassifyIdMap[v.ChartClassifyId]; ok {
			hasEdbClassifyList = append(hasEdbClassifyList, v)
		}
	}

	// 待删除的分类id集合
	deleteClassifyIdMap := make(map[int]int)
	// 全部的分类id集合
	for _, v := range classifyList {
		deleteClassifyIdMap[v.ChartClassifyId] = v.ChartClassifyId
	}

	// 存在指标的的分类id集合
	hasClassifyIdStrList := make([]string, 0)
	for _, v := range hasEdbClassifyList {
		hasClassifyIdStrList = append(hasClassifyIdStrList, v.ChartClassifyIdPath)
	}
	classifyIdStrList := strings.Split(strings.Join(hasClassifyIdStrList, ","), ",")
	for _, v := range classifyIdStrList {
		classifyId, err := strconv.Atoi(v)
		if err != nil {
			fmt.Println("分类字符串转int失败，", err.Error())
			return
		}
		delete(deleteClassifyIdMap, classifyId)
	}

	// 删除空分类
	if len(deleteClassifyIdMap) > 0 {
		deleteClassifyIdList := make([]int, 0)
		for _, v := range deleteClassifyIdMap {
			deleteClassifyIdList = append(deleteClassifyIdList, v)
		}

		err = data_manage.DelChartClassifyByIdList(deleteClassifyIdList)
		if err != nil {
			fmt.Println("删除空分类失败", err.Error())
			return
		}
	}
}

// createNewChartClassify
// @Description: 创建图表分类
// @author: Roc
// @datetime 2024-11-29 17:57:47
// @param userId int
// @param parentId int
// @param rootId int
// @param level int
// @param sysUserName string
// @param parentClassifyNamePath string
// @param parentClassifyIdPath string
// @param node *data_manage.EdbClassifyItems
// @param classifyRelationMap map[int]int
func createNewChartClassify(userId, parentId, rootId, level int, sysUserName, parentClassifyNamePath, parentClassifyIdPath string, node *data_manage.ChartClassifyItems, classifyRelationMap map[int]int) {
	var classifyNamePath, classifyIdPath string

	if parentId > 0 {
		classifyNamePath = fmt.Sprint(parentClassifyNamePath, "|", node.ChartClassifyName)
		classifyIdPath = fmt.Sprint(parentClassifyIdPath, ",")
	} else {
		classifyNamePath = node.ChartClassifyName
	}

	timestamp := strconv.FormatInt(time.Now().UnixNano(), 10)
	classifyInfo := &data_manage.ChartClassify{
		ChartClassifyId:   0,
		ChartClassifyName: node.ChartClassifyName,
		ParentId:          parentId,
		HasData:           1,
		CreateTime:        time.Now(),
		ModifyTime:        time.Now(),
		SysUserId:         userId,
		SysUserRealName:   sysUserName,
		Level:             level + 1,
		UniqueCode:        utils.MD5(utils.CHART_PREFIX + "_" + timestamp),
		Sort:              node.Sort,
		Source:            node.Source,
		//IsJoinPermission:      0,
		ChartClassifyNameEn:   node.ChartClassifyNameEn,
		RootId:                rootId,
		ChartClassifyNamePath: classifyNamePath,
		ChartClassifyIdPath:   "",
	}
	err := data_manage.AddChartClassify(classifyInfo)
	if err != nil {
		fmt.Println("创建分类失败：", err.Error())
		return
	}

	updateCols := []string{"ChartClassifyIdPath"}
	// 更改分类id完整路径path
	classifyInfo.ChartClassifyIdPath = fmt.Sprint(classifyIdPath, classifyInfo.ChartClassifyId)

	if rootId <= 0 {
		rootId = classifyInfo.ChartClassifyId
		classifyInfo.RootId = classifyInfo.ChartClassifyId
		updateCols = append(updateCols, "RootId")
	}

	_ = classifyInfo.Update(updateCols)

	classifyRelationMap[node.ChartClassifyId] = classifyInfo.ChartClassifyId

	level = classifyInfo.Level

	if node.Children == nil {
		return
	}
	if len(node.Children) <= 0 {
		return
	}
	for _, child := range node.Children {
		createNewChartClassify(userId, classifyInfo.ChartClassifyId, rootId, level, sysUserName, classifyNamePath, classifyInfo.ChartClassifyIdPath, child, classifyRelationMap)
	}

	return
}

// updateChartPath
// @Description: 更新图表分类的完整路径
// @author: Roc
// @datetime 2024-12-09 10:30:16
// @param item *data_manage.ChartClassifyItems
// @param classifyNamePath string
// @param classifyIdPath string
func updateChartPath(item *data_manage.ChartClassifyItems, classifyNamePath, classifyIdPath string) {
	if classifyNamePath == `` {
		classifyNamePath = item.ChartClassifyName
		classifyIdPath = fmt.Sprint(item.ChartClassifyId)
	} else {
		classifyNamePath = fmt.Sprint(classifyNamePath, "|", item.ChartClassifyName)
		classifyIdPath = fmt.Sprint(classifyIdPath, ",", item.ChartClassifyId)
	}
	err := data_manage.UpdateChartClassify(item.ChartClassifyId, classifyNamePath, classifyIdPath)
	if err != nil {
		fmt.Println(item.ChartClassifyId, ";更新失败", err)
	}
	if item.Children != nil {
		for _, v := range item.Children {
			updateChartPath(v, classifyNamePath, classifyIdPath)
		}
	}
}

// initEs
// @Description: 修复ES中的指标和图表数据
// @author: Roc
// @datetime 2024-11-29 16:18:02
func initEs() {
	// 更新ES中的指标数据
	data.AddOrEditAllEdbInfoToEs()
	// 更新es中的图表数据
	//data.AddAllChartInfo()

	fmt.Println("全部es数据修复完成")
}