Merge branch 'eta/1.2.5'

.gitignore

@@ -17,4 +17,5 @@
 /static/images/*.svg
 eta_api.exe
 eta_api.exe~
-/static/tmpFile/*
+/static/tmpFile/*
+/etalogs

models/data_manage/edb_info.go

@@ -1776,6 +1776,7 @@ type TraceEdbInfoResp struct {
 	UniqueCode  string             `description:"唯一编码"`
 	ClassifyId  int                `description:"分类ID"`
 	Child       []TraceEdbInfoResp `description:"下级来源"`
+	EdbInfo     *EdbInfo           `description:"指标信息" json:"-"`
 }
 
 // BeforeAndAfterDateDataResp 前后几期数据

services/data/edb_info.go

@@ -29,194 +29,15 @@ func EdbInfoRefreshAllFromBaseV2(edbInfoId int, refreshAll bool) (err error, isA
 
 	err, isAsync = EdbInfoRefreshAllFromBaseV3([]int{edbInfoId}, refreshAll, false)
 	return
-	//// 获取关联的基础指标
-	//newBaseEdbInfoArr, newCalculateMap, newPredictCalculateMap, calculateArr, predictCalculateArr, err, errmsg := getRefreshEdbInfoList(edbInfoId)
-	//if err != nil {
-	//	return
-	//}
-	//
-	//var startDate string
-	//for _, bv := range newBaseEdbInfoArr {
-	//	//source := bv.Source
-	//	//edbInfoId := bv.EdbInfoId
-	//	//edbCode := bv.EdbCode
-	//	if bv.StartDate == "0000-00-00" {
-	//		continue
-	//	}
-	//	//开始时间
-	//	startDate = ``
-	//	if refreshAll { //刷新所有数据，用开始时间作为起始日期去刷新
-	//		sTime, err := time.Parse(utils.FormatDate, bv.StartDate)
-	//		if err != nil {
-	//			return err
-	//		}
-	//		startDate = sTime.Format(utils.FormatDate)
-	//	} else {
-	//		sTime, err := time.Parse(utils.FormatDate, bv.EndDate)
-	//		if err != nil {
-	//			return err
-	//		}
-	//		frequency := bv.Frequency
-	//		var limitDay int
-	//		switch frequency {
-	//		case "日度":
-	//			limitDay = utils.DATA_START_REFRESH_LIMIT
-	//		case "周度":
-	//			limitDay = utils.DATA_START_REFRESH_LIMIT * 7
-	//		case "月度":
-	//			limitDay = utils.DATA_START_REFRESH_LIMIT * 30
-	//		case "季度":
-	//			limitDay = utils.DATA_START_REFRESH_LIMIT * 90
-	//		case "年度":
-	//			limitDay = utils.DATA_START_REFRESH_LIMIT * 365
-	//		default:
-	//			limitDay = utils.DATA_START_REFRESH_LIMIT
-	//		}
-	//		startDate = sTime.AddDate(0, 0, -limitDay).Format(utils.FormatDate)
-	//	}
-	//	result, err := RefreshEdbData(bv.EdbInfoId, bv.Source, bv.EdbCode, startDate)
-	//	if err != nil {
-	//		fmt.Println(bv.EdbInfoId, "RefreshBaseEdbData err", time.Now())
-	//		errmsg = "RefreshBaseEdbData Err:" + err.Error()
-	//		return err
-	//	}
-	//	if result.Ret != 200 {
-	//		fmt.Println(bv.EdbInfoId, "RefreshBaseEdbData err;msg:", result.Msg, ";errMsg:", result.ErrMsg)
-	//		errmsg = fmt.Sprint(bv.EdbInfoId, "RefreshBaseEdbData err;msg:", result.Msg, ";errMsg:", result.ErrMsg)
-	//		return fmt.Errorf("刷新失败, err:", errmsg)
-	//	}
-	//
-	//	//maxAndMinItem, err := data_manage.GetEdbInfoMaxAndMinInfo(source, edbCode)
-	//	//if err != nil {
-	//	//	if err.Error() == utils.ErrNoRow() { //找不到数据，那么就进入到下一条数据做处理
-	//	//		continue
-	//	//	}
-	//	//	return err
-	//	//}
-	//	//if maxAndMinItem != nil {
-	//	//	err = data_manage.ModifyEdbInfoMaxAndMinInfo(edbInfoId, maxAndMinItem)
-	//	//	if err != nil {
-	//	//		return err
-	//	//	}
-	//	//}
-	//	fmt.Println("end newBaseEdbInfoArr:", bv, time.Now())
-	//}
-	//
-	////刷新相关普通计算指标
-	//for _, v := range calculateArr {
-	//	edbInfo := newCalculateMap[v]
-	//	if edbInfo == nil {
-	//		return err
-	//	}
-	//	startDate = edbInfo.StartDate
-	//	source := edbInfo.Source
-	//	if startDate == "" || startDate == "0000-00-00" { //如果没有开始日期，说明还没有计算出来数据，那么就往前面推40年吧（也意味着重新计算了）
-	//		startDate = time.Now().AddDate(-40, 0, 0).Format(utils.FormatDate)
-	//	} else {
-	//		if source == utils.DATA_SOURCE_CALCULATE {
-	//			startDate = ``
-	//			if refreshAll { //刷新所有数据，用开始时间作为起始日期去刷新
-	//				startDate = edbInfo.StartDate
-	//			} else {
-	//				sTime, err := time.Parse(utils.FormatDate, edbInfo.EndDate)
-	//				if err != nil {
-	//					return err
-	//				}
-	//				frequency := edbInfo.Frequency
-	//				var limitDay int
-	//				switch frequency {
-	//				case "日度":
-	//					limitDay = utils.DATA_START_REFRESH_LIMIT
-	//				case "周度":
-	//					limitDay = utils.DATA_START_REFRESH_LIMIT * 7
-	//				case "月度":
-	//					limitDay = utils.DATA_START_REFRESH_LIMIT * 30
-	//				case "季度":
-	//					limitDay = utils.DATA_START_REFRESH_LIMIT * 90
-	//				case "年度":
-	//					limitDay = utils.DATA_START_REFRESH_LIMIT * 365
-	//				default:
-	//					limitDay = utils.DATA_START_REFRESH_LIMIT
-	//				}
-	//				startDate = sTime.AddDate(0, 0, -limitDay).Format(utils.FormatDate)
-	//			}
-	//		}
-	//	}
-	//
-	//	result, err := RefreshEdbCalculateData(edbInfo.EdbInfoId, edbInfo.EdbCode, startDate)
-	//	if err != nil {
-	//		fmt.Println(v, "RefreshEdbCalculateData err", time.Now())
-	//		errmsg = "RefreshEdbCalculateData Err:" + err.Error()
-	//		return err
-	//	}
-	//	if result.Ret != 200 {
-	//		fmt.Println(v, "RefreshEdbCalculateData err;msg:", result.Msg, ";errMsg:", result.ErrMsg)
-	//		errmsg = fmt.Sprint(v, "RefreshEdbCalculateData err;msg:", result.Msg, ";errMsg:", result.ErrMsg)
-	//		return fmt.Errorf("刷新失败")
-	//	}
-	//}
-	//
-	////刷新相关预测计算指标
-	//for _, v := range predictCalculateArr {
-	//	edbInfo := newPredictCalculateMap[v]
-	//	if edbInfo == nil {
-	//		return err
-	//	}
-	//	startDate = edbInfo.StartDate
-	//	source := edbInfo.Source
-	//	if startDate == "" || startDate == "0000-00-00" { //如果没有开始日期，说明还没有计算出来数据，那么就往前面推40年吧（也意味着重新计算了）
-	//		startDate = time.Now().AddDate(-40, 0, 0).Format(utils.FormatDate)
-	//	} else {
-	//		if source == utils.DATA_SOURCE_PREDICT_CALCULATE {
-	//			startDate = ``
-	//			if refreshAll { //刷新所有数据，用开始时间作为起始日期去刷新
-	//				startDate = edbInfo.StartDate
-	//			} else {
-	//				sTime, err := time.Parse(utils.FormatDate, edbInfo.EndDate)
-	//				if err != nil {
-	//					return err
-	//				}
-	//				frequency := edbInfo.Frequency
-	//				var limitDay int
-	//				switch frequency {
-	//				case "日度":
-	//					limitDay = utils.DATA_START_REFRESH_LIMIT
-	//				case "周度":
-	//					limitDay = utils.DATA_START_REFRESH_LIMIT * 7
-	//				case "月度":
-	//					limitDay = utils.DATA_START_REFRESH_LIMIT * 30
-	//				case "季度":
-	//					limitDay = utils.DATA_START_REFRESH_LIMIT * 90
-	//				case "年度":
-	//					limitDay = utils.DATA_START_REFRESH_LIMIT * 365
-	//				default:
-	//					limitDay = utils.DATA_START_REFRESH_LIMIT
-	//				}
-	//				startDate = sTime.AddDate(0, 0, -limitDay).Format(utils.FormatDate)
-	//			}
-	//		}
-	//	}
-	//
-	//	result, err := RefreshPredictEdbCalculateData(edbInfo.EdbInfoId, edbInfo.EdbCode, startDate)
-	//	if err != nil {
-	//		fmt.Println(v, "RefreshEdbCalculateData err", time.Now())
-	//		errmsg = "RefreshEdbCalculateData Err:" + err.Error()
-	//		return err
-	//	}
-	//	if result.Ret != 200 {
-	//		fmt.Println(v, "RefreshEdbCalculateData err;msg:", result.Msg, ";errMsg:", result.ErrMsg)
-	//		errmsg = fmt.Sprint(v, "RefreshEdbCalculateData err;msg:", result.Msg, ";errMsg:", result.ErrMsg)
-	//		return fmt.Errorf("刷新失败")
-	//	}
-	//}
-	//return err
 }
 
-// EdbInfoRefreshAllFromBaseV3 全部刷新指标（切换到edb_lib服务）
+// EdbInfoRefreshAllFromBaseV3Bak 全部刷新指标（切换到edb_lib服务）
 // @author Roc
 // @datetime 2022-09-16 11:04:44
 // @description 将原有的单个指标刷新，调整为批量多个指标刷新
-func EdbInfoRefreshAllFromBaseV3(edbInfoIdList []int, refreshAll, isSync bool) (err error, isAsync bool) {
+// DeprecatedTime 2023-10-23 09:38:19废弃
+// Deprecated
+func EdbInfoRefreshAllFromBaseV3Bak(edbInfoIdList []int, refreshAll, isSync bool) (err error, isAsync bool) {
 	var errmsg string
 	defer func() {
 		if err != nil {
@@ -245,6 +66,155 @@ func EdbInfoRefreshAllFromBaseV3(edbInfoIdList []int, refreshAll, isSync bool) (
 	return
 }
 
+// EdbInfoRefreshAllFromBaseV3
+//
+//	@Description: 全部刷新指标（切换到edb_lib服务）
+//	@author: Roc
+//	@datetime2023-10-23 09:57:55
+//	@param edbInfoIdList []int
+//	@param refreshAll bool
+//	@param isSync bool
+//	@return err error
+//	@return isAsync bool
+func EdbInfoRefreshAllFromBaseV3(edbInfoIdList []int, refreshAll, isSync bool) (err error, isAsync bool) {
+	var errmsg string
+	defer func() {
+		if err != nil {
+			fmt.Println("EdbInfoRefreshAllFromBaseV3 Err:" + err.Error() + ";errmsg:" + errmsg)
+			go alarm_msg.SendAlarmMsg("EdbInfoRefreshAllFromBaseV3,Err"+err.Error()+";errMsg:"+errmsg, 3)
+		}
+	}()
+	traceEdbInfoList, err := TraceEdbInfoByEdbInfoIdList(edbInfoIdList)
+	if err != nil {
+		return
+	}
+	// existEdbInfoIdMap 已经处理了的指标id map
+	existEdbInfoIdMap := make(map[int]int)
+
+	// 基础指标
+	newBaseEdbInfoArr := make([]*data_manage.EdbInfo, 0)
+	newBasePredictEdbInfoArr := make([]*data_manage.EdbInfo, 0)
+	newBaseMap := make(map[int]*data_manage.EdbInfo)
+	newPredictBaseMap := make(map[int]*data_manage.EdbInfo)
+
+	// 计算指标
+	newCalculateMap := make(map[int]*data_manage.EdbInfo)
+	newPredictCalculateMap := make(map[int]*data_manage.EdbInfo)
+	calculateArr := make([]int, 0)
+	predictCalculateArr := make([]int, 0)
+
+	// 获取关联指标
+	for _, traceEdbInfo := range traceEdbInfoList {
+		tmpBaseEdbInfoArr, tmpBasePredictEdbInfoArr, tmpCalculateMap, tmpPredictCalculateMap, _, _ := getRefreshEdbInfoListByTraceEdbInfo(traceEdbInfo, existEdbInfoIdMap)
+
+		// 普通基础指标
+		for _, edbInfo := range tmpBaseEdbInfoArr {
+			if _, ok := newBaseMap[edbInfo.EdbInfoId]; !ok {
+				newBaseMap[edbInfo.EdbInfoId] = edbInfo
+				newBaseEdbInfoArr = append(newBaseEdbInfoArr, edbInfo)
+			}
+		}
+
+		// 预测基础指标
+		for _, edbInfo := range tmpBasePredictEdbInfoArr {
+			if _, ok := newPredictBaseMap[edbInfo.EdbInfoId]; !ok {
+				newPredictBaseMap[edbInfo.EdbInfoId] = edbInfo
+				newBasePredictEdbInfoArr = append(newBasePredictEdbInfoArr, edbInfo)
+			}
+		}
+
+		// 普通计算指标
+		for _, edbInfo := range tmpCalculateMap {
+			if _, ok := newCalculateMap[edbInfo.EdbInfoId]; !ok {
+				newCalculateMap[edbInfo.EdbInfoId] = edbInfo
+				calculateArr = append(calculateArr, edbInfo.EdbInfoId)
+			}
+		}
+
+		// 预测计算指标
+		for _, edbInfo := range tmpPredictCalculateMap {
+			if _, ok := newPredictCalculateMap[edbInfo.EdbInfoId]; !ok {
+				newPredictCalculateMap[edbInfo.EdbInfoId] = edbInfo
+				predictCalculateArr = append(predictCalculateArr, edbInfo.EdbInfoId)
+			}
+		}
+	}
+
+	// 普通计算指标的id
+	sort.Ints(calculateArr)
+	// 预测计算指标的id
+	sort.Ints(predictCalculateArr)
+
+	// 需要刷新的指标数量
+	totalEdbInfo := len(newBaseEdbInfoArr) + len(calculateArr) + len(predictCalculateArr)
+
+	if totalEdbInfo <= 20 || isSync {
+		err = edbInfoRefreshAll(refreshAll, newBaseEdbInfoArr, newBasePredictEdbInfoArr, newCalculateMap, newPredictCalculateMap, calculateArr, predictCalculateArr)
+	} else {
+		isAsync = true
+		go edbInfoRefreshAll(refreshAll, newBaseEdbInfoArr, newBasePredictEdbInfoArr, newCalculateMap, newPredictCalculateMap, calculateArr, predictCalculateArr)
+	}
+
+	return
+}
+
+// getRefreshEdbInfoListByTraceEdbInfo根据溯源获取关联指标
+func getRefreshEdbInfoListByTraceEdbInfo(traceEdbInfo data_manage.TraceEdbInfoResp, existEdbInfoIdMap map[int]int) (newBaseEdbInfoArr, newBasePredictEdbInfoArr []*data_manage.EdbInfo, newCalculateMap, newPredictCalculateMap map[int]*data_manage.EdbInfo, calculateArr, predictCalculateArr []int) {
+	newBaseEdbInfoArr = make([]*data_manage.EdbInfo, 0)
+	newBasePredictEdbInfoArr = make([]*data_manage.EdbInfo, 0)
+	newCalculateMap = make(map[int]*data_manage.EdbInfo)
+	newPredictCalculateMap = make(map[int]*data_manage.EdbInfo)
+	calculateArr = make([]int, 0)
+	predictCalculateArr = make([]int, 0)
+
+	_, ok := existEdbInfoIdMap[traceEdbInfo.EdbInfoId]
+	if ok {
+		return
+	}
+
+	existEdbInfoIdMap[traceEdbInfo.EdbInfoId] = traceEdbInfo.EdbInfoId
+
+	switch traceEdbInfo.EdbInfoType {
+	//0-普通指标; 1-预测指标
+	case 0: // 0-普通指标
+		if traceEdbInfo.EdbType == 1 { //1-基础指标
+			newBaseEdbInfoArr = append(newBaseEdbInfoArr, traceEdbInfo.EdbInfo)
+		} else if traceEdbInfo.EdbType == 2 { //2-计算指标
+			newCalculateMap[traceEdbInfo.EdbInfoId] = traceEdbInfo.EdbInfo
+			calculateArr = append(calculateArr, traceEdbInfo.EdbInfoId)
+		}
+	case 1: // 1-预测指标
+		if traceEdbInfo.EdbType == 1 { //1-基础指标
+			newBasePredictEdbInfoArr = append(newBasePredictEdbInfoArr, traceEdbInfo.EdbInfo)
+		} else if traceEdbInfo.EdbType == 2 { //2-计算指标
+			newPredictCalculateMap[traceEdbInfo.EdbInfoId] = traceEdbInfo.EdbInfo
+			predictCalculateArr = append(predictCalculateArr, traceEdbInfo.EdbInfoId)
+		}
+	}
+
+	if traceEdbInfo.Child != nil && len(traceEdbInfo.Child) > 0 {
+		for _, v := range traceEdbInfo.Child {
+			tmpBaseEdbInfoArr, tmpPredictEdbInfoArr, tmpCalculateMap, tmpPredictCalculateMap, tmpCalculateArr, tmpPredictCalculateArr := getRefreshEdbInfoListByTraceEdbInfo(v, existEdbInfoIdMap)
+
+			newBaseEdbInfoArr = append(newBaseEdbInfoArr, tmpBaseEdbInfoArr...)
+			newBasePredictEdbInfoArr = append(newBasePredictEdbInfoArr, tmpPredictEdbInfoArr...)
+
+			for k, tmpEdbInfo := range tmpCalculateMap {
+				newCalculateMap[k] = tmpEdbInfo
+			}
+
+			for k, tmpEdbInfo := range tmpPredictCalculateMap {
+				newPredictCalculateMap[k] = tmpEdbInfo
+			}
+
+			calculateArr = append(calculateArr, tmpCalculateArr...)
+			predictCalculateArr = append(predictCalculateArr, tmpPredictCalculateArr...)
+		}
+	}
+
+	return
+}
+
 func edbInfoRefreshAll(refreshAll bool, newBaseEdbInfoArr, newBasePredictEdbInfoArr []*data_manage.EdbInfo, newCalculateMap, newPredictCalculateMap map[int]*data_manage.EdbInfo, calculateArr, predictCalculateArr []int) (err error) {
 	var errmsg string
 	defer func() {
@@ -2690,6 +2660,7 @@ func TraceEdbInfoByEdbInfoId(edbInfoId int) (traceEdbInfo data_manage.TraceEdbIn
 		//Source:      edbInfo.Source,
 		UniqueCode: edbInfo.UniqueCode,
 		ClassifyId: edbInfo.ClassifyId,
+		EdbInfo:    edbInfo,
 	}
 	findIdMap := make(map[int]int)
 	findIdMap[edbInfoId] = edbInfoId
@@ -2712,6 +2683,55 @@ func TraceEdbInfoByEdbInfoId(edbInfoId int) (traceEdbInfo data_manage.TraceEdbIn
 	return
 }
 
+// TraceEdbInfoByEdbInfoIdList 指标追溯
+func TraceEdbInfoByEdbInfoIdList(edbInfoIdList []int) (traceEdbInfoList []data_manage.TraceEdbInfoResp, err error) {
+	traceEdbInfoList = make([]data_manage.TraceEdbInfoResp, 0)
+	edbInfoList, err := data_manage.GetEdbInfoByIdList(edbInfoIdList)
+	if err != nil {
+		return
+	}
+	edbInfoRuleMap := make(map[int]string, 0)
+	edbMappingMap := make(map[int][]*data_manage.EdbInfoCalculateMappingInfo)
+
+	findIdMap := make(map[int]int)
+	existMap := make(map[int]data_manage.TraceEdbInfoResp)
+
+	for _, edbInfo := range edbInfoList {
+		findIdMap[edbInfo.EdbInfoId] = edbInfo.EdbInfoId
+		//edbInfoRuleMap[edbInfoId] = getEdbRuleTitle(edbInfo)
+		traceEdbInfo := data_manage.TraceEdbInfoResp{
+			//EdbInfoId: edbInfo.EdbInfoId,
+			EdbInfoId:   edbInfo.EdbInfoId,
+			EdbInfoType: edbInfo.EdbInfoType,
+			EdbName:     edbInfo.EdbName,
+			EdbType:     edbInfo.EdbType,
+			//Source:      edbInfo.Source,
+			UniqueCode: edbInfo.UniqueCode,
+			ClassifyId: edbInfo.ClassifyId,
+			EdbInfo:    edbInfo,
+		}
+		traceEdbInfo.Child, err = traceEdbInfoByEdbInfoId(edbInfo.EdbInfoId, traceEdbInfo, edbInfoRuleMap, findIdMap, existMap, edbMappingMap)
+		traceEdbInfoList = append(traceEdbInfoList, traceEdbInfo)
+	}
+
+	//findEdbInfoIdList := make([]int, 0)
+	//for _, v := range findIdMap {
+	//	findEdbInfoIdList = append(findEdbInfoIdList, v)
+	//}
+	//findEdbInfoList, err := data_manage.GetEdbInfoByIdList(findEdbInfoIdList)
+	//if err != nil {
+	//	return
+	//}
+	//edbInfoMap := make(map[int]*data_manage.EdbInfo)
+	//for _, tmpEdbInfo := range findEdbInfoList {
+	//	edbInfoMap[tmpEdbInfo.EdbInfoId] = tmpEdbInfo
+	//}
+	//for k, traceEdbInfo := range traceEdbInfoList {
+	//	traceEdbInfoList[k], err = handleTraceEdbInfo(traceEdbInfo, 0, edbInfoMap, edbMappingMap)
+	//}
+	return
+}
+
 // traceEdbInfoByEdbInfoId 指标追溯
 func traceEdbInfoByEdbInfoId(edbInfoId int, traceEdbInfo data_manage.TraceEdbInfoResp, edbInfoRuleMap map[int]string, findIdMap map[int]int, existMap map[int]data_manage.TraceEdbInfoResp, edbMappingMap map[int][]*data_manage.EdbInfoCalculateMappingInfo) (child []data_manage.TraceEdbInfoResp, err error) {
 	traceEdbInfo, ok := existMap[edbInfoId]
@@ -2725,8 +2745,26 @@ func traceEdbInfoByEdbInfoId(edbInfoId int, traceEdbInfo data_manage.TraceEdbInf
 		err = fmt.Errorf("GetEdbInfoCalculateMappingListByEdbInfoId err: %s", e.Error())
 		return
 	}
-	edbMappingMap[edbInfoId] = edbInfoMappingList
 
+	// 指标信息map
+	edbInfoMap := make(map[int]*data_manage.EdbInfo)
+	if len(edbInfoMappingList) > 0 {
+		fromEdbInfoIdList := make([]int, 0)
+		for _, v := range edbInfoMappingList {
+			fromEdbInfoIdList = append(fromEdbInfoIdList, v.FromEdbInfoId)
+		}
+		edbInfoList, tmpErr := data_manage.GetEdbInfoByIdList(fromEdbInfoIdList)
+		if tmpErr != nil {
+			err = fmt.Errorf("traceEdbInfoByEdbInfoId GetEdbInfoByIdList err: %s", tmpErr.Error())
+			return
+		}
+		for _, v := range edbInfoList {
+			edbInfoMap[v.EdbInfoId] = v
+		}
+
+	}
+
+	edbMappingMap[edbInfoId] = edbInfoMappingList
 	for _, v := range edbInfoMappingList {
 		tmpEdbInfoId := v.FromEdbInfoId
 		tmpTraceEdbInfo := data_manage.TraceEdbInfoResp{
@@ -2735,6 +2773,7 @@ func traceEdbInfoByEdbInfoId(edbInfoId int, traceEdbInfo data_manage.TraceEdbInf
 			EdbType:     v.FromEdbType,
 			UniqueCode:  v.FromUniqueCode,
 			ClassifyId:  v.FromClassifyId,
+			EdbInfo:     edbInfoMap[v.FromEdbInfoId],
 		}
 
 		// 计算指标/预测指标继续溯源