package chart import ( "errors" "fmt" chartEdbMappingModel "hongze/hongze_yb/models/tables/chart_edb_mapping" edbInfoModel "hongze/hongze_yb/models/tables/edb_info" predictEdbConfCalculateMappingModel "hongze/hongze_yb/models/tables/predict_edb_conf_calculate_mapping" "hongze/hongze_yb/services/alarm_msg" "hongze/hongze_yb/utils" "sort" "time" ) // EdbInfoRefreshAllFromBase 全部刷新指标(切换到edb_lib服务) // @author Roc // @datetime 2022-09-16 11:04:44 // @description 将原有的单个指标刷新,调整为批量多个指标刷新 func EdbInfoRefreshAllFromBase(edbInfoIdList []int, refreshAll bool) (err error, errmsg string) { defer func() { if err != nil { fmt.Println("EdbInfoRefreshAllFromBaseV2 Err:" + err.Error() + ";errmsg:" + errmsg) go alarm_msg.SendAlarmMsg("EdbInfoRefreshFromBaseV2,Err"+err.Error()+";errMsg:"+errmsg, 3) //go utils.SendEmail(utils.APPNAME+"【"+utils.RunMode+"】"+"失败提醒", "EdbInfoRefreshFromBase:"+errmsg, utils.EmailSendToUsers) } }() // 获取关联的基础指标 newBaseEdbInfoArr, newBasePredictEdbInfoArr, newCalculateMap, newPredictCalculateMap, calculateArr, predictCalculateArr, err, errmsg := getRefreshEdbInfoListByIds(edbInfoIdList) if err != nil { return } var startDate string for _, bv := range newBaseEdbInfoArr { //source := bv.Source //edbInfoId := bv.EdbInfoId //edbCode := bv.EdbCode if bv.StartDate.IsZero() { continue } //开始时间 startDate = `` if refreshAll { //刷新所有数据,用开始时间作为起始日期去刷新 startDate = bv.StartDate.Format(utils.FormatDate) } else { sTime := bv.EndDate frequency := bv.Frequency var limitDay int switch frequency { case "日度": limitDay = utils.DATA_START_REFRESH_LIMIT case "周度": limitDay = utils.DATA_START_REFRESH_LIMIT * 7 case "月度": limitDay = utils.DATA_START_REFRESH_LIMIT * 30 case "季度": limitDay = utils.DATA_START_REFRESH_LIMIT * 90 case "年度": limitDay = utils.DATA_START_REFRESH_LIMIT * 365 default: limitDay = utils.DATA_START_REFRESH_LIMIT } startDate = sTime.AddDate(0, 0, -limitDay).Format(utils.FormatDate) } result, tmpErr := RefreshEdbData(bv.EdbInfoId, bv.Source, bv.SubSource, bv.EdbCode, startDate) if tmpErr != nil { err = tmpErr fmt.Println(bv.EdbInfoId, "RefreshBaseEdbData err", time.Now()) errmsg = "RefreshBaseEdbData Err:" + err.Error() return } if result.Ret != 200 { fmt.Println(bv.EdbInfoId, "RefreshBaseEdbData err;msg:", result.Msg, ";errMsg:", result.ErrMsg) errmsg = fmt.Sprint(bv.EdbInfoId, "RefreshBaseEdbData err;msg:", result.Msg, ";errMsg:", result.ErrMsg) err = fmt.Errorf("刷新失败, err:", errmsg) return } //maxAndMinItem, err := data_manage.GetEdbInfoMaxAndMinInfo(source, edbCode) //if err != nil { // if err.Error() == utils.ErrNoRow() { //找不到数据,那么就进入到下一条数据做处理 // continue // } // return err //} //if maxAndMinItem != nil { // err = data_manage.ModifyEdbInfoMaxAndMinInfo(edbInfoId, maxAndMinItem) // if err != nil { // return err // } //} fmt.Println("end newBaseEdbInfoArr:", bv, time.Now()) } //刷新相关普通计算指标 for _, v := range calculateArr { edbInfo := newCalculateMap[v] if edbInfo == nil { return } startDate = edbInfo.StartDate.Format(utils.FormatDate) source := edbInfo.Source if edbInfo.StartDate.IsZero() { //如果没有开始日期,说明还没有计算出来数据,那么就往前面推40年吧(也意味着重新计算了) startDate = time.Now().AddDate(-40, 0, 0).Format(utils.FormatDate) } else { if source == utils.DATA_SOURCE_CALCULATE { startDate = `` if refreshAll { //刷新所有数据,用开始时间作为起始日期去刷新 startDate = edbInfo.StartDate.Format(utils.FormatDate) } else { sTime := edbInfo.EndDate frequency := edbInfo.Frequency var limitDay int switch frequency { case "日度": limitDay = utils.DATA_START_REFRESH_LIMIT case "周度": limitDay = utils.DATA_START_REFRESH_LIMIT * 7 case "月度": limitDay = utils.DATA_START_REFRESH_LIMIT * 30 case "季度": limitDay = utils.DATA_START_REFRESH_LIMIT * 90 case "年度": limitDay = utils.DATA_START_REFRESH_LIMIT * 365 default: limitDay = utils.DATA_START_REFRESH_LIMIT } startDate = sTime.AddDate(0, 0, -limitDay).Format(utils.FormatDate) } } } result, tmpErr := RefreshEdbCalculateData(edbInfo.EdbInfoId, edbInfo.EdbCode, startDate) if tmpErr != nil { err = tmpErr fmt.Println(v, "RefreshEdbCalculateData err", time.Now()) errmsg = "RefreshEdbCalculateData Err:" + err.Error() return } if result.Ret != 200 { fmt.Println(v, "RefreshEdbCalculateData err;msg:", result.Msg, ";errMsg:", result.ErrMsg) errmsg = fmt.Sprint(v, "RefreshEdbCalculateData err;msg:", result.Msg, ";errMsg:", result.ErrMsg) err = fmt.Errorf("刷新失败") return } } //刷新相关预测计算指标 for _, v := range predictCalculateArr { edbInfo := newPredictCalculateMap[v] if edbInfo == nil { return } startDate = edbInfo.StartDate.Format(utils.FormatDate) source := edbInfo.Source if edbInfo.StartDate.IsZero() { //如果没有开始日期,说明还没有计算出来数据,那么就往前面推40年吧(也意味着重新计算了) startDate = time.Now().AddDate(-40, 0, 0).Format(utils.FormatDate) } else { if source == utils.DATA_SOURCE_PREDICT_CALCULATE { startDate = `` if refreshAll { //刷新所有数据,用开始时间作为起始日期去刷新 startDate = edbInfo.StartDate.Format(utils.FormatDate) } else { sTime := edbInfo.EndDate frequency := edbInfo.Frequency var limitDay int switch frequency { case "日度": limitDay = utils.DATA_START_REFRESH_LIMIT case "周度": limitDay = utils.DATA_START_REFRESH_LIMIT * 7 case "月度": limitDay = utils.DATA_START_REFRESH_LIMIT * 30 case "季度": limitDay = utils.DATA_START_REFRESH_LIMIT * 90 case "年度": limitDay = utils.DATA_START_REFRESH_LIMIT * 365 default: limitDay = utils.DATA_START_REFRESH_LIMIT } startDate = sTime.AddDate(0, 0, -limitDay).Format(utils.FormatDate) } } } result, tmpErr := RefreshPredictEdbCalculateData(edbInfo.EdbInfoId, edbInfo.EdbCode, startDate) if tmpErr != nil { err = tmpErr fmt.Println(v, "RefreshPredictEdbCalculateData err", time.Now()) errmsg = "RefreshPredictEdbCalculateData Err:" + err.Error() return } if result.Ret != 200 { fmt.Println(v, "RefreshPredictEdbCalculateData err;msg:", result.Msg, ";errMsg:", result.ErrMsg) errmsg = fmt.Sprint(v, "RefreshPredictEdbCalculateData err;msg:", result.Msg, ";errMsg:", result.ErrMsg) err = fmt.Errorf("刷新失败") return } } // 4、更新动态环差值 { predictEdbInfoIdList := make([]int, 0) for _, v := range newBasePredictEdbInfoArr { predictEdbInfoIdList = append(predictEdbInfoIdList, v.EdbInfoId) } fmt.Println("predictEdbConfCalculateMappingDetailList:", predictEdbInfoIdList) predictEdbConfCalculateMappingDetailList, tmpErr := predictEdbConfCalculateMappingModel.GetPredictEdbRuleDataList(predictEdbInfoIdList) if tmpErr != nil { err = tmpErr errmsg = fmt.Sprint("更新动态环差值时,获取数据失败:" + err.Error()) return } for _, bv := range predictEdbConfCalculateMappingDetailList { result, tmpErr := RefreshEdbData(bv.EdbInfoId, utils.DATA_SOURCE_PREDICT, utils.DATA_SUB_SOURCE_EDB, bv.EdbCode, "") if tmpErr != nil { err = tmpErr fmt.Println(bv.EdbInfoId, "RefreshBasePredictEdbData err", time.Now()) errmsg = "RefreshBasePredictEdbData Err:" + err.Error() return } if result.Ret != 200 { fmt.Println(bv.EdbInfoId, "RefreshBasePredictEdbData err;msg:", result.Msg, ";errMsg:", result.ErrMsg) errmsg = fmt.Sprint(bv.EdbInfoId, "RefreshBasePredictEdbData err;msg:", result.Msg, ";errMsg:", result.ErrMsg) err = errors.New("刷新失败:" + errmsg) return } fmt.Println("end predictEdbConfCalculateMappingDetailList:", bv, time.Now()) } } //5、刷新相关预测计算指标 for _, v := range predictCalculateArr { edbInfo := newPredictCalculateMap[v] if edbInfo == nil { return } startDate = edbInfo.StartDate.Format(utils.FormatDate) source := edbInfo.Source if edbInfo.StartDate.IsZero() { //如果没有开始日期,说明还没有计算出来数据,那么就往前面推40年吧(也意味着重新计算了) startDate = time.Now().AddDate(-40, 0, 0).Format(utils.FormatDate) } else { if source == utils.DATA_SOURCE_PREDICT_CALCULATE { startDate = `` if refreshAll { //刷新所有数据,用开始时间作为起始日期去刷新 startDate = edbInfo.StartDate.Format(utils.FormatDate) } else { sTime := edbInfo.EndDate frequency := edbInfo.Frequency var limitDay int switch frequency { case "日度": limitDay = utils.DATA_START_REFRESH_LIMIT case "周度": limitDay = utils.DATA_START_REFRESH_LIMIT * 7 case "月度": limitDay = utils.DATA_START_REFRESH_LIMIT * 30 case "季度": limitDay = utils.DATA_START_REFRESH_LIMIT * 90 case "年度": limitDay = utils.DATA_START_REFRESH_LIMIT * 365 default: limitDay = utils.DATA_START_REFRESH_LIMIT } startDate = sTime.AddDate(0, 0, -limitDay).Format(utils.FormatDate) } } } result, tmpErr := RefreshPredictEdbCalculateData(edbInfo.EdbInfoId, edbInfo.EdbCode, startDate) if tmpErr != nil { err = tmpErr fmt.Println(v, "RefreshPredictEdbCalculateData err", time.Now()) errmsg = "RefreshPredictEdbCalculateData Err:" + err.Error() return } if result.Ret != 200 { fmt.Println(v, "RefreshPredictEdbCalculateData err;msg:", result.Msg, ";errMsg:", result.ErrMsg) errmsg = fmt.Sprint(v, "RefreshPredictEdbCalculateData err;msg:", result.Msg, ";errMsg:", result.ErrMsg) err = fmt.Errorf("刷新失败") return } } return } // getRefreshEdbInfoList 获取待更新的指标(普通基础指标、普通运算指标,预测运算指标) func getRefreshEdbInfoListByIds(edbInfoIdList []int) (newBaseEdbInfoArr, newBasePredictEdbInfoArr []*edbInfoModel.EdbInfo, newCalculateMap, newPredictCalculateMap map[int]*edbInfoModel.EdbInfo, calculateArr, predictCalculateArr []int, err error, errMsg string) { calculateList, err := edbInfoModel.GetEdbInfoAllCalculateByEdbInfoIdList(edbInfoIdList) if err != nil && err != utils.ErrNoRow { err = errors.New("GetEdbInfoAllCalculate Err:" + err.Error()) return } // 获取指标信息 edbInfoList, err := edbInfoModel.GetEdbInfoByIdList(edbInfoIdList) if err != nil { err = errors.New("GetEdbInfoAllCalGetEdbInfoByIdr:" + err.Error()) return } baseEdbInfoArr := make([]*edbInfoModel.EdbInfo, 0) // 基础普通指标 calculateInfoArr := make([]*edbInfoModel.EdbInfo, 0) //基础运算指标 basePredictEdbInfoArr := make([]*edbInfoModel.EdbInfo, 0) // 预测普通指标 predictCalculateInfoArr := make([]*edbInfoModel.EdbInfo, 0) //预测运算指标 for _, baseEdbInfo := range edbInfoList { if baseEdbInfo.EdbInfoType == 0 { //普通指标 switch baseEdbInfo.EdbType { case 1: //基础指标 baseEdbInfoArr = append(baseEdbInfoArr, baseEdbInfo) case 2: //计算指标 calculateList = append(calculateList, baseEdbInfo) } } else { //预测指标 switch baseEdbInfo.EdbType { case 1: //基础指标 basePredictEdbInfoArr = append(basePredictEdbInfoArr, baseEdbInfo) case 2: //计算指标 predictCalculateInfoArr = append(predictCalculateInfoArr, baseEdbInfo) } } } // 计算指标 for _, v := range calculateList { getBaseEdbInfoArr, getCalculateInfoArr, tmpErr := edbInfoModel.GetRefreshEdbInfoFromBase(v.EdbInfoId, v.Source) if tmpErr != nil { err = tmpErr fmt.Println("GetRefreshEdbInfoFromBase err:" + err.Error()) errMsg = "获取需要刷新的指标失败:Err:" + err.Error() return } for _, tmpEdbInfo := range getBaseEdbInfoArr { switch tmpEdbInfo.EdbInfoType { case 0: //普通基础指标 baseEdbInfoArr = append(baseEdbInfoArr, tmpEdbInfo) case 1: //预测基础指标 basePredictEdbInfoArr = append(basePredictEdbInfoArr, tmpEdbInfo) } } for _, tmpEdbInfo := range getCalculateInfoArr { switch tmpEdbInfo.EdbInfoType { case 0: //普通运算指标 calculateInfoArr = append(calculateInfoArr, tmpEdbInfo) case 1: //预测运算指标 //predictCalculateInfoArr = append(predictCalculateInfoArr, tmpEdbInfo) if tmpEdbInfo.EdbType == 1 { //预测普通指标 basePredictEdbInfoArr = append(basePredictEdbInfoArr, tmpEdbInfo) } else { predictCalculateInfoArr = append(predictCalculateInfoArr, tmpEdbInfo) } } } switch v.EdbInfoType { case 0: //普通运算指标 calculateInfoArr = append(calculateInfoArr, v) case 1: //预测运算指标 //predictCalculateInfoArr = append(predictCalculateInfoArr, v) if v.EdbType == 1 { //预测普通指标 basePredictEdbInfoArr = append(basePredictEdbInfoArr, v) } else { predictCalculateInfoArr = append(predictCalculateInfoArr, v) } } } // 预测计算指标 for _, v := range predictCalculateInfoArr { getBaseEdbInfoArr, getCalculateInfoArr, tmpErr := edbInfoModel.GetRefreshEdbInfoFromBase(v.EdbInfoId, v.Source) if tmpErr != nil { err = tmpErr fmt.Println("GetRefreshEdbInfoFromBase err:" + err.Error()) errMsg = "获取需要刷新的指标失败:Err:" + err.Error() return } for _, tmpEdbInfo := range getBaseEdbInfoArr { switch tmpEdbInfo.EdbInfoType { case 0: //普通基础指标 baseEdbInfoArr = append(baseEdbInfoArr, tmpEdbInfo) case 1: //预测基础指标 //basePredictEdbInfoArr = append(basePredictEdbInfoArr, tmpEdbInfo) if tmpEdbInfo.EdbType == 1 { //预测普通指标 basePredictEdbInfoArr = append(basePredictEdbInfoArr, tmpEdbInfo) } else { predictCalculateInfoArr = append(predictCalculateInfoArr, tmpEdbInfo) } } } for _, tmpEdbInfo := range getCalculateInfoArr { switch tmpEdbInfo.EdbInfoType { case 0: //普通运算指标 calculateInfoArr = append(calculateInfoArr, tmpEdbInfo) case 1: //预测运算指标 //predictCalculateInfoArr = append(predictCalculateInfoArr, tmpEdbInfo) if tmpEdbInfo.EdbType == 1 { //预测普通指标 basePredictEdbInfoArr = append(basePredictEdbInfoArr, tmpEdbInfo) } else { predictCalculateInfoArr = append(predictCalculateInfoArr, tmpEdbInfo) } } } switch v.EdbInfoType { case 0: //普通运算指标 calculateInfoArr = append(calculateInfoArr, v) case 1: //预测运算指标 //predictCalculateInfoArr = append(predictCalculateInfoArr, v) if v.EdbType == 1 { //预测普通指标 basePredictEdbInfoArr = append(basePredictEdbInfoArr, v) } else { predictCalculateInfoArr = append(predictCalculateInfoArr, v) } } } // 第一次的计算指标map newCalculateMap = make(map[int]*edbInfoModel.EdbInfo) for _, v := range calculateInfoArr { if v.NoUpdate == 1 { continue } if _, ok := newCalculateMap[v.EdbInfoId]; !ok { calculateArr = append(calculateArr, v.EdbInfoId) } newCalculateMap[v.EdbInfoId] = v } // 基础预测指标 if len(basePredictEdbInfoArr) > 0 { basePredictEdbInfoIdList := make([]int, 0) for _, v := range basePredictEdbInfoArr { basePredictEdbInfoIdList = append(basePredictEdbInfoIdList, v.EdbInfoId) } baseEdbInfoList, tmpErr := edbInfoModel.GetPredictEdbInfoAllCalculate(basePredictEdbInfoIdList) if tmpErr != nil { err = tmpErr return } tmpCalculateList := make([]*edbInfoModel.EdbInfo, 0) for _, v := range baseEdbInfoList { switch v.EdbType { case 1: baseEdbInfoArr = append(baseEdbInfoArr, v) case 2: // 如果不在第一次里面,那么在第二次 刷新预测指标的时候 需要加进去 if _, ok := newCalculateMap[v.EdbInfoId]; !ok { tmpCalculateList = append(tmpCalculateList, v) } } } // 这里是查询预测指标用到的 普通计算指标 for _, v := range tmpCalculateList { getBaseEdbInfoArr, getCalculateInfoArr, tmpErr := edbInfoModel.GetRefreshEdbInfoFromBase(v.EdbInfoId, v.Source) if tmpErr != nil { err = tmpErr fmt.Println("GetRefreshEdbInfoFromBase err:" + err.Error()) errMsg = "获取需要刷新的指标失败:Err:" + err.Error() return } // 基础指标 for _, tmpEdbInfo := range getBaseEdbInfoArr { switch tmpEdbInfo.EdbInfoType { case 0: //普通基础指标 baseEdbInfoArr = append(baseEdbInfoArr, tmpEdbInfo) case 1: //预测基础指标 basePredictEdbInfoArr = append(basePredictEdbInfoArr, tmpEdbInfo) } } // 计算指标 for _, tmpEdbInfo := range getCalculateInfoArr { switch tmpEdbInfo.EdbInfoType { case 0: //普通运算指标 calculateInfoArr = append(calculateInfoArr, tmpEdbInfo) case 1: //预测运算指标 //predictCalculateInfoArr = append(predictCalculateInfoArr, tmpEdbInfo) if v.EdbType == 1 { //预测普通指标 basePredictEdbInfoArr = append(basePredictEdbInfoArr, tmpEdbInfo) } else { predictCalculateInfoArr = append(predictCalculateInfoArr, tmpEdbInfo) } } } switch v.EdbInfoType { case 0: //普通运算指标 calculateInfoArr = append(calculateInfoArr, v) case 1: //预测运算指标 //predictCalculateInfoArr = append(predictCalculateInfoArr, v) if v.EdbType == 1 { //预测普通指标 basePredictEdbInfoArr = append(basePredictEdbInfoArr, v) } else { //预测运算指标 predictCalculateInfoArr = append(predictCalculateInfoArr, v) } } } // 第二次计算指标的map for _, v := range calculateInfoArr { if v.NoUpdate == 1 { continue } if _, ok := newCalculateMap[v.EdbInfoId]; !ok { calculateArr = append(calculateArr, v.EdbInfoId) } newCalculateMap[v.EdbInfoId] = v } } // 普通基础指标去重 newBaseEdbInfoArr = make([]*edbInfoModel.EdbInfo, 0) baseMap := make(map[int]int) for _, v := range baseEdbInfoArr { if v.NoUpdate == 1 { continue } if _, ok := baseMap[v.EdbInfoId]; !ok { newBaseEdbInfoArr = append(newBaseEdbInfoArr, v) } baseMap[v.EdbInfoId] = v.EdbInfoId } // 普通计算指标的id sort.Ints(calculateArr) // 普通预测指标去重 newBasePredictEdbInfoArr = make([]*edbInfoModel.EdbInfo, 0) basePredictMap := make(map[int]int) for _, v := range basePredictEdbInfoArr { if _, ok := basePredictMap[v.EdbInfoId]; !ok { newBasePredictEdbInfoArr = append(newBasePredictEdbInfoArr, v) } basePredictMap[v.EdbInfoId] = v.EdbInfoId } // 预测计算指标去重 newPredictCalculateMap = make(map[int]*edbInfoModel.EdbInfo) for _, v := range predictCalculateInfoArr { if _, ok := newPredictCalculateMap[v.EdbInfoId]; !ok { predictCalculateArr = append(predictCalculateArr, v.EdbInfoId) } newPredictCalculateMap[v.EdbInfoId] = v } // 预测计算指标的id sort.Ints(predictCalculateArr) return } // GetEdbSourceByEdbInfoIdList 获取关联指标的来源 func GetEdbSourceByEdbInfoIdList(chartEdbInfoMappingList []*chartEdbMappingModel.ChartEdbInfoMappingList) (sourceNameList, sourceNameEnList []string) { sourceNameList = []string{} sourceNameEnList = []string{} sourceMap := make(map[int]string) for _, v := range chartEdbInfoMappingList { // 指标类型:1:基础指标,2:计算指标 if v.EdbType == 2 || v.EdbInfoCategoryType == 1 { //sourceMap[0] = "弘则研究" baseEdbInfoArr, _, _ := edbInfoModel.GetRefreshEdbInfoFromBase(v.EdbInfoId, v.Source) for _, baseEdbInfo := range baseEdbInfoArr { if baseEdbInfo.EdbInfoType == 0 { //普通指标才参与,预测指标不参与 sourceMap[baseEdbInfo.Source] = baseEdbInfo.SourceName } } } else { sourceMap[v.Source] = v.SourceName } } for source, sourceName := range sourceMap { if utils.InArrayByInt([]int{utils.DATA_SOURCE_MANUAL}, source) { continue } sourceNameList = append(sourceNameList, sourceName) sourceNameEn, ok := utils.DataSourceEnMap[source] if !ok { sourceNameEn = sourceName } sourceNameEnList = append(sourceNameEnList, sourceNameEn) } sourceNameList = append(sourceNameList, `弘则研究`) sourceNameEnList = append(sourceNameEnList, `Horizon Insights`) return }