package data import ( "encoding/json" "errors" "eta/eta_mobile/models/data_manage" "eta/eta_mobile/services/alarm_msg" "eta/eta_mobile/services/elastic" "eta/eta_mobile/utils" "fmt" "sort" "strconv" "strings" "time" ) // EdbInfoRefreshAllFromBaseV2 全部刷新指标（切换到edb_lib服务） func EdbInfoRefreshAllFromBaseV2(edbInfoId int, refreshAll bool) (err error, isAsync bool) { var errmsg string defer func() { if err != nil { fmt.Println("EdbInfoRefreshAllFromBaseV2 Err:" + err.Error() + ";errmsg:" + errmsg) go alarm_msg.SendAlarmMsg("EdbInfoRefreshFromBaseV2,Err"+err.Error()+";errMsg:"+errmsg, 3) //go utils.SendEmail(utils.APPNAME+"【"+utils.RunMode+"】"+"失败提醒", "EdbInfoRefreshFromBase:"+errmsg, utils.EmailSendToUsers) } }() err, isAsync = EdbInfoRefreshAllFromBaseV3([]int{edbInfoId}, refreshAll, false) return //// 获取关联的基础指标 //newBaseEdbInfoArr, newCalculateMap, newPredictCalculateMap, calculateArr, predictCalculateArr, err, errmsg := getRefreshEdbInfoList(edbInfoId) //if err != nil { // return //} // //var startDate string //for _, bv := range newBaseEdbInfoArr { // //source := bv.Source // //edbInfoId := bv.EdbInfoId // //edbCode := bv.EdbCode // if bv.StartDate == "0000-00-00" { // continue // } // //开始时间 // startDate = `` // if refreshAll { //刷新所有数据，用开始时间作为起始日期去刷新 // sTime, err := time.Parse(utils.FormatDate, bv.StartDate) // if err != nil { // return err // } // startDate = sTime.Format(utils.FormatDate) // } else { // sTime, err := time.Parse(utils.FormatDate, bv.EndDate) // if err != nil { // return err // } // frequency := bv.Frequency // var limitDay int // switch frequency { // case "日度": // limitDay = utils.DATA_START_REFRESH_LIMIT // case "周度": // limitDay = utils.DATA_START_REFRESH_LIMIT * 7 // case "月度": // limitDay = utils.DATA_START_REFRESH_LIMIT * 30 // case "季度": // limitDay = utils.DATA_START_REFRESH_LIMIT * 90 // case "年度": // limitDay = utils.DATA_START_REFRESH_LIMIT * 365 // default: // limitDay = utils.DATA_START_REFRESH_LIMIT // } // startDate = sTime.AddDate(0, 0, -limitDay).Format(utils.FormatDate) // } // result, err := RefreshEdbData(bv.EdbInfoId, bv.Source, bv.EdbCode, startDate) // if err != nil { // fmt.Println(bv.EdbInfoId, "RefreshBaseEdbData err", time.Now()) // errmsg = "RefreshBaseEdbData Err:" + err.Error() // return err // } // if result.Ret != 200 { // fmt.Println(bv.EdbInfoId, "RefreshBaseEdbData err;msg:", result.Msg, ";errMsg:", result.ErrMsg) // errmsg = fmt.Sprint(bv.EdbInfoId, "RefreshBaseEdbData err;msg:", result.Msg, ";errMsg:", result.ErrMsg) // return fmt.Errorf("刷新失败, err:", errmsg) // } // // //maxAndMinItem, err := data_manage.GetEdbInfoMaxAndMinInfo(source, edbCode) // //if err != nil { // // if err.Error() == utils.ErrNoRow() { //找不到数据，那么就进入到下一条数据做处理 // // continue // // } // // return err // //} // //if maxAndMinItem != nil { // // err = data_manage.ModifyEdbInfoMaxAndMinInfo(edbInfoId, maxAndMinItem) // // if err != nil { // // return err // // } // //} // fmt.Println("end newBaseEdbInfoArr:", bv, time.Now()) //} // ////刷新相关普通计算指标 //for _, v := range calculateArr { // edbInfo := newCalculateMap[v] // if edbInfo == nil { // return err // } // startDate = edbInfo.StartDate // source := edbInfo.Source // if startDate == "" || startDate == "0000-00-00" { //如果没有开始日期，说明还没有计算出来数据，那么就往前面推40年吧（也意味着重新计算了） // startDate = time.Now().AddDate(-40, 0, 0).Format(utils.FormatDate) // } else { // if source == utils.DATA_SOURCE_CALCULATE { // startDate = `` // if refreshAll { //刷新所有数据，用开始时间作为起始日期去刷新 // startDate = edbInfo.StartDate // } else { // sTime, err := time.Parse(utils.FormatDate, edbInfo.EndDate) // if err != nil { // return err // } // frequency := edbInfo.Frequency // var limitDay int // switch frequency { // case "日度": // limitDay = utils.DATA_START_REFRESH_LIMIT // case "周度": // limitDay = utils.DATA_START_REFRESH_LIMIT * 7 // case "月度": // limitDay = utils.DATA_START_REFRESH_LIMIT * 30 // case "季度": // limitDay = utils.DATA_START_REFRESH_LIMIT * 90 // case "年度": // limitDay = utils.DATA_START_REFRESH_LIMIT * 365 // default: // limitDay = utils.DATA_START_REFRESH_LIMIT // } // startDate = sTime.AddDate(0, 0, -limitDay).Format(utils.FormatDate) // } // } // } // // result, err := RefreshEdbCalculateData(edbInfo.EdbInfoId, edbInfo.EdbCode, startDate) // if err != nil { // fmt.Println(v, "RefreshEdbCalculateData err", time.Now()) // errmsg = "RefreshEdbCalculateData Err:" + err.Error() // return err // } // if result.Ret != 200 { // fmt.Println(v, "RefreshEdbCalculateData err;msg:", result.Msg, ";errMsg:", result.ErrMsg) // errmsg = fmt.Sprint(v, "RefreshEdbCalculateData err;msg:", result.Msg, ";errMsg:", result.ErrMsg) // return fmt.Errorf("刷新失败") // } //} // ////刷新相关预测计算指标 //for _, v := range predictCalculateArr { // edbInfo := newPredictCalculateMap[v] // if edbInfo == nil { // return err // } // startDate = edbInfo.StartDate // source := edbInfo.Source // if startDate == "" || startDate == "0000-00-00" { //如果没有开始日期，说明还没有计算出来数据，那么就往前面推40年吧（也意味着重新计算了） // startDate = time.Now().AddDate(-40, 0, 0).Format(utils.FormatDate) // } else { // if source == utils.DATA_SOURCE_PREDICT_CALCULATE { // startDate = `` // if refreshAll { //刷新所有数据，用开始时间作为起始日期去刷新 // startDate = edbInfo.StartDate // } else { // sTime, err := time.Parse(utils.FormatDate, edbInfo.EndDate) // if err != nil { // return err // } // frequency := edbInfo.Frequency // var limitDay int // switch frequency { // case "日度": // limitDay = utils.DATA_START_REFRESH_LIMIT // case "周度": // limitDay = utils.DATA_START_REFRESH_LIMIT * 7 // case "月度": // limitDay = utils.DATA_START_REFRESH_LIMIT * 30 // case "季度": // limitDay = utils.DATA_START_REFRESH_LIMIT * 90 // case "年度": // limitDay = utils.DATA_START_REFRESH_LIMIT * 365 // default: // limitDay = utils.DATA_START_REFRESH_LIMIT // } // startDate = sTime.AddDate(0, 0, -limitDay).Format(utils.FormatDate) // } // } // } // // result, err := RefreshPredictEdbCalculateData(edbInfo.EdbInfoId, edbInfo.EdbCode, startDate) // if err != nil { // fmt.Println(v, "RefreshEdbCalculateData err", time.Now()) // errmsg = "RefreshEdbCalculateData Err:" + err.Error() // return err // } // if result.Ret != 200 { // fmt.Println(v, "RefreshEdbCalculateData err;msg:", result.Msg, ";errMsg:", result.ErrMsg) // errmsg = fmt.Sprint(v, "RefreshEdbCalculateData err;msg:", result.Msg, ";errMsg:", result.ErrMsg) // return fmt.Errorf("刷新失败") // } //} //return err } // 获取频度的英文版 func GetFrequencyEn(frequency string) (frequencyEn string) { switch frequency { case "日度": frequencyEn = "day" return case "周度": frequencyEn = "week" return case "旬度": frequencyEn = "ten days" return case "月度": frequencyEn = "month" return case "季度": frequencyEn = "quarter" return case "年度": frequencyEn = "year" return } return } func GetLeadUnitEn(unit string) (unitEn string) { switch unit { case "天": unitEn = "day" return case "周": unitEn = "week" return case "月": unitEn = "month" return case "季": unitEn = "quarter" return case "年": unitEn = "year" return } return } // EdbInfoRefreshAllFromBaseV3Bak 全部刷新指标（切换到edb_lib服务） // @author Roc // @datetime 2022-09-16 11:04:44 // @description 将原有的单个指标刷新，调整为批量多个指标刷新 // DeprecatedTime 2023-10-23 09:38:19废弃 // Deprecated func EdbInfoRefreshAllFromBaseV3Bak(edbInfoIdList []int, refreshAll, isSync bool) (err error, isAsync bool) { var errmsg string defer func() { if err != nil { fmt.Println("EdbInfoRefreshAllFromBaseV2 Err:" + err.Error() + ";errmsg:" + errmsg) go alarm_msg.SendAlarmMsg("EdbInfoRefreshFromBaseV2,Err"+err.Error()+";errMsg:"+errmsg, 3) //go utils.SendEmail(utils.APPNAME+"【"+utils.RunMode+"】"+"失败提醒", "EdbInfoRefreshFromBase:"+errmsg, utils.EmailSendToUsers) } }() // 获取关联的基础指标 newBaseEdbInfoArr, newBasePredictEdbInfoArr, newCalculateMap, newPredictCalculateMap, calculateArr, predictCalculateArr, err, errmsg := getRefreshEdbInfoListByIds(edbInfoIdList) if err != nil { return } // 需要刷新的指标数量 totalEdbInfo := len(newBaseEdbInfoArr) + len(calculateArr) + len(predictCalculateArr) if totalEdbInfo <= 20 || isSync { err = edbInfoRefreshAll(refreshAll, newBaseEdbInfoArr, newBasePredictEdbInfoArr, newCalculateMap, newPredictCalculateMap, calculateArr, predictCalculateArr) } else { isAsync = true go edbInfoRefreshAll(refreshAll, newBaseEdbInfoArr, newBasePredictEdbInfoArr, newCalculateMap, newPredictCalculateMap, calculateArr, predictCalculateArr) } return } // EdbInfoRefreshAllFromBaseV3 // // @Description: 全部刷新指标（切换到edb_lib服务） // @author: Roc // @datetime2023-10-23 09:57:55 // @param edbInfoIdList []int // @param refreshAll bool // @param isSync bool 是否同步执行 // @return err error // @return isAsync bool 是否异步刷新 func EdbInfoRefreshAllFromBaseV3(edbInfoIdList []int, refreshAll, isSync bool) (err error, isAsync bool) { var errmsg string defer func() { if err != nil { fmt.Println("EdbInfoRefreshAllFromBaseV3 Err:" + err.Error() + ";errmsg:" + errmsg) go alarm_msg.SendAlarmMsg("EdbInfoRefreshAllFromBaseV3,Err"+err.Error()+";errMsg:"+errmsg, 3) } }() // 获取需要刷新的指标列表 newBaseEdbInfoArr, newBasePredictEdbInfoArr, newCalculateMap, newPredictCalculateMap, calculateArr, predictCalculateArr := getEdbInfoIdList(edbInfoIdList) // 需要刷新的指标数量 totalEdbInfo := len(newBaseEdbInfoArr) + len(calculateArr) + len(predictCalculateArr) if totalEdbInfo <= 20 || isSync { err = edbInfoRefreshAll(refreshAll, newBaseEdbInfoArr, newBasePredictEdbInfoArr, newCalculateMap, newPredictCalculateMap, calculateArr, predictCalculateArr) } else { isAsync = true go edbInfoRefreshAll(refreshAll, newBaseEdbInfoArr, newBasePredictEdbInfoArr, newCalculateMap, newPredictCalculateMap, calculateArr, predictCalculateArr) } return } // getEdbInfoIdList // @Description: 获取待更新的指标列表 // @author: Roc // @datetime 2023-12-01 16:19:42 // @param edbInfoIdList []int // @return newBaseEdbInfoArr []*data_manage.EdbInfo // @return newBasePredictEdbInfoArr []*data_manage.EdbInfo // @return newCalculateMap map[int]*data_manage.EdbInfo // @return newPredictCalculateMap map[int]*data_manage.EdbInfo // @return calculateArr []int // @return predictCalculateArr []int func getEdbInfoIdList(edbInfoIdList []int) (newBaseEdbInfoArr, newBasePredictEdbInfoArr []*data_manage.EdbInfo, newCalculateMap, newPredictCalculateMap map[int]*data_manage.EdbInfo, calculateArr, predictCalculateArr []int) { traceEdbInfoList, err := TraceEdbInfoByEdbInfoIdList(edbInfoIdList) if err != nil { return } // existEdbInfoIdMap 已经处理了的指标id map existEdbInfoIdMap := make(map[int]int) // 基础指标 newBaseEdbInfoArr = make([]*data_manage.EdbInfo, 0) newBasePredictEdbInfoArr = make([]*data_manage.EdbInfo, 0) newBaseMap := make(map[int]*data_manage.EdbInfo) newPredictBaseMap := make(map[int]*data_manage.EdbInfo) // 计算指标 newCalculateMap = make(map[int]*data_manage.EdbInfo) newPredictCalculateMap = make(map[int]*data_manage.EdbInfo) calculateArr = make([]int, 0) predictCalculateArr = make([]int, 0) // 获取关联指标 for _, traceEdbInfo := range traceEdbInfoList { tmpBaseEdbInfoArr, tmpBasePredictEdbInfoArr, tmpCalculateMap, tmpPredictCalculateMap, _, _ := getRefreshEdbInfoListByTraceEdbInfo(traceEdbInfo, existEdbInfoIdMap) // 普通基础指标 for _, edbInfo := range tmpBaseEdbInfoArr { if _, ok := newBaseMap[edbInfo.EdbInfoId]; !ok { newBaseMap[edbInfo.EdbInfoId] = edbInfo newBaseEdbInfoArr = append(newBaseEdbInfoArr, edbInfo) } } // 预测基础指标 for _, edbInfo := range tmpBasePredictEdbInfoArr { if _, ok := newPredictBaseMap[edbInfo.EdbInfoId]; !ok { newPredictBaseMap[edbInfo.EdbInfoId] = edbInfo newBasePredictEdbInfoArr = append(newBasePredictEdbInfoArr, edbInfo) } } // 普通计算指标 for _, edbInfo := range tmpCalculateMap { if _, ok := newCalculateMap[edbInfo.EdbInfoId]; !ok { newCalculateMap[edbInfo.EdbInfoId] = edbInfo calculateArr = append(calculateArr, edbInfo.EdbInfoId) } } // 预测计算指标 for _, edbInfo := range tmpPredictCalculateMap { if _, ok := newPredictCalculateMap[edbInfo.EdbInfoId]; !ok { newPredictCalculateMap[edbInfo.EdbInfoId] = edbInfo predictCalculateArr = append(predictCalculateArr, edbInfo.EdbInfoId) } } } // 普通计算指标的id sort.Ints(calculateArr) // 预测计算指标的id sort.Ints(predictCalculateArr) return } // getRefreshEdbInfoListByTraceEdbInfo // // @Description: 根据溯源获取关联指标 // @author: Roc // @datetime2023-10-23 10:55:01 // @param traceEdbInfo data_manage.TraceEdbInfoResp // @param existEdbInfoIdMap map[int]int // @return newBaseEdbInfoArr []*data_manage.EdbInfo // @return newBasePredictEdbInfoArr []*data_manage.EdbInfo // @return newCalculateMap map[int]*data_manage.EdbInfo // @return newPredictCalculateMap map[int]*data_manage.EdbInfo // @return calculateArr []int // @return predictCalculateArr []int func getRefreshEdbInfoListByTraceEdbInfo(traceEdbInfo data_manage.TraceEdbInfoResp, existEdbInfoIdMap map[int]int) (newBaseEdbInfoArr, newBasePredictEdbInfoArr []*data_manage.EdbInfo, newCalculateMap, newPredictCalculateMap map[int]*data_manage.EdbInfo, calculateArr, predictCalculateArr []int) { newBaseEdbInfoArr = make([]*data_manage.EdbInfo, 0) newBasePredictEdbInfoArr = make([]*data_manage.EdbInfo, 0) newCalculateMap = make(map[int]*data_manage.EdbInfo) newPredictCalculateMap = make(map[int]*data_manage.EdbInfo) calculateArr = make([]int, 0) predictCalculateArr = make([]int, 0) _, ok := existEdbInfoIdMap[traceEdbInfo.EdbInfoId] if ok { return } existEdbInfoIdMap[traceEdbInfo.EdbInfoId] = traceEdbInfo.EdbInfoId switch traceEdbInfo.EdbInfoType { //0-普通指标; 1-预测指标 case 0: // 0-普通指标 if traceEdbInfo.EdbType == 1 { //1-基础指标 newBaseEdbInfoArr = append(newBaseEdbInfoArr, traceEdbInfo.EdbInfo) } else if traceEdbInfo.EdbType == 2 { //2-计算指标 newCalculateMap[traceEdbInfo.EdbInfoId] = traceEdbInfo.EdbInfo calculateArr = append(calculateArr, traceEdbInfo.EdbInfoId) } case 1: // 1-预测指标 if traceEdbInfo.EdbType == 1 { //1-基础指标 newBasePredictEdbInfoArr = append(newBasePredictEdbInfoArr, traceEdbInfo.EdbInfo) } else if traceEdbInfo.EdbType == 2 { //2-计算指标 newPredictCalculateMap[traceEdbInfo.EdbInfoId] = traceEdbInfo.EdbInfo predictCalculateArr = append(predictCalculateArr, traceEdbInfo.EdbInfoId) } } if traceEdbInfo.Child != nil && len(traceEdbInfo.Child) > 0 { for _, v := range traceEdbInfo.Child { tmpBaseEdbInfoArr, tmpPredictEdbInfoArr, tmpCalculateMap, tmpPredictCalculateMap, tmpCalculateArr, tmpPredictCalculateArr := getRefreshEdbInfoListByTraceEdbInfo(v, existEdbInfoIdMap) newBaseEdbInfoArr = append(newBaseEdbInfoArr, tmpBaseEdbInfoArr...) newBasePredictEdbInfoArr = append(newBasePredictEdbInfoArr, tmpPredictEdbInfoArr...) for k, tmpEdbInfo := range tmpCalculateMap { newCalculateMap[k] = tmpEdbInfo } for k, tmpEdbInfo := range tmpPredictCalculateMap { newPredictCalculateMap[k] = tmpEdbInfo } calculateArr = append(calculateArr, tmpCalculateArr...) predictCalculateArr = append(predictCalculateArr, tmpPredictCalculateArr...) } } return } func edbInfoRefreshAll(refreshAll bool, newBaseEdbInfoArr, newBasePredictEdbInfoArr []*data_manage.EdbInfo, newCalculateMap, newPredictCalculateMap map[int]*data_manage.EdbInfo, calculateArr, predictCalculateArr []int) (err error) { var errmsg string defer func() { if err != nil { fmt.Println("EdbInfoRefreshAllFromBaseV2 Err:" + err.Error() + ";errmsg:" + errmsg) go alarm_msg.SendAlarmMsg("EdbInfoRefreshFromBaseV2,Err"+err.Error()+";errMsg:"+errmsg, 3) //go utils.SendEmail(utils.APPNAME+"【"+utils.RunMode+"】"+"失败提醒", "EdbInfoRefreshFromBase:"+errmsg, utils.EmailSendToUsers) } }() var startDate string for _, bv := range newBaseEdbInfoArr { //source := bv.Source //edbInfoId := bv.EdbInfoId //edbCode := bv.EdbCode if bv.StartDate == "0000-00-00" { continue } //开始时间 startDate = `` if refreshAll { //刷新所有数据，用开始时间作为起始日期去刷新 sTime, err := time.Parse(utils.FormatDate, bv.StartDate) if err != nil { return err } startDate = sTime.Format(utils.FormatDate) } else { sTime, err := time.Parse(utils.FormatDate, bv.EndDate) if err != nil { return err } frequency := bv.Frequency var limitDay int switch frequency { case "日度": limitDay = utils.DATA_START_REFRESH_LIMIT case "周度": limitDay = utils.DATA_START_REFRESH_LIMIT * 7 case "月度": limitDay = utils.DATA_START_REFRESH_LIMIT * 30 case "季度": limitDay = utils.DATA_START_REFRESH_LIMIT * 90 case "年度": limitDay = utils.DATA_START_REFRESH_LIMIT * 365 default: limitDay = utils.DATA_START_REFRESH_LIMIT } startDate = sTime.AddDate(0, 0, -limitDay).Format(utils.FormatDate) } result, err := RefreshEdbData(bv.EdbInfoId, bv.Source, bv.EdbCode, startDate) if err != nil { fmt.Println(bv.EdbInfoId, "RefreshBaseEdbData err", time.Now()) errmsg = "RefreshBaseEdbData Err:" + err.Error() return err } if result.Ret != 200 { fmt.Println(bv.EdbInfoId, "RefreshBaseEdbData err;msg:", result.Msg, ";errMsg:", result.ErrMsg) errmsg = fmt.Sprint(bv.EdbInfoId, "RefreshBaseEdbData err;msg:", result.Msg, ";errMsg:", result.ErrMsg) return fmt.Errorf("刷新失败, err:", errmsg) } //maxAndMinItem, err := data_manage.GetEdbInfoMaxAndMinInfo(source, edbCode) //if err != nil { // if err.Error() == utils.ErrNoRow() { //找不到数据，那么就进入到下一条数据做处理 // continue // } // return err //} //if maxAndMinItem != nil { // err = data_manage.ModifyEdbInfoMaxAndMinInfo(edbInfoId, maxAndMinItem) // if err != nil { // return err // } //} fmt.Println("end newBaseEdbInfoArr:", bv, time.Now()) } //刷新相关普通计算指标 for _, v := range calculateArr { edbInfo := newCalculateMap[v] if edbInfo == nil { return err } //计算指标就不用开始时间了吧，因为存在说开始时间变更的情况 startDate = `` if !refreshAll { //如果不是全部更新，那么以指标的开始时间作为刷新 startDate = edbInfo.StartDate source := edbInfo.Source if startDate == "" || startDate == "0000-00-00" { //如果没有开始日期，说明还没有计算出来数据，那么就往前面推40年吧（也意味着重新计算了） startDate = time.Now().AddDate(-40, 0, 0).Format(utils.FormatDate) } else { if source == utils.DATA_SOURCE_CALCULATE { sTime, err := time.Parse(utils.FormatDate, edbInfo.EndDate) if err != nil { return err } frequency := edbInfo.Frequency var limitDay int switch frequency { case "日度": limitDay = utils.DATA_START_REFRESH_LIMIT case "周度": limitDay = utils.DATA_START_REFRESH_LIMIT * 7 case "月度": limitDay = utils.DATA_START_REFRESH_LIMIT * 30 case "季度": limitDay = utils.DATA_START_REFRESH_LIMIT * 90 case "年度": limitDay = utils.DATA_START_REFRESH_LIMIT * 365 default: limitDay = utils.DATA_START_REFRESH_LIMIT } startDate = sTime.AddDate(0, 0, -limitDay).Format(utils.FormatDate) } } } result, err := RefreshEdbCalculateData(edbInfo.EdbInfoId, edbInfo.EdbCode, startDate) if err != nil { fmt.Println(v, "RefreshEdbCalculateData err", time.Now()) errmsg = "RefreshEdbCalculateData Err:" + err.Error() return err } if result.Ret != 200 { fmt.Println(v, "RefreshEdbCalculateData err;msg:", result.Msg, ";errMsg:", result.ErrMsg) errmsg = fmt.Sprint(v, "RefreshEdbCalculateData err;msg:", result.Msg, ";errMsg:", result.ErrMsg) return fmt.Errorf("刷新失败") } } //刷新相关预测计算指标 for _, v := range predictCalculateArr { edbInfo := newPredictCalculateMap[v] if edbInfo == nil { return err } startDate = edbInfo.StartDate source := edbInfo.Source if startDate == "" || startDate == "0000-00-00" { //如果没有开始日期，说明还没有计算出来数据，那么就往前面推40年吧（也意味着重新计算了） startDate = time.Now().AddDate(-40, 0, 0).Format(utils.FormatDate) } else { if source == utils.DATA_SOURCE_PREDICT_CALCULATE { startDate = `` if refreshAll { //刷新所有数据，用开始时间作为起始日期去刷新 startDate = edbInfo.StartDate } else { sTime, err := time.Parse(utils.FormatDate, edbInfo.EndDate) if err != nil { return err } frequency := edbInfo.Frequency var limitDay int switch frequency { case "日度": limitDay = utils.DATA_START_REFRESH_LIMIT case "周度": limitDay = utils.DATA_START_REFRESH_LIMIT * 7 case "月度": limitDay = utils.DATA_START_REFRESH_LIMIT * 30 case "季度": limitDay = utils.DATA_START_REFRESH_LIMIT * 90 case "年度": limitDay = utils.DATA_START_REFRESH_LIMIT * 365 default: limitDay = utils.DATA_START_REFRESH_LIMIT } startDate = sTime.AddDate(0, 0, -limitDay).Format(utils.FormatDate) } } } result, err := RefreshPredictEdbCalculateData(edbInfo.EdbInfoId, edbInfo.EdbCode, startDate) if err != nil { fmt.Println(v, "RefreshEdbCalculateData err", time.Now()) errmsg = "RefreshEdbCalculateData Err:" + err.Error() return err } if result.Ret != 200 { fmt.Println(v, "RefreshPredictEdbCalculateData err;msg:", result.Msg, ";errMsg:", result.ErrMsg) errmsg = fmt.Sprint(v, "RefreshPredictEdbCalculateData err;msg:", result.Msg, ";errMsg:", result.ErrMsg) return fmt.Errorf("刷新失败") } } // 4、更新动态环差值 { predictEdbInfoIdList := make([]int, 0) for _, v := range newBasePredictEdbInfoArr { predictEdbInfoIdList = append(predictEdbInfoIdList, v.EdbInfoId) } fmt.Println("predictEdbConfCalculateMappingDetailList:", predictEdbInfoIdList) predictEdbConfCalculateMappingDetailList, err := data_manage.GetPredictEdbConfCalculateMappingDetailListByEdbInfoId(predictEdbInfoIdList) if err != nil { errmsg = fmt.Sprint("更新动态环差值时，获取数据失败:" + err.Error()) return fmt.Errorf("刷新失败, err:", errmsg) } for _, bv := range predictEdbConfCalculateMappingDetailList { result, err := RefreshEdbData(bv.EdbInfoId, utils.DATA_SOURCE_PREDICT, bv.EdbCode, "") if err != nil { fmt.Println(bv.EdbInfoId, "RefreshBasePredictEdbData err", time.Now()) errmsg = "RefreshBasePredictEdbData Err:" + err.Error() return err } if result.Ret != 200 { fmt.Println(bv.EdbInfoId, "RefreshBasePredictEdbData err;msg:", result.Msg, ";errMsg:", result.ErrMsg) errmsg = fmt.Sprint(bv.EdbInfoId, "RefreshBasePredictEdbData err;msg:", result.Msg, ";errMsg:", result.ErrMsg) return fmt.Errorf("刷新失败, err:%s", errmsg) } fmt.Println("end predictEdbConfCalculateMappingDetailList:", bv, time.Now()) } } //5、再次刷新相关预测计算指标 for _, v := range predictCalculateArr { edbInfo := newPredictCalculateMap[v] if edbInfo == nil { return err } startDate = edbInfo.StartDate source := edbInfo.Source if startDate == "" || startDate == "0000-00-00" { //如果没有开始日期，说明还没有计算出来数据，那么就往前面推40年吧（也意味着重新计算了） startDate = time.Now().AddDate(-40, 0, 0).Format(utils.FormatDate) } else { if source == utils.DATA_SOURCE_PREDICT_CALCULATE { startDate = `` if refreshAll { //刷新所有数据，用开始时间作为起始日期去刷新 startDate = edbInfo.StartDate } else { sTime, err := time.Parse(utils.FormatDate, edbInfo.EndDate) if err != nil { return err } frequency := edbInfo.Frequency var limitDay int switch frequency { case "日度": limitDay = utils.DATA_START_REFRESH_LIMIT case "周度": limitDay = utils.DATA_START_REFRESH_LIMIT * 7 case "月度": limitDay = utils.DATA_START_REFRESH_LIMIT * 30 case "季度": limitDay = utils.DATA_START_REFRESH_LIMIT * 90 case "年度": limitDay = utils.DATA_START_REFRESH_LIMIT * 365 default: limitDay = utils.DATA_START_REFRESH_LIMIT } startDate = sTime.AddDate(0, 0, -limitDay).Format(utils.FormatDate) } } } result, err := RefreshPredictEdbCalculateData(edbInfo.EdbInfoId, edbInfo.EdbCode, startDate) if err != nil { fmt.Println(v, "RefreshEdbCalculateData err", time.Now()) errmsg = "RefreshEdbCalculateData Err:" + err.Error() return err } if result.Ret != 200 { fmt.Println(v, "RefreshPredictEdbCalculateData err;msg:", result.Msg, ";errMsg:", result.ErrMsg) errmsg = fmt.Sprint(v, "RefreshPredictEdbCalculateData err;msg:", result.Msg, ";errMsg:", result.ErrMsg) return fmt.Errorf("刷新失败") } } return err } // getRefreshEdbInfoList 获取待更新的指标（普通基础指标、普通运算指标，预测运算指标） func getRefreshEdbInfoListByIds(edbInfoIdList []int) (newBaseEdbInfoArr, newBasePredictEdbInfoArr []*data_manage.EdbInfo, newCalculateMap, newPredictCalculateMap map[int]*data_manage.EdbInfo, calculateArr, predictCalculateArr []int, err error, errMsg string) { calculateList, err := data_manage.GetEdbInfoAllCalculateByEdbInfoIdList(edbInfoIdList) if err != nil && err.Error() != utils.ErrNoRow() { err = errors.New("GetEdbInfoAllCalculate Err:" + err.Error()) return } // 获取指标信息 edbInfoList, err := data_manage.GetEdbInfoByIdList(edbInfoIdList) if err != nil { err = errors.New("GetEdbInfoAllCalGetEdbInfoByIdr:" + err.Error()) return } baseEdbInfoArr := make([]*data_manage.EdbInfo, 0) // 基础普通指标 calculateInfoArr := make([]*data_manage.EdbInfo, 0) //基础运算指标 basePredictEdbInfoArr := make([]*data_manage.EdbInfo, 0) // 预测普通指标 predictCalculateInfoArr := make([]*data_manage.EdbInfo, 0) //预测运算指标 for _, baseEdbInfo := range edbInfoList { if baseEdbInfo.EdbInfoType == 0 { //普通指标 switch baseEdbInfo.EdbType { case 1: //基础指标 baseEdbInfoArr = append(baseEdbInfoArr, baseEdbInfo) case 2: //计算指标 calculateList = append(calculateList, baseEdbInfo) } } else { //预测指标 switch baseEdbInfo.EdbType { case 1: //基础指标 basePredictEdbInfoArr = append(basePredictEdbInfoArr, baseEdbInfo) case 2: //计算指标 predictCalculateInfoArr = append(predictCalculateInfoArr, baseEdbInfo) } } } // 计算指标 for _, v := range calculateList { getBaseEdbInfoArr, getCalculateInfoArr, tmpErr := data_manage.GetRefreshEdbInfoFromBase(v.EdbInfoId, v.Source) if tmpErr != nil { err = tmpErr fmt.Println("GetRefreshEdbInfoFromBase err:" + err.Error()) errMsg = "获取需要刷新的指标失败:Err:" + err.Error() return } for _, tmpEdbInfo := range getBaseEdbInfoArr { switch tmpEdbInfo.EdbInfoType { case 0: //普通基础指标 baseEdbInfoArr = append(baseEdbInfoArr, tmpEdbInfo) case 1: //预测基础指标 basePredictEdbInfoArr = append(basePredictEdbInfoArr, tmpEdbInfo) } } for _, tmpEdbInfo := range getCalculateInfoArr { switch tmpEdbInfo.EdbInfoType { case 0: //普通运算指标 calculateInfoArr = append(calculateInfoArr, tmpEdbInfo) case 1: //预测运算指标 //predictCalculateInfoArr = append(predictCalculateInfoArr, tmpEdbInfo) if tmpEdbInfo.EdbType == 1 { //预测普通指标 basePredictEdbInfoArr = append(basePredictEdbInfoArr, tmpEdbInfo) } else { predictCalculateInfoArr = append(predictCalculateInfoArr, tmpEdbInfo) } } } switch v.EdbInfoType { case 0: //普通运算指标 calculateInfoArr = append(calculateInfoArr, v) case 1: //预测指标 if v.EdbType == 1 { //预测普通指标 basePredictEdbInfoArr = append(basePredictEdbInfoArr, v) } else { predictCalculateInfoArr = append(predictCalculateInfoArr, v) } } } // 预测计算指标 for _, v := range predictCalculateInfoArr { getBaseEdbInfoArr, getCalculateInfoArr, tmpErr := data_manage.GetRefreshEdbInfoFromBase(v.EdbInfoId, v.Source) if tmpErr != nil { err = tmpErr fmt.Println("GetRefreshEdbInfoFromBase err:" + err.Error()) errMsg = "获取需要刷新的指标失败:Err:" + err.Error() return } for _, tmpEdbInfo := range getBaseEdbInfoArr { switch tmpEdbInfo.EdbInfoType { case 0: //普通基础指标 baseEdbInfoArr = append(baseEdbInfoArr, tmpEdbInfo) case 1: //预测基础指标 if tmpEdbInfo.EdbType == 1 { //预测普通指标 basePredictEdbInfoArr = append(basePredictEdbInfoArr, tmpEdbInfo) } else { predictCalculateInfoArr = append(predictCalculateInfoArr, tmpEdbInfo) } //basePredictEdbInfoArr = append(basePredictEdbInfoArr, tmpEdbInfo) } } for _, tmpEdbInfo := range getCalculateInfoArr { switch tmpEdbInfo.EdbInfoType { case 0: //普通运算指标 calculateInfoArr = append(calculateInfoArr, tmpEdbInfo) case 1: //预测运算指标 if tmpEdbInfo.EdbType == 1 { //预测普通指标 basePredictEdbInfoArr = append(basePredictEdbInfoArr, tmpEdbInfo) } else { predictCalculateInfoArr = append(predictCalculateInfoArr, tmpEdbInfo) } //predictCalculateInfoArr = append(predictCalculateInfoArr, tmpEdbInfo) } } switch v.EdbInfoType { case 0: //普通运算指标 calculateInfoArr = append(calculateInfoArr, v) case 1: //预测指标 if v.EdbType == 1 { //预测普通指标 basePredictEdbInfoArr = append(basePredictEdbInfoArr, v) } else { predictCalculateInfoArr = append(predictCalculateInfoArr, v) } } } // 第一次的计算指标map newCalculateMap = make(map[int]*data_manage.EdbInfo) for _, v := range calculateInfoArr { if _, ok := newCalculateMap[v.EdbInfoId]; !ok { calculateArr = append(calculateArr, v.EdbInfoId) } newCalculateMap[v.EdbInfoId] = v } // 基础预测指标 if len(basePredictEdbInfoArr) > 0 { basePredictEdbInfoIdList := make([]int, 0) for _, v := range basePredictEdbInfoArr { basePredictEdbInfoIdList = append(basePredictEdbInfoIdList, v.EdbInfoId) } baseEdbInfoList, tmpErr := data_manage.GetPredictEdbInfoAllCalculate(basePredictEdbInfoIdList) if tmpErr != nil { err = tmpErr return } tmpCalculateList := make([]*data_manage.EdbInfo, 0) for _, v := range baseEdbInfoList { switch v.EdbType { case 1: baseEdbInfoArr = append(baseEdbInfoArr, v) case 2: // 如果不在第一次里面，那么在第二次 刷新预测指标的时候 需要加进去 if _, ok := newCalculateMap[v.EdbInfoId]; !ok { tmpCalculateList = append(tmpCalculateList, v) } } } // 这里是查询预测指标用到的 普通计算指标 for _, v := range tmpCalculateList { getBaseEdbInfoArr, getCalculateInfoArr, tmpErr := data_manage.GetRefreshEdbInfoFromBase(v.EdbInfoId, v.Source) if tmpErr != nil { err = tmpErr fmt.Println("GetRefreshEdbInfoFromBase err:" + err.Error()) errMsg = "获取需要刷新的指标失败:Err:" + err.Error() return } // 基础指标 for _, tmpEdbInfo := range getBaseEdbInfoArr { switch tmpEdbInfo.EdbInfoType { case 0: //普通基础指标 baseEdbInfoArr = append(baseEdbInfoArr, tmpEdbInfo) case 1: //预测基础指标 basePredictEdbInfoArr = append(basePredictEdbInfoArr, tmpEdbInfo) } } // 计算指标 for _, tmpEdbInfo := range getCalculateInfoArr { switch tmpEdbInfo.EdbInfoType { case 0: //普通运算指标 calculateInfoArr = append(calculateInfoArr, tmpEdbInfo) case 1: //预测运算指标 if v.EdbType == 1 { //预测普通指标 basePredictEdbInfoArr = append(basePredictEdbInfoArr, tmpEdbInfo) } else { predictCalculateInfoArr = append(predictCalculateInfoArr, tmpEdbInfo) } } } switch v.EdbInfoType { case 0: //普通运算指标 calculateInfoArr = append(calculateInfoArr, v) case 1: //预测指标 if v.EdbType == 1 { //预测普通指标 basePredictEdbInfoArr = append(basePredictEdbInfoArr, v) } else { //预测运算指标 predictCalculateInfoArr = append(predictCalculateInfoArr, v) } } } // 第二次计算指标的map for _, v := range calculateInfoArr { if _, ok := newCalculateMap[v.EdbInfoId]; !ok { calculateArr = append(calculateArr, v.EdbInfoId) } newCalculateMap[v.EdbInfoId] = v } } // 普通基础指标去重 newBaseEdbInfoArr = make([]*data_manage.EdbInfo, 0) baseMap := make(map[int]int) for _, v := range baseEdbInfoArr { if _, ok := baseMap[v.EdbInfoId]; !ok { newBaseEdbInfoArr = append(newBaseEdbInfoArr, v) } baseMap[v.EdbInfoId] = v.EdbInfoId } // 普通计算指标的id sort.Ints(calculateArr) // 普通预测指标去重 newBasePredictEdbInfoArr = make([]*data_manage.EdbInfo, 0) basePredictMap := make(map[int]int) for _, v := range basePredictEdbInfoArr { if _, ok := basePredictMap[v.EdbInfoId]; !ok { newBasePredictEdbInfoArr = append(newBasePredictEdbInfoArr, v) } basePredictMap[v.EdbInfoId] = v.EdbInfoId } // 预测计算指标去重 newPredictCalculateMap = make(map[int]*data_manage.EdbInfo) for _, v := range predictCalculateInfoArr { if _, ok := newPredictCalculateMap[v.EdbInfoId]; !ok { predictCalculateArr = append(predictCalculateArr, v.EdbInfoId) } newPredictCalculateMap[v.EdbInfoId] = v } // 预测计算指标的id sort.Ints(predictCalculateArr) return } // AddOrEditEdbInfoToEs 添加/修改ES中的指标 func AddOrEditEdbInfoToEs(edbInfoId int) { //添加es itemInfo, _ := data_manage.GetEdbInfoByCondition("AND edb_info_id=?", []interface{}{edbInfoId}) go elastic.EsAddOrEditEdbInfoData(utils.DATA_INDEX_NAME, strconv.Itoa(itemInfo.EdbInfoId), itemInfo) } // DeleteEdbInfoToEs 删除ES中的指标 func DeleteEdbInfoToEs(edbInfoId int) { //添加es go elastic.EsDeleteEdbInfoData(utils.DATA_INDEX_NAME, strconv.Itoa(edbInfoId)) } // EdbInfoAdd 添加指标到指标库 func EdbInfoAdd(source, subSource, classifyId int, edbCode, edbName, frequency, unit, startDate, endDate string, sysUserId int, sysUserRealName string) (edbInfo *data_manage.EdbInfo, err error, errMsg string, isSendEmail bool) { isSendEmail = true //判断指标名称是否存在 var condition string var pars []interface{} //condition += " AND source=? " //pars = append(pars, source) condition += " AND edb_name=? " pars = append(pars, edbName) count, err := data_manage.GetEdbInfoCountByCondition(condition, pars) if err != nil { errMsg = "判断指标名称是否存在失败" err = errors.New("判断指标名称是否存在失败,Err:" + err.Error()) return } if count > 0 { errMsg = "指标名称已存在，请重新填写" err = errors.New(errMsg) isSendEmail = false return } edbInfo = new(data_manage.EdbInfo) edbInfo.Source = source sourceNameMap := map[int]string{ utils.DATA_SOURCE_THS: "同花顺", utils.DATA_SOURCE_WIND: "wind", utils.DATA_SOURCE_PB: "彭博", utils.DATA_SOURCE_PB_FINANCE: "彭博财务", utils.DATA_SOURCE_MANUAL: "手工数据", utils.DATA_SOURCE_LZ: "隆众", utils.DATA_SOURCE_YS: "SMM", utils.DATA_SOURCE_GL: "钢联", utils.DATA_SOURCE_ZZ: "郑商所", utils.DATA_SOURCE_DL: "大商所", utils.DATA_SOURCE_SH: "上期所", utils.DATA_SOURCE_CFFEX: "中金所", utils.DATA_SOURCE_SHFE: "上期能源", utils.DATA_SOURCE_GIE: "欧洲天然气", utils.DATA_SOURCE_LT: "路透", utils.DATA_SOURCE_COAL: "中国煤炭网", utils.DATA_SOURCE_GOOGLE_TRAVEL: "our world in data", utils.DATA_SOURCE_MYSTEEL_CHEMICAL: "钢联化工", utils.DATA_SOURCE_EIA_STEO: "EIA STEO报告", utils.DATA_SOURCE_COM_TRADE: "UN", utils.DATA_SOURCE_SCI: "SCI", utils.DATA_SOURCE_BAIINFO: "BAIINFO", utils.DATA_SOURCE_STOCK_PLANT: "存量装置", utils.DATA_SOURCE_NATIONAL_STATISTICS: "国家统计局", utils.DATA_SOURCE_FUBAO: "富宝数据", } sourceName, ok := sourceNameMap[source] if !ok { edbSource := data_manage.EdbSourceIdMap[source] if edbSource != nil { sourceName = edbSource.SourceName } if sourceName == "" { errMsg = "指标来源异常" err = errors.New(errMsg) return } } edbInfo.SourceName = sourceName edbType := 1 //基础指标 if source == utils.DATA_SOURCE_STOCK_PLANT { edbType = 2 //计算指标 } //从缓存中获取 serverUrl := `` if edbInfo.Source == utils.DATA_SOURCE_WIND { windCacheKey := utils.CACHE_WIND_URL + ":" + edbCode serverUrl, _ = utils.Rc.RedisString(windCacheKey) if serverUrl == `` { if len(utils.Hz_Data_WIND_Url_List) >= 1 { serverUrl = utils.Hz_Data_WIND_Url_List[len(utils.Hz_Data_WIND_Url_List)-1] //默认是最后一个服务器地址 } } } //获取该层级下最大的排序数 maxSort, err := GetEdbClassifyMaxSort(classifyId, 0) if err != nil { errMsg = "获取失败" err = errors.New("查询排序信息失败，Err:" + err.Error()) return } edbInfo.EdbCode = edbCode edbInfo.EdbName = edbName edbInfo.EdbNameSource = edbName edbInfo.Frequency = frequency edbInfo.Unit = unit edbInfo.StartDate = startDate edbInfo.EndDate = endDate edbInfo.ClassifyId = classifyId edbInfo.SysUserId = sysUserId edbInfo.SysUserRealName = sysUserRealName edbInfo.CreateTime = time.Now() edbInfo.ModifyTime = time.Now() edbInfo.ServerUrl = serverUrl edbInfo.Sort = maxSort + 1 edbInfo.DataDateType = `交易日` timestamp := strconv.FormatInt(time.Now().UnixNano(), 10) edbInfo.UniqueCode = utils.MD5(utils.DATA_PREFIX + "_" + timestamp) itemVal, err := data_manage.GetEdbInfoMaxAndMinInfo(source, subSource, edbCode) if itemVal != nil && err == nil { edbInfo.MaxValue = itemVal.MaxValue edbInfo.MinValue = itemVal.MinValue } edbInfo.EdbType = edbType edbInfoId, err := data_manage.AddEdbInfo(edbInfo) if err != nil { errMsg = "保存失败" err = errors.New("保存失败，Err:" + err.Error()) return } edbInfo.EdbInfoId = int(edbInfoId) //保存数据 data_manage.ModifyEdbInfoDataStatus(edbInfoId, source, subSource, edbCode) //处理同名指标 //{ // edbNameList, tmpErr := data_manage.GetEdbInfoByName(edbName) // if tmpErr != nil { // errMsg = "保存失败" // err = errors.New("获取指标信息失败，Err:" + tmpErr.Error()) // return // } // if len(edbNameList) >= 2 { // for _, v := range edbNameList { // edbName := v.EdbName + "(" + v.SourceName + ")" // err = data_manage.ModifyEdbInfoNameSource(edbName, v.EdbInfoId) // if err != nil { // errMsg = "保存失败" // err = errors.New("修改指标名称失败，Err:" + err.Error()) // return // } // } // } //} //上传到ES //indexName := utils.DATA_INDEX_NAME //docId := strconv.Itoa(int(edbInfoId)) // //go data.EsAddOrEditEdbInfo(indexName, docId, edbInfo) maxAndMinItem, _ := data_manage.GetEdbInfoMaxAndMinInfo(source, subSource, edbCode) if maxAndMinItem != nil { err = data_manage.ModifyEdbInfoMaxAndMinInfo(int(edbInfoId), maxAndMinItem) } //添加es AddOrEditEdbInfoToEs(int(edbInfoId)) return } // TraceEdbInfoByEdbInfoIdList 指标追溯 func TraceEdbInfoByEdbInfoIdList(edbInfoIdList []int) (traceEdbInfoList []data_manage.TraceEdbInfoResp, err error) { traceEdbInfoList = make([]data_manage.TraceEdbInfoResp, 0) edbInfoList, err := data_manage.GetEdbInfoByIdList(edbInfoIdList) if err != nil { return } edbInfoRuleMap := make(map[int]string, 0) edbMappingMap := make(map[int][]*data_manage.EdbInfoCalculateMappingInfo) findIdMap := make(map[int]int) existMap := make(map[int]data_manage.TraceEdbInfoResp) for _, edbInfo := range edbInfoList { findIdMap[edbInfo.EdbInfoId] = edbInfo.EdbInfoId //edbInfoRuleMap[edbInfoId] = getEdbRuleTitle(edbInfo) traceEdbInfo := data_manage.TraceEdbInfoResp{ //EdbInfoId: edbInfo.EdbInfoId, EdbInfoId: edbInfo.EdbInfoId, EdbInfoType: edbInfo.EdbInfoType, EdbName: edbInfo.EdbName, EdbType: edbInfo.EdbType, //Source: edbInfo.Source, UniqueCode: edbInfo.UniqueCode, ClassifyId: edbInfo.ClassifyId, EdbInfo: edbInfo, } traceEdbInfo.Child, err = traceEdbInfoByEdbInfoId(edbInfo.EdbInfoId, traceEdbInfo, edbInfoRuleMap, findIdMap, existMap, edbMappingMap) traceEdbInfoList = append(traceEdbInfoList, traceEdbInfo) } //findEdbInfoIdList := make([]int, 0) //for _, v := range findIdMap { // findEdbInfoIdList = append(findEdbInfoIdList, v) //} //findEdbInfoList, err := data_manage.GetEdbInfoByIdList(findEdbInfoIdList) //if err != nil { // return //} //edbInfoMap := make(map[int]*data_manage.EdbInfo) //for _, tmpEdbInfo := range findEdbInfoList { // edbInfoMap[tmpEdbInfo.EdbInfoId] = tmpEdbInfo //} //for k, traceEdbInfo := range traceEdbInfoList { // traceEdbInfoList[k], err = handleTraceEdbInfo(traceEdbInfo, 0, edbInfoMap, edbMappingMap) //} return } // TraceEdbInfoByEdbInfoId 指标追溯 func TraceEdbInfoByEdbInfoId(edbInfoId int) (traceEdbInfo data_manage.TraceEdbInfoResp, err error) { edbInfo, err := data_manage.GetEdbInfoById(edbInfoId) if err != nil { return } edbInfoRuleMap := make(map[int]string, 0) edbMappingMap := make(map[int][]*data_manage.EdbInfoCalculateMappingInfo) //edbInfoRuleMap[edbInfoId] = getEdbRuleTitle(edbInfo) traceEdbInfo = data_manage.TraceEdbInfoResp{ //EdbInfoId: edbInfo.EdbInfoId, EdbInfoId: edbInfoId, EdbName: edbInfo.EdbName, EdbInfo: edbInfo, } findIdMap := make(map[int]int) findIdMap[edbInfoId] = edbInfoId existMap := make(map[int]data_manage.TraceEdbInfoResp) traceEdbInfo.Child, err = traceEdbInfoByEdbInfoId(edbInfoId, traceEdbInfo, edbInfoRuleMap, findIdMap, existMap, edbMappingMap) edbInfoIdList := make([]int, 0) for _, v := range findIdMap { edbInfoIdList = append(edbInfoIdList, v) } edbInfoList, err := data_manage.GetEdbInfoByIdList(edbInfoIdList) if err != nil { return } edbInfoMap := make(map[int]*data_manage.EdbInfo) for _, tmpEdbInfo := range edbInfoList { edbInfoMap[tmpEdbInfo.EdbInfoId] = tmpEdbInfo } traceEdbInfo, err = handleTraceEdbInfo(traceEdbInfo, 0, edbInfoMap, edbMappingMap) return } // traceEdbInfoByEdbInfoId 指标追溯 func traceEdbInfoByEdbInfoId(edbInfoId int, traceEdbInfo data_manage.TraceEdbInfoResp, edbInfoRuleMap map[int]string, findIdMap map[int]int, existMap map[int]data_manage.TraceEdbInfoResp, edbMappingMap map[int][]*data_manage.EdbInfoCalculateMappingInfo) (child []data_manage.TraceEdbInfoResp, err error) { traceEdbInfo, ok := existMap[edbInfoId] if ok { return } child = make([]data_manage.TraceEdbInfoResp, 0) edbInfoMappingList, err := data_manage.GetEdbInfoCalculateMappingListByEdbInfoId(edbInfoId) if err != nil { err = fmt.Errorf("GetEdbInfoCalculateMappingListByEdbInfoId err: %s", err.Error()) return } // 指标信息map edbInfoMap := make(map[int]*data_manage.EdbInfo) if len(edbInfoMappingList) > 0 { fromEdbInfoIdList := make([]int, 0) for _, v := range edbInfoMappingList { fromEdbInfoIdList = append(fromEdbInfoIdList, v.FromEdbInfoId) } edbInfoList, tmpErr := data_manage.GetEdbInfoByIdList(fromEdbInfoIdList) if tmpErr != nil { err = fmt.Errorf("traceEdbInfoByEdbInfoId GetEdbInfoByIdList err: %s", tmpErr.Error()) return } for _, v := range edbInfoList { edbInfoMap[v.EdbInfoId] = v } } edbMappingMap[edbInfoId] = edbInfoMappingList for _, v := range edbInfoMappingList { tmpEdbInfoId := v.FromEdbInfoId tmpTraceEdbInfo := data_manage.TraceEdbInfoResp{ EdbInfoId: tmpEdbInfoId, EdbInfo: edbInfoMap[v.FromEdbInfoId], //EdbName: v.EdbName, //RuleTitle: edbInfoRuleMap[v.EdbInfoId], //Child: make([]data_manage.TraceEdbInfoResp, 0), } if v.EdbType == 2 { // 计算指标 // 查过了就不查了 if _, ok2 := findIdMap[tmpEdbInfoId]; !ok2 { tmpTraceEdbInfo.Child, err = traceEdbInfoByEdbInfoId(tmpEdbInfoId, tmpTraceEdbInfo, edbInfoRuleMap, findIdMap, existMap, edbMappingMap) } //else { // tmpTraceEdbInfo = existMap[v.EdbInfoId] //} } child = append(child, tmpTraceEdbInfo) findIdMap[tmpEdbInfoId] = tmpEdbInfoId //existMap[v.EdbInfoId] = tmpTraceEdbInfo } existMap[edbInfoId] = traceEdbInfo return } func handleTraceEdbInfo(traceEdbInfoResp data_manage.TraceEdbInfoResp, parentEdbInfoId int, edbInfoMap map[int]*data_manage.EdbInfo, edbMappingMap map[int][]*data_manage.EdbInfoCalculateMappingInfo) (newTraceEdbInfoResp data_manage.TraceEdbInfoResp, err error) { edbInfo, ok := edbInfoMap[traceEdbInfoResp.EdbInfoId] if !ok { err = errors.New("指标异常") return } var parentEdbInfo *data_manage.EdbInfo if parentEdbInfoId > 0 { parentEdbInfo, ok = edbInfoMap[parentEdbInfoId] if !ok { err = errors.New("指标异常") return } } //traceEdbInfoResp.EdbName = edbInfo.EdbName traceEdbInfoResp.EdbName, traceEdbInfoResp.RuleTitle = getEdbRuleTitle(edbInfo, parentEdbInfo, traceEdbInfoResp.Child, edbInfoMap, edbMappingMap) if traceEdbInfoResp.Child != nil && len(traceEdbInfoResp.Child) > 0 { for k, v := range traceEdbInfoResp.Child { traceEdbInfoResp.Child[k], err = handleTraceEdbInfo(v, traceEdbInfoResp.EdbInfoId, edbInfoMap, edbMappingMap) if err != nil { return } } } newTraceEdbInfoResp = traceEdbInfoResp return } // getEdbRule 获取规则名称 func getEdbRuleTitle(edbInfo, parentEdbInfo *data_manage.EdbInfo, childList []data_manage.TraceEdbInfoResp, edbInfoMap map[int]*data_manage.EdbInfo, edbMappingMap map[int][]*data_manage.EdbInfoCalculateMappingInfo) (edbName, ruleTitle string) { edbName = edbInfo.EdbName ruleTitle = `来源于` + edbInfo.SourceName if edbInfo.EdbType == 1 { // 基础指标的话，直接返回来源名称 //ruleTitle = `来源于`+edbInfo.SourceName return } if parentEdbInfo != nil { edbMappingList, ok := edbMappingMap[parentEdbInfo.EdbInfoId] if !ok { edbMappingList = []*data_manage.EdbInfoCalculateMappingInfo{} } // 指标名称 switch parentEdbInfo.Source { case utils.DATA_SOURCE_CALCULATE, utils.DATA_SOURCE_PREDICT_CALCULATE: for _, v := range edbMappingList { if v.FromEdbInfoId == edbInfo.EdbInfoId { edbName = fmt.Sprintf("%s（%s）", edbInfo.EdbName, v.FromTag) } } case utils.DATA_SOURCE_CALCULATE_ZJPJ, utils.DATA_SOURCE_PREDICT_CALCULATE_ZJPJ, utils.DATA_SOURCE_CALCULATE_LJZTBPJ, utils.DATA_SOURCE_PREDICT_CALCULATE_LJZTBPJ: // 直接拼接 ,累计值同比拼接 for _, v := range edbMappingList { if v.FromEdbInfoId == edbInfo.EdbInfoId { tmpName := `` if v.FromTag == `A` { tmpName = `拼接日期前` } else if v.FromTag == `B` { tmpName = `拼接日期后` } edbName = fmt.Sprintf("%s（%s）", edbInfo.EdbName, tmpName) } } case utils.DATA_SOURCE_CALCULATE_NHCC, utils.DATA_SOURCE_PREDICT_CALCULATE_NHCC: //计算指标（拟合残差） for _, v := range edbMappingList { //（需对上游指标+自变量，领先10天/因变量） if v.FromEdbInfoId == edbInfo.EdbInfoId { tmpName := `` if v.FromTag == `A` { tmpName = fmt.Sprintf(`自变量，领先%d天`, v.MoveValue) } else if v.FromTag == `B` { tmpName = `因变量` } edbName = fmt.Sprintf("%s（%s）", edbInfo.EdbName, tmpName) } } case utils.DATA_SOURCE_CALCULATE_CORRELATION: // 滚动相关性 for _, v := range edbMappingList { if v.FromEdbInfoId == edbInfo.EdbInfoId { edbName = fmt.Sprintf("%s（%s）", edbInfo.EdbName, v.FromTag) } } } } // 规则 switch edbInfo.Source { case utils.DATA_SOURCE_CALCULATE, utils.DATA_SOURCE_PREDICT_CALCULATE: ruleTitle = "=" + edbInfo.CalculateFormula case utils.DATA_SOURCE_CALCULATE_LJZZY, utils.DATA_SOURCE_PREDICT_CALCULATE_LJZZY: ruleTitle = `累计转月值计算` case utils.DATA_SOURCE_CALCULATE_TBZ, utils.DATA_SOURCE_PREDICT_CALCULATE_TBZ: ruleTitle = `同比值计算` case utils.DATA_SOURCE_CALCULATE_TCZ, utils.DATA_SOURCE_PREDICT_CALCULATE_TCZ: ruleTitle = `同差值计算` case utils.DATA_SOURCE_CALCULATE_NSZYDPJJS, utils.DATA_SOURCE_PREDICT_CALCULATE_NSZYDPJJS: ruleTitle = fmt.Sprintf("N数值移动均值计算（N=%s）", edbInfo.CalculateFormula) case utils.DATA_SOURCE_CALCULATE_HBZ, utils.DATA_SOURCE_PREDICT_CALCULATE_HBZ: ruleTitle = fmt.Sprintf("N数值环比值计算（N=%s）", edbInfo.CalculateFormula) case utils.DATA_SOURCE_CALCULATE_HCZ, utils.DATA_SOURCE_PREDICT_CALCULATE_HCZ: ruleTitle = fmt.Sprintf("N数值环差值计算（N=%s）", edbInfo.CalculateFormula) case utils.DATA_SOURCE_CALCULATE_TIME_SHIFT, utils.DATA_SOURCE_PREDICT_CALCULATE_TIME_SHIFT: moveType := `领先` if edbInfo.MoveType == 2 { moveType = "滞后" } ruleTitle = fmt.Sprintf("时间移位计算（%s%s%s）", moveType, edbInfo.CalculateFormula, edbInfo.MoveFrequency) case utils.DATA_SOURCE_CALCULATE_BP, utils.DATA_SOURCE_PREDICT_CALCULATE_BP: // 变频 childFrequency := `` if len(childList) > 0 { if childEdbInfo, ok := edbInfoMap[childList[0].EdbInfoId]; ok { childFrequency = childEdbInfo.Frequency } } ruleTitle = fmt.Sprintf("升频计算（%s转%s）", childFrequency, edbInfo.Frequency) case utils.DATA_SOURCE_CALCULATE_ZJPJ, utils.DATA_SOURCE_PREDICT_CALCULATE_ZJPJ: // 直接拼接 ruleTitle = fmt.Sprintf("直接拼接计算（%s）", edbInfo.CalculateFormula) case utils.DATA_SOURCE_CALCULATE_LJZTBPJ, utils.DATA_SOURCE_PREDICT_CALCULATE_LJZTBPJ: // 累计值同比拼 ruleTitle = fmt.Sprintf("累计值同比值拼接计算（%s）", edbInfo.CalculateFormula) case utils.DATA_SOURCE_PYTHON: ruleTitle = `代码运算` case utils.DATA_SOURCE_CALCULATE_CJJX, utils.DATA_SOURCE_PREDICT_CALCULATE_CJJX: ruleTitle = fmt.Sprintf("超季节性计算（N=%s，%s）", edbInfo.CalculateFormula, edbInfo.Calendar) case utils.DATA_SOURCE_CALCULATE_NHCC, utils.DATA_SOURCE_PREDICT_CALCULATE_NHCC: //计算指标（拟合残差） var startDate, endDate string dateList := strings.Split(edbInfo.CalculateFormula, ",") if len(dateList) == 2 { startDate = dateList[0] endDate = dateList[1] } ruleTitle = fmt.Sprintf("拟合残差计算（%s至%s）", startDate, endDate) case utils.DATA_SOURCE_CALCULATE_ADJUST: ruleTitle = `数据调整` case utils.DATA_SOURCE_CALCULATE_NH, utils.DATA_SOURCE_PREDICT_CALCULATE_NH: ruleTitle = `年化计算` case utils.DATA_SOURCE_CALCULATE_KSZS, utils.DATA_SOURCE_PREDICT_CALCULATE_KSZS: // 扩散指数->53 type KszsConfig struct { DateType int `description:"扩散指标日期；1：全部指标日期并集;2:部分指标日期并集"` CheckList []string `description:"选中的数据,A,B,C"` } var config KszsConfig err := json.Unmarshal([]byte(edbInfo.CalculateFormula), &config) if err != nil { return } var startDate, endDate time.Time childEdbInfoIdList := make([]int, 0) if config.DateType == 1 { for _, v := range childList { childEdbInfoIdList = append(childEdbInfoIdList, v.EdbInfoId) } } else { edbMappingList, ok := edbMappingMap[parentEdbInfo.EdbInfoId] if !ok { edbMappingList = []*data_manage.EdbInfoCalculateMappingInfo{} } tagMap := make(map[string]int) for _, v := range edbMappingList { tagMap[v.FromTag] = v.FromEdbInfoId } for _, v := range config.CheckList { if tmpEdbInfoId, ok := tagMap[v]; ok { childEdbInfoIdList = append(childEdbInfoIdList, tmpEdbInfoId) } } } for _, v := range childEdbInfoIdList { if childEdbInfo, ok := edbInfoMap[v]; ok { tmpStartDate, tmpErr := time.ParseInLocation(utils.FormatDate, childEdbInfo.StartDate, time.Local) if tmpErr != nil { return } if startDate.IsZero() || startDate.After(tmpStartDate) { startDate = tmpStartDate } tmpEndDate, tmpErr := time.ParseInLocation(utils.FormatDate, childEdbInfo.EndDate, time.Local) if tmpErr != nil { return } if endDate.IsZero() || endDate.Before(tmpEndDate) { endDate = tmpEndDate } } } ruleTitle = fmt.Sprintf("扩散指数计算（%s至%s）", startDate.Format(utils.FormatDate), endDate.Format(utils.FormatDate)) case utils.DATA_SOURCE_STOCK_PLANT: ruleTitle = `来源于装置分析` case utils.DATA_SOURCE_CALCULATE_CORRELATION: type EdbCalculateFormula struct { BaseCalculateValue int `description:"基础计算窗口"` BaseCalculateUnit string `description:"基础计算频度"` LeadValue int `description:"领先期数"` LeadUnit string `description:"频度"` CalculateValue int `description:"计算窗口"` CalculateUnit string `description:"计算频度"` } var correlationConf EdbCalculateFormula err := json.Unmarshal([]byte(edbInfo.CalculateFormula), &correlationConf) if err != nil { return } ruleTitle = fmt.Sprintf("滚动相关性（计算窗口%d%s，B领先A%d%s）", correlationConf.CalculateValue, correlationConf.CalculateUnit, correlationConf.LeadValue, correlationConf.LeadUnit) case utils.DATA_SOURCE_CALCULATE_JP, utils.DATA_SOURCE_PREDICT_CALCULATE_JP: childFrequency := `` if len(childList) > 0 { if childEdbInfo, ok := edbInfoMap[childList[0].EdbInfoId]; ok { childFrequency = childEdbInfo.Frequency } } ruleTitle = fmt.Sprintf("降频计算（%s转%s，%s）", childFrequency, edbInfo.Frequency, edbInfo.CalculateFormula) } return } // EdbInfoAdd 添加指标到指标库 func EdbInfoWsdAdd(item *data_manage.EdbInfo) (edbInfo *data_manage.EdbInfo, err error, errMsg string, isSendEmail bool) { isSendEmail = true //判断指标名称是否存在 var condition string var pars []interface{} condition += " AND edb_name=? " pars = append(pars, item.EdbName) count, err := data_manage.GetEdbInfoCountByCondition(condition, pars) if err != nil { errMsg = "判断指标名称是否存在失败" err = errors.New("判断指标名称是否存在失败,Err:" + err.Error()) return } if count > 0 { errMsg = "指标名称已存在，请重新填写" err = errors.New(errMsg) isSendEmail = false return } source := item.Source edbInfo = new(data_manage.EdbInfo) edbInfo.Source = source sourceNameMap := map[int]string{ utils.DATA_SOURCE_THS: "同花顺", utils.DATA_SOURCE_WIND: "wind", utils.DATA_SOURCE_PB: "彭博", utils.DATA_SOURCE_PB_FINANCE: "彭博财务", utils.DATA_SOURCE_MANUAL: "手工数据", utils.DATA_SOURCE_LZ: "隆众", utils.DATA_SOURCE_YS: "SMM", utils.DATA_SOURCE_GL: "钢联", utils.DATA_SOURCE_ZZ: "郑商所", utils.DATA_SOURCE_DL: "大商所", utils.DATA_SOURCE_SH: "上期所", utils.DATA_SOURCE_CFFEX: "中金所", utils.DATA_SOURCE_SHFE: "上期能源", utils.DATA_SOURCE_GIE: "欧洲天然气", utils.DATA_SOURCE_LT: "路透", utils.DATA_SOURCE_COAL: "中国煤炭网", utils.DATA_SOURCE_GOOGLE_TRAVEL: "our world in data", utils.DATA_SOURCE_MYSTEEL_CHEMICAL: "钢联化工", utils.DATA_SOURCE_EIA_STEO: "EIA STEO报告", utils.DATA_SOURCE_COM_TRADE: "UN", utils.DATA_SOURCE_SCI: "SCI", utils.DATA_SOURCE_BAIINFO: "BAIINFO", utils.DATA_SOURCE_STOCK_PLANT: "存量装置", utils.DATA_SOURCE_NATIONAL_STATISTICS: "国家统计局", utils.DATA_SOURCE_FUBAO: "富宝数据", } sourceName, ok := sourceNameMap[source] if !ok { edbSource := data_manage.EdbSourceIdMap[source] if edbSource != nil { sourceName = edbSource.SourceName } if sourceName == "" { errMsg = "指标来源异常" err = errors.New(errMsg) return } } edbInfo.SourceName = sourceName edbType := 1 //基础指标 if source == utils.DATA_SOURCE_STOCK_PLANT { edbType = 2 //计算指标 } //从缓存中获取 serverUrl := `` if edbInfo.Source == utils.DATA_SOURCE_WIND { windCacheKey := utils.CACHE_WIND_URL + ":" + item.EdbCode serverUrl, _ = utils.Rc.RedisString(windCacheKey) if serverUrl == `` { if len(utils.Hz_Data_WIND_Url_List) >= 1 { serverUrl = utils.Hz_Data_WIND_Url_List[len(utils.Hz_Data_WIND_Url_List)-1] //默认是最后一个服务器地址 } } } //获取该层级下最大的排序数 maxSort, err := GetEdbClassifyMaxSort(item.ClassifyId, 0) if err != nil { errMsg = "获取失败" err = errors.New("查询排序信息失败，Err:" + err.Error()) return } edbInfo.EdbCode = item.EdbCode edbInfo.EdbName = item.EdbName edbInfo.EdbNameSource = item.EdbName edbInfo.Frequency = item.Frequency edbInfo.Unit = item.Unit edbInfo.ClassifyId = item.ClassifyId edbInfo.SysUserId = item.SysUserId edbInfo.SysUserRealName = item.SysUserRealName edbInfo.CreateTime = time.Now() edbInfo.ModifyTime = time.Now() edbInfo.ServerUrl = serverUrl edbInfo.Sort = maxSort + 1 edbInfo.DataDateType = `交易日` timestamp := strconv.FormatInt(time.Now().UnixNano(), 10) edbInfo.UniqueCode = utils.MD5(utils.DATA_PREFIX + "_" + timestamp) edbInfo.EdbType = edbType edbInfo.SubSource = item.SubSource edbInfo.SubSourceName = "日期序列" edbInfo.IndicatorCode = item.IndicatorCode edbInfo.StockCode = item.StockCode edbInfoId, err := data_manage.AddEdbInfo(edbInfo) if err != nil { errMsg = "保存失败" err = errors.New("保存失败，Err:" + err.Error()) return } edbInfo.EdbInfoId = int(edbInfoId) //保存数据 err = data_manage.ModifyEdbInfoWindWsdDataStatus(source, item.SubSource, edbInfoId, item.EdbCode) if err != nil { errMsg = "保存失败" err = errors.New("修改数据对应指标ID失败，Err:" + err.Error()) return } maxAndMinItem, _ := data_manage.GetEdbInfoWsdMaxAndMinInfo(source, item.SubSource, item.EdbCode) if maxAndMinItem != nil { err = data_manage.ModifyEdbInfoMaxAndMinInfo(int(edbInfoId), maxAndMinItem) } //添加es AddOrEditEdbInfoToEs(int(edbInfoId)) return }