package data import ( "encoding/json" "errors" "eta/eta_api/cache" "eta/eta_api/models" "eta/eta_api/models/data_manage" "eta/eta_api/models/data_manage/excel" "eta/eta_api/services/alarm_msg" "eta/eta_api/services/data/data_manage_permission" "eta/eta_api/services/elastic" "eta/eta_api/utils" "fmt" "sort" "strconv" "strings" "time" ) // EdbInfoRefreshAllFromBaseV2 全部刷新指标（切换到edb_lib服务） func EdbInfoRefreshAllFromBaseV2(edbInfoId int, refreshAll, isRefreshTop bool) (err error, isAsync bool) { var errmsg string defer func() { if err != nil { fmt.Println("EdbInfoRefreshAllFromBaseV2 Err:" + err.Error() + ";errmsg:" + errmsg) go alarm_msg.SendAlarmMsg("EdbInfoRefreshFromBaseV2,Err"+err.Error()+";errMsg:"+errmsg, 3) //go utils.SendEmail(utils.APPNAME+"【"+utils.RunMode+"】"+"失败提醒", "EdbInfoRefreshFromBase:"+errmsg, utils.EmailSendToUsers) } }() err, isAsync = EdbInfoRefreshAllFromBaseV3([]int{edbInfoId}, refreshAll, false, isRefreshTop) return } // EdbInfoRefreshAllFromBaseV3Bak 全部刷新指标（切换到edb_lib服务） // @author Roc // @datetime 2022-09-16 11:04:44 // @description 将原有的单个指标刷新，调整为批量多个指标刷新 // DeprecatedTime 2023-10-23 09:38:19废弃 // Deprecated func EdbInfoRefreshAllFromBaseV3Bak(edbInfoIdList []int, refreshAll, isSync bool) (err error, isAsync bool) { var errmsg string defer func() { if err != nil { fmt.Println("EdbInfoRefreshAllFromBaseV2 Err:" + err.Error() + ";errmsg:" + errmsg) go alarm_msg.SendAlarmMsg("EdbInfoRefreshFromBaseV2,Err"+err.Error()+";errMsg:"+errmsg, 3) //go utils.SendEmail(utils.APPNAME+"【"+utils.RunMode+"】"+"失败提醒", "EdbInfoRefreshFromBase:"+errmsg, utils.EmailSendToUsers) } }() // 获取关联的基础指标 newBaseEdbInfoArr, newBasePredictEdbInfoArr, newCalculateMap, newPredictCalculateMap, calculateArr, predictCalculateArr, err, errmsg := getRefreshEdbInfoListByIds(edbInfoIdList) if err != nil { return } // 需要刷新的指标数量 totalEdbInfo := len(newBaseEdbInfoArr) + len(calculateArr) + len(predictCalculateArr) if totalEdbInfo <= 20 || isSync { err = edbInfoRefreshAll(refreshAll, newBaseEdbInfoArr, newBasePredictEdbInfoArr, newCalculateMap, newPredictCalculateMap, calculateArr, predictCalculateArr) } else { isAsync = true go edbInfoRefreshAll(refreshAll, newBaseEdbInfoArr, newBasePredictEdbInfoArr, newCalculateMap, newPredictCalculateMap, calculateArr, predictCalculateArr) } return } // EdbInfoRefreshAllFromBaseV3 // // @Description: 全部刷新指标（切换到edb_lib服务） // @author: Roc // @datetime 2023-10-23 09:57:55 // @param edbInfoIdList []int // @param refreshAll bool // @param isSync bool 是否同步执行 // @param isRefreshTop bool 是否刷新上面的指标 // @return err error // @return isAsync bool 是否异步刷新 func EdbInfoRefreshAllFromBaseV3(edbInfoIdList []int, refreshAll, isSync, isRefreshTop bool) (err error, isAsync bool) { var errmsg string defer func() { if err != nil { fmt.Println("EdbInfoRefreshAllFromBaseV3 Err:" + err.Error() + ";errmsg:" + errmsg) go alarm_msg.SendAlarmMsg("EdbInfoRefreshAllFromBaseV3,Err"+err.Error()+";errMsg:"+errmsg, 3) } }() var newBaseEdbInfoArr, newBasePredictEdbInfoArr []*data_manage.EdbInfo var newCalculateMap, newPredictCalculateMap map[int]*data_manage.EdbInfo var calculateArr, predictCalculateArr []int // 获取关联指标 if isRefreshTop { // 获取所有关联的指标，上下所有的指标 newBaseEdbInfoArr, newBasePredictEdbInfoArr, newCalculateMap, newPredictCalculateMap, calculateArr, predictCalculateArr, err, errmsg = getRefreshEdbInfoListByIds(edbInfoIdList) if err != nil { return } } else { // 获取溯源的指标 newBaseEdbInfoArr, newBasePredictEdbInfoArr, newCalculateMap, newPredictCalculateMap, calculateArr, predictCalculateArr = getEdbInfoIdList(edbInfoIdList) } // 需要刷新的指标数量 totalEdbInfo := len(newBaseEdbInfoArr) + len(calculateArr) + len(predictCalculateArr) + len(newBasePredictEdbInfoArr) if totalEdbInfo == 0 { return } if totalEdbInfo <= 20 || isSync { err = edbInfoRefreshAll(refreshAll, newBaseEdbInfoArr, newBasePredictEdbInfoArr, newCalculateMap, newPredictCalculateMap, calculateArr, predictCalculateArr) } else { isAsync = true go edbInfoRefreshAll(refreshAll, newBaseEdbInfoArr, newBasePredictEdbInfoArr, newCalculateMap, newPredictCalculateMap, calculateArr, predictCalculateArr) } return } // getEdbInfoIdList // @Description: 获取待更新的指标列表 // @author: Roc // @datetime 2023-12-01 16:19:42 // @param edbInfoIdList []int // @return newBaseEdbInfoArr []*data_manage.EdbInfo // @return newBasePredictEdbInfoArr []*data_manage.EdbInfo // @return newCalculateMap map[int]*data_manage.EdbInfo // @return newPredictCalculateMap map[int]*data_manage.EdbInfo // @return calculateArr []int // @return predictCalculateArr []int func getEdbInfoIdList(edbInfoIdList []int) (newBaseEdbInfoArr, newBasePredictEdbInfoArr []*data_manage.EdbInfo, newCalculateMap, newPredictCalculateMap map[int]*data_manage.EdbInfo, calculateArr, predictCalculateArr []int) { traceEdbInfoList, err := TraceEdbInfoByEdbInfoIdList(edbInfoIdList) if err != nil { return } // existEdbInfoIdMap 已经处理了的指标id map existEdbInfoIdMap := make(map[int]int) // 基础指标 newBaseEdbInfoArr = make([]*data_manage.EdbInfo, 0) newBasePredictEdbInfoArr = make([]*data_manage.EdbInfo, 0) newBaseMap := make(map[int]*data_manage.EdbInfo) newPredictBaseMap := make(map[int]*data_manage.EdbInfo) // 计算指标 newCalculateMap = make(map[int]*data_manage.EdbInfo) newPredictCalculateMap = make(map[int]*data_manage.EdbInfo) calculateArr = make([]int, 0) predictCalculateArr = make([]int, 0) // 获取关联指标 for _, traceEdbInfo := range traceEdbInfoList { tmpBaseEdbInfoArr, tmpBasePredictEdbInfoArr, tmpCalculateMap, tmpPredictCalculateMap, _, _ := getRefreshEdbInfoListByTraceEdbInfo(traceEdbInfo, existEdbInfoIdMap) // 普通基础指标 for _, edbInfo := range tmpBaseEdbInfoArr { if _, ok := newBaseMap[edbInfo.EdbInfoId]; !ok { newBaseMap[edbInfo.EdbInfoId] = edbInfo if edbInfo.NoUpdate == 1 { continue } newBaseEdbInfoArr = append(newBaseEdbInfoArr, edbInfo) } } // 预测基础指标 for _, edbInfo := range tmpBasePredictEdbInfoArr { if _, ok := newPredictBaseMap[edbInfo.EdbInfoId]; !ok { newPredictBaseMap[edbInfo.EdbInfoId] = edbInfo newBasePredictEdbInfoArr = append(newBasePredictEdbInfoArr, edbInfo) } } // 普通计算指标 for _, edbInfo := range tmpCalculateMap { if _, ok := newCalculateMap[edbInfo.EdbInfoId]; !ok { if edbInfo.NoUpdate == 1 { continue } newCalculateMap[edbInfo.EdbInfoId] = edbInfo calculateArr = append(calculateArr, edbInfo.EdbInfoId) } } // 预测计算指标 for _, edbInfo := range tmpPredictCalculateMap { if _, ok := newPredictCalculateMap[edbInfo.EdbInfoId]; !ok { newPredictCalculateMap[edbInfo.EdbInfoId] = edbInfo predictCalculateArr = append(predictCalculateArr, edbInfo.EdbInfoId) } } } // 普通计算指标的id sort.Ints(calculateArr) // 预测计算指标的id sort.Ints(predictCalculateArr) return } // getRefreshEdbInfoListByTraceEdbInfo根据溯源获取关联指标 func getRefreshEdbInfoListByTraceEdbInfo(traceEdbInfo data_manage.TraceEdbInfoResp, existEdbInfoIdMap map[int]int) (newBaseEdbInfoArr, newBasePredictEdbInfoArr []*data_manage.EdbInfo, newCalculateMap, newPredictCalculateMap map[int]*data_manage.EdbInfo, calculateArr, predictCalculateArr []int) { newBaseEdbInfoArr = make([]*data_manage.EdbInfo, 0) newBasePredictEdbInfoArr = make([]*data_manage.EdbInfo, 0) newCalculateMap = make(map[int]*data_manage.EdbInfo) newPredictCalculateMap = make(map[int]*data_manage.EdbInfo) calculateArr = make([]int, 0) predictCalculateArr = make([]int, 0) _, ok := existEdbInfoIdMap[traceEdbInfo.EdbInfoId] if ok { return } existEdbInfoIdMap[traceEdbInfo.EdbInfoId] = traceEdbInfo.EdbInfoId switch traceEdbInfo.EdbInfoType { //0-普通指标; 1-预测指标 case 0: // 0-普通指标 if traceEdbInfo.EdbType == 1 { //1-基础指标 newBaseEdbInfoArr = append(newBaseEdbInfoArr, traceEdbInfo.EdbInfo) } else if traceEdbInfo.EdbType == 2 { //2-计算指标 newCalculateMap[traceEdbInfo.EdbInfoId] = traceEdbInfo.EdbInfo calculateArr = append(calculateArr, traceEdbInfo.EdbInfoId) } case 1: // 1-预测指标 if traceEdbInfo.EdbType == 1 { //1-基础指标 newBasePredictEdbInfoArr = append(newBasePredictEdbInfoArr, traceEdbInfo.EdbInfo) } else if traceEdbInfo.EdbType == 2 { //2-计算指标 newPredictCalculateMap[traceEdbInfo.EdbInfoId] = traceEdbInfo.EdbInfo predictCalculateArr = append(predictCalculateArr, traceEdbInfo.EdbInfoId) } } if traceEdbInfo.Child != nil && len(traceEdbInfo.Child) > 0 { for _, v := range traceEdbInfo.Child { tmpBaseEdbInfoArr, tmpPredictEdbInfoArr, tmpCalculateMap, tmpPredictCalculateMap, tmpCalculateArr, tmpPredictCalculateArr := getRefreshEdbInfoListByTraceEdbInfo(v, existEdbInfoIdMap) newBaseEdbInfoArr = append(newBaseEdbInfoArr, tmpBaseEdbInfoArr...) newBasePredictEdbInfoArr = append(newBasePredictEdbInfoArr, tmpPredictEdbInfoArr...) for k, tmpEdbInfo := range tmpCalculateMap { newCalculateMap[k] = tmpEdbInfo } for k, tmpEdbInfo := range tmpPredictCalculateMap { newPredictCalculateMap[k] = tmpEdbInfo } calculateArr = append(calculateArr, tmpCalculateArr...) predictCalculateArr = append(predictCalculateArr, tmpPredictCalculateArr...) } } return } func edbInfoRefreshAll(refreshAll bool, newBaseEdbInfoArr, newBasePredictEdbInfoArr []*data_manage.EdbInfo, newCalculateMap, newPredictCalculateMap map[int]*data_manage.EdbInfo, calculateArr, predictCalculateArr []int) (err error) { var errmsg string defer func() { if err != nil { fmt.Println("EdbInfoRefreshAllFromBaseV2 Err:" + err.Error() + ";errmsg:" + errmsg) go alarm_msg.SendAlarmMsg("EdbInfoRefreshFromBaseV2,Err"+err.Error()+";errMsg:"+errmsg, 3) //go utils.SendEmail(utils.APPNAME+"【"+utils.RunMode+"】"+"失败提醒", "EdbInfoRefreshFromBase:"+errmsg, utils.EmailSendToUsers) } }() var startDate string for _, bv := range newBaseEdbInfoArr { //source := bv.Source //edbInfoId := bv.EdbInfoId //edbCode := bv.EdbCode if bv.StartDate == "0000-00-00" { continue } //开始时间 startDate = `` if refreshAll { //刷新所有数据，用开始时间作为起始日期去刷新 sTime, err := time.Parse(utils.FormatDate, bv.StartDate) if err != nil { return err } startDate = sTime.Format(utils.FormatDate) } else { sTime, err := time.Parse(utils.FormatDate, bv.EndDate) if err != nil { return err } frequency := bv.Frequency var limitDay int switch frequency { case "日度": limitDay = utils.DATA_START_REFRESH_LIMIT case "周度": limitDay = utils.DATA_START_REFRESH_LIMIT * 7 case "月度": limitDay = utils.DATA_START_REFRESH_LIMIT * 30 case "季度": limitDay = utils.DATA_START_REFRESH_LIMIT * 90 case "年度": limitDay = utils.DATA_START_REFRESH_LIMIT * 365 default: limitDay = utils.DATA_START_REFRESH_LIMIT } startDate = sTime.AddDate(0, 0, -limitDay).Format(utils.FormatDate) } result, err := RefreshEdbData(bv.EdbInfoId, bv.Source, bv.SubSource, bv.EdbCode, startDate) if err != nil { fmt.Println(bv.EdbInfoId, "RefreshBaseEdbData err", time.Now()) errmsg = "RefreshBaseEdbData Err:" + err.Error() return err } if result.Ret != 200 { fmt.Println(bv.EdbInfoId, "RefreshBaseEdbData err;msg:", result.Msg, ";errMsg:", result.ErrMsg) errmsg = fmt.Sprint(bv.EdbInfoId, "RefreshBaseEdbData err;msg:", result.Msg, ";errMsg:", result.ErrMsg) return fmt.Errorf("刷新失败, err:", errmsg) } //maxAndMinItem, err := data_manage.GetEdbInfoMaxAndMinInfo(source, edbCode) //if err != nil { // if err.Error() == utils.ErrNoRow() { //找不到数据，那么就进入到下一条数据做处理 // continue // } // return err //} //if maxAndMinItem != nil { // err = data_manage.ModifyEdbInfoMaxAndMinInfo(edbInfoId, maxAndMinItem) // if err != nil { // return err // } //} fmt.Println("end newBaseEdbInfoArr:", bv, time.Now()) } //刷新相关普通计算指标 for _, v := range calculateArr { edbInfo := newCalculateMap[v] if edbInfo == nil { return err } //计算指标就不用开始时间了吧，因为存在说开始时间变更的情况 startDate = `` if !refreshAll { //如果不是全部更新，那么以指标的开始时间作为刷新 startDate = edbInfo.StartDate source := edbInfo.Source if startDate == "" || startDate == "0000-00-00" { //如果没有开始日期，说明还没有计算出来数据，那么就往前面推40年吧（也意味着重新计算了） startDate = time.Now().AddDate(-40, 0, 0).Format(utils.FormatDate) } else { if source == utils.DATA_SOURCE_CALCULATE { sTime, err := time.Parse(utils.FormatDate, edbInfo.EndDate) if err != nil { return err } frequency := edbInfo.Frequency var limitDay int switch frequency { case "日度": limitDay = utils.DATA_START_REFRESH_LIMIT case "周度": limitDay = utils.DATA_START_REFRESH_LIMIT * 7 case "月度": limitDay = utils.DATA_START_REFRESH_LIMIT * 30 case "季度": limitDay = utils.DATA_START_REFRESH_LIMIT * 90 case "年度": limitDay = utils.DATA_START_REFRESH_LIMIT * 365 default: limitDay = utils.DATA_START_REFRESH_LIMIT } startDate = sTime.AddDate(0, 0, -limitDay).Format(utils.FormatDate) } } } result, err := RefreshEdbCalculateData(edbInfo.EdbInfoId, edbInfo.EdbCode, startDate) if err != nil { fmt.Println(v, "RefreshEdbCalculateData err", time.Now()) errmsg = "RefreshEdbCalculateData Err:" + err.Error() return err } if result.Ret != 200 { fmt.Println(v, "RefreshEdbCalculateData err;msg:", result.Msg, ";errMsg:", result.ErrMsg) errmsg = fmt.Sprint(v, "RefreshEdbCalculateData err;msg:", result.Msg, ";errMsg:", result.ErrMsg) return fmt.Errorf("刷新失败") } } //刷新相关预测计算指标 for _, v := range predictCalculateArr { edbInfo := newPredictCalculateMap[v] if edbInfo == nil { return err } startDate = edbInfo.StartDate source := edbInfo.Source if startDate == "" || startDate == "0000-00-00" { //如果没有开始日期，说明还没有计算出来数据，那么就往前面推40年吧（也意味着重新计算了） startDate = time.Now().AddDate(-40, 0, 0).Format(utils.FormatDate) } else { if source == utils.DATA_SOURCE_PREDICT_CALCULATE { startDate = `` if refreshAll { //刷新所有数据，用开始时间作为起始日期去刷新 startDate = edbInfo.StartDate } else { sTime, err := time.Parse(utils.FormatDate, edbInfo.EndDate) if err != nil { return err } frequency := edbInfo.Frequency var limitDay int switch frequency { case "日度": limitDay = utils.DATA_START_REFRESH_LIMIT case "周度": limitDay = utils.DATA_START_REFRESH_LIMIT * 7 case "月度": limitDay = utils.DATA_START_REFRESH_LIMIT * 30 case "季度": limitDay = utils.DATA_START_REFRESH_LIMIT * 90 case "年度": limitDay = utils.DATA_START_REFRESH_LIMIT * 365 default: limitDay = utils.DATA_START_REFRESH_LIMIT } startDate = sTime.AddDate(0, 0, -limitDay).Format(utils.FormatDate) } } } result, err := RefreshPredictEdbCalculateData(edbInfo.EdbInfoId, edbInfo.EdbCode, startDate) if err != nil { fmt.Println(v, "RefreshEdbCalculateData err", time.Now()) errmsg = "RefreshEdbCalculateData Err:" + err.Error() return err } if result.Ret != 200 { fmt.Println(v, "RefreshPredictEdbCalculateData err;msg:", result.Msg, ";errMsg:", result.ErrMsg) errmsg = fmt.Sprint(v, "RefreshPredictEdbCalculateData err;msg:", result.Msg, ";errMsg:", result.ErrMsg) return fmt.Errorf("刷新失败") } } // 4、更新动态环差值 { predictEdbInfoIdList := make([]int, 0) for _, v := range newBasePredictEdbInfoArr { predictEdbInfoIdList = append(predictEdbInfoIdList, v.EdbInfoId) } fmt.Println("predictEdbConfCalculateMappingDetailList:", predictEdbInfoIdList) predictEdbConfCalculateMappingDetailList, err := data_manage.GetPredictEdbConfCalculateMappingDetailListByEdbInfoId(predictEdbInfoIdList) if err != nil { errmsg = fmt.Sprint("更新动态环差值时，获取数据失败:" + err.Error()) return fmt.Errorf("刷新失败, err:", errmsg) } for _, bv := range predictEdbConfCalculateMappingDetailList { result, err := RefreshEdbData(bv.EdbInfoId, utils.DATA_SOURCE_PREDICT, utils.DATA_SUB_SOURCE_EDB, bv.EdbCode, "") if err != nil { fmt.Println(bv.EdbInfoId, "RefreshBasePredictEdbData err", time.Now()) errmsg = "RefreshBasePredictEdbData Err:" + err.Error() return err } if result.Ret != 200 { fmt.Println(bv.EdbInfoId, "RefreshBasePredictEdbData err;msg:", result.Msg, ";errMsg:", result.ErrMsg) errmsg = fmt.Sprint(bv.EdbInfoId, "RefreshBasePredictEdbData err;msg:", result.Msg, ";errMsg:", result.ErrMsg) return fmt.Errorf("刷新失败, err:%s", errmsg) } fmt.Println("end predictEdbConfCalculateMappingDetailList:", bv, time.Now()) } } //5、再次刷新相关预测计算指标 for _, v := range predictCalculateArr { edbInfo := newPredictCalculateMap[v] if edbInfo == nil { return err } startDate = edbInfo.StartDate source := edbInfo.Source if startDate == "" || startDate == "0000-00-00" { //如果没有开始日期，说明还没有计算出来数据，那么就往前面推40年吧（也意味着重新计算了） startDate = time.Now().AddDate(-40, 0, 0).Format(utils.FormatDate) } else { if source == utils.DATA_SOURCE_PREDICT_CALCULATE { startDate = `` if refreshAll { //刷新所有数据，用开始时间作为起始日期去刷新 startDate = edbInfo.StartDate } else { sTime, err := time.Parse(utils.FormatDate, edbInfo.EndDate) if err != nil { return err } frequency := edbInfo.Frequency var limitDay int switch frequency { case "日度": limitDay = utils.DATA_START_REFRESH_LIMIT case "周度": limitDay = utils.DATA_START_REFRESH_LIMIT * 7 case "月度": limitDay = utils.DATA_START_REFRESH_LIMIT * 30 case "季度": limitDay = utils.DATA_START_REFRESH_LIMIT * 90 case "年度": limitDay = utils.DATA_START_REFRESH_LIMIT * 365 default: limitDay = utils.DATA_START_REFRESH_LIMIT } startDate = sTime.AddDate(0, 0, -limitDay).Format(utils.FormatDate) } } } result, err := RefreshPredictEdbCalculateData(edbInfo.EdbInfoId, edbInfo.EdbCode, startDate) if err != nil { fmt.Println(v, "RefreshEdbCalculateData err", time.Now()) errmsg = "RefreshEdbCalculateData Err:" + err.Error() return err } if result.Ret != 200 { fmt.Println(v, "RefreshPredictEdbCalculateData err;msg:", result.Msg, ";errMsg:", result.ErrMsg) errmsg = fmt.Sprint(v, "RefreshPredictEdbCalculateData err;msg:", result.Msg, ";errMsg:", result.ErrMsg) return fmt.Errorf("刷新失败") } } return err } // EdbInfoRefresh 刷新单个指标,不会刷新关联指标 func EdbInfoRefresh(edbInfo *data_manage.EdbInfoView, refreshAll bool) (err error) { var errmsg string defer func() { if err != nil { fmt.Println("EdbInfoRefresh Err:" + err.Error() + ";errmsg:" + errmsg) go alarm_msg.SendAlarmMsg("EdbInfoRefresh,Err"+err.Error()+";errMsg:"+errmsg, 3) //go utils.SendEmail(utils.APPNAME+"【"+utils.RunMode+"】"+"失败提醒", "EdbInfoRefreshFromBase:"+errmsg, utils.EmailSendToUsers) } }() switch edbInfo.EdbInfoType { //指标类型，0：普通指标，1：预测指标 case 0: if edbInfo.StartDate == "0000-00-00" { return } //开始时间 startDate := `` if edbInfo.EdbType == 1 { // 基础指标 if refreshAll { //刷新所有数据，用开始时间作为起始日期去刷新 sTime, err := time.Parse(utils.FormatDate, edbInfo.StartDate) if err != nil { return err } startDate = sTime.Format(utils.FormatDate) } else { sTime, err := time.Parse(utils.FormatDate, edbInfo.EndDate) if err != nil { return err } frequency := edbInfo.Frequency var limitDay int switch frequency { case "日度": limitDay = utils.DATA_START_REFRESH_LIMIT case "周度": limitDay = utils.DATA_START_REFRESH_LIMIT * 7 case "月度": limitDay = utils.DATA_START_REFRESH_LIMIT * 30 case "季度": limitDay = utils.DATA_START_REFRESH_LIMIT * 90 case "年度": limitDay = utils.DATA_START_REFRESH_LIMIT * 365 default: limitDay = utils.DATA_START_REFRESH_LIMIT } startDate = sTime.AddDate(0, 0, -limitDay).Format(utils.FormatDate) } result, err := RefreshEdbData(edbInfo.EdbInfoId, edbInfo.Source, edbInfo.SubSource, edbInfo.EdbCode, startDate) if err != nil { fmt.Println(edbInfo.EdbInfoId, "RefreshBaseEdbData err", time.Now()) errmsg = "RefreshBaseEdbData Err:" + err.Error() return err } if result.Ret != 200 { fmt.Println(edbInfo.EdbInfoId, "RefreshBaseEdbData err;msg:", result.Msg, ";errMsg:", result.ErrMsg) errmsg = fmt.Sprint(edbInfo.EdbInfoId, "RefreshBaseEdbData err;msg:", result.Msg, ";errMsg:", result.ErrMsg) return fmt.Errorf("刷新失败, err:", errmsg) } } else { //计算指标 //计算指标就不用开始时间了吧，因为存在说开始时间变更的情况 if !refreshAll { //如果不是全部更新，那么以指标的开始时间作为刷新 startDate = edbInfo.StartDate source := edbInfo.Source if startDate == "" || startDate == "0000-00-00" { //如果没有开始日期，说明还没有计算出来数据，那么就往前面推40年吧（也意味着重新计算了） startDate = time.Now().AddDate(-40, 0, 0).Format(utils.FormatDate) } else { if source == utils.DATA_SOURCE_CALCULATE { sTime, err := time.Parse(utils.FormatDate, edbInfo.EndDate) if err != nil { return err } frequency := edbInfo.Frequency var limitDay int switch frequency { case "日度": limitDay = utils.DATA_START_REFRESH_LIMIT case "周度": limitDay = utils.DATA_START_REFRESH_LIMIT * 7 case "月度": limitDay = utils.DATA_START_REFRESH_LIMIT * 30 case "季度": limitDay = utils.DATA_START_REFRESH_LIMIT * 90 case "年度": limitDay = utils.DATA_START_REFRESH_LIMIT * 365 default: limitDay = utils.DATA_START_REFRESH_LIMIT } startDate = sTime.AddDate(0, 0, -limitDay).Format(utils.FormatDate) } } } result, err := RefreshEdbCalculateData(edbInfo.EdbInfoId, edbInfo.EdbCode, startDate) if err != nil { fmt.Println(edbInfo.EdbInfoId, "RefreshEdbCalculateData err", time.Now()) errmsg = "RefreshEdbCalculateData Err:" + err.Error() return err } if result.Ret != 200 { fmt.Println(edbInfo.EdbInfoId, "RefreshEdbCalculateData err;msg:", result.Msg, ";errMsg:", result.ErrMsg) errmsg = fmt.Sprint(edbInfo.EdbInfoId, "RefreshEdbCalculateData err;msg:", result.Msg, ";errMsg:", result.ErrMsg) return fmt.Errorf("刷新失败") } } case 1: startDate := edbInfo.StartDate var result *models.BaseResponse if startDate == "" || startDate == "0000-00-00" { //如果没有开始日期，说明还没有计算出来数据，那么就往前面推40年吧（也意味着重新计算了） startDate = time.Now().AddDate(-40, 0, 0).Format(utils.FormatDate) } else { startDate = `` if refreshAll { //刷新所有数据，用开始时间作为起始日期去刷新 startDate = edbInfo.StartDate } else { sTime, err := time.Parse(utils.FormatDate, edbInfo.EndDate) if err != nil { return err } frequency := edbInfo.Frequency var limitDay int switch frequency { case "日度": limitDay = utils.DATA_START_REFRESH_LIMIT case "周度": limitDay = utils.DATA_START_REFRESH_LIMIT * 7 case "月度": limitDay = utils.DATA_START_REFRESH_LIMIT * 30 case "季度": limitDay = utils.DATA_START_REFRESH_LIMIT * 90 case "年度": limitDay = utils.DATA_START_REFRESH_LIMIT * 365 default: limitDay = utils.DATA_START_REFRESH_LIMIT } startDate = sTime.AddDate(0, 0, -limitDay).Format(utils.FormatDate) } } if edbInfo.EdbType == 1 { //基础指标 result, err = RefreshPredictEdbCalculateData(edbInfo.EdbInfoId, edbInfo.EdbCode, startDate) } else { // 预测计算指标 // 4、更新动态环差值 { predictEdbConfCalculateMappingDetailList, err := data_manage.GetPredictEdbConfCalculateMappingDetailListByEdbInfoId([]int{edbInfo.EdbInfoId}) if err != nil { errmsg = fmt.Sprint("更新动态环差值时，获取数据失败:" + err.Error()) return fmt.Errorf("刷新失败, err:", errmsg) } for _, bv := range predictEdbConfCalculateMappingDetailList { result, err := RefreshEdbData(bv.EdbInfoId, utils.DATA_SOURCE_PREDICT, utils.DATA_SUB_SOURCE_EDB, bv.EdbCode, "") if err != nil { fmt.Println(bv.EdbInfoId, "RefreshBasePredictEdbData err", time.Now()) errmsg = "RefreshBasePredictEdbData Err:" + err.Error() return err } if result.Ret != 200 { fmt.Println(bv.EdbInfoId, "RefreshBasePredictEdbData err;msg:", result.Msg, ";errMsg:", result.ErrMsg) errmsg = fmt.Sprint(bv.EdbInfoId, "RefreshBasePredictEdbData err;msg:", result.Msg, ";errMsg:", result.ErrMsg) return fmt.Errorf("刷新失败, err:%s", errmsg) } fmt.Println("end predictEdbConfCalculateMappingDetailList:", bv, time.Now()) } } } if err != nil { fmt.Println(edbInfo.EdbInfoId, "RefreshEdbCalculateData err", time.Now()) errmsg = "RefreshEdbCalculateData Err:" + err.Error() return err } if result.Ret != 200 { fmt.Println(edbInfo.EdbInfoId, "RefreshPredictEdbCalculateData err;msg:", result.Msg, ";errMsg:", result.ErrMsg) errmsg = fmt.Sprint(edbInfo.EdbInfoId, "RefreshPredictEdbCalculateData err;msg:", result.Msg, ";errMsg:", result.ErrMsg) return fmt.Errorf("刷新失败") } } return err } // AddEdbInfo 新增手工指标数据 func AddEdbInfo(secName, unit, frequency, noticeTime, mobile string, classifyId, userId int, userName string) (err error) { tradeCode, err := models.GetMaxTradeCode() if err != nil { return } if tradeCode == "" { err = errors.New("最大编码为空") return } maxTradeCode, err := utils.GetMaxTradeCode(tradeCode) if err != nil { err = errors.New("获取编码信息失败,Err:" + err.Error()) return } if maxTradeCode == "" { err = errors.New("编码为空，请联系技术") return } //判断指标名称是否为空 if secName == "" { err = errors.New("指标名称不能为空") return } item, err := models.GetEdbinfoBySecName(secName) if err != nil && err.Error() != utils.ErrNoRow() { err = errors.New("获取信息失败，Err:" + err.Error()) return } if item != nil { err = errors.New("指标名称已存在") return } // 先删除该指标编码的历史数据（避免上海删除指标时，没有删除该指标数据） _ = models.DeleteAllEdbData(maxTradeCode) // 先删除历史的操作日志 _ = models.DelEdbinfoOpRecordByTradeCode(maxTradeCode) // 再次新增指标 err = models.AddEdbinfo(maxTradeCode, secName, unit, "手动", frequency, noticeTime, classifyId, userId, userName) if err != nil { err = errors.New("新增失败，Err:" + err.Error()) return } err = models.AddEdbinfoUser(maxTradeCode, mobile) return } //全部刷新指标 //func EdbInfoRefreshAllFromBase(edbInfoId int) (err error) { // var errmsg string // defer func() { // if err != nil { // fmt.Println("EdbInfoRefreshAllFromBase Err:" + err.Error() + ";errmsg:" + errmsg) // go utils.SendEmail(utils.APPNAME+"【"+utils.RunMode+"】"+"失败提醒", "EdbInfoRefreshFromBase:"+errmsg, utils.EmailSendToUsers) // } // }() // // calculateList, err := data_manage.GetEdbInfoAllCalculate(edbInfoId) // if err != nil && err.Error() != utils.ErrNoRow() { // return errors.New("GetEdbInfoAllCalculate Err:" + err.Error()) // } // baseEdbInfoArr := make([]*data_manage.EdbInfo, 0) // calculateInfoArr := make([]*data_manage.EdbInfo, 0) // baseEdbInfo, err := data_manage.GetEdbInfoById(edbInfoId) // if err != nil { // return errors.New("GetEdbInfoAllCalGetEdbInfoByIdr:" + err.Error()) // } // baseEdbInfoArr = append(baseEdbInfoArr, baseEdbInfo) // // for _, v := range calculateList { // getBaseEdbInfoArr, getCalculateInfoArr, err := data_manage.GetRefreshEdbInfoFromBase(v.EdbInfoId, v.Source) // if err != nil { // fmt.Println("GetRefreshEdbInfoFromBase err:" + err.Error()) // errmsg = "获取需要刷新的指标失败:Err:" + err.Error() // return err // } // baseEdbInfoArr = append(baseEdbInfoArr, getBaseEdbInfoArr...) // calculateInfoArr = append(calculateInfoArr, getCalculateInfoArr...) // calculateInfoArr = append(calculateInfoArr, v) // } // //取重 // newBaseEdbInfoArr := make([]*data_manage.EdbInfo, 0) // baseMap := make(map[int]int) // for _, v := range baseEdbInfoArr { // if _, ok := baseMap[v.EdbInfoId]; !ok { // newBaseEdbInfoArr = append(newBaseEdbInfoArr, v) // } // baseMap[v.EdbInfoId] = v.EdbInfoId // } // // var calculateArr []int // newCalculateMap := make(map[int]*data_manage.EdbInfo) // for _, v := range calculateInfoArr { // if _, ok := newCalculateMap[v.EdbInfoId]; !ok { // calculateArr = append(calculateArr, v.EdbInfoId) // } // newCalculateMap[v.EdbInfoId] = v // } // sort.Ints(calculateArr) // var startDate string // for _, bv := range newBaseEdbInfoArr { // source := bv.Source // edbInfoId := bv.EdbInfoId // edbCode := bv.EdbCode // // sTime, err := time.Parse(utils.FormatDate, bv.StartDate) // if err != nil { // return err // } // startDate = sTime.Format(utils.FormatDate) // // if source == utils.DATA_SOURCE_THS { // endDate := time.Now().AddDate(utils.DATA_END_DATE_LIMIT, 0, 0).Format(utils.FormatDate) // err = data_manage.RefreshAllEdbDataByThs(edbInfoId, source, edbCode, startDate, endDate) // if err != nil { // errmsg = "RefreshAllEdbDataByThs Err:" + err.Error() // } // } else if source == utils.DATA_SOURCE_WIND { // endDate := time.Now().Format(utils.FormatDate) // err = data_manage.RefreshAllEdbDataByWind(edbInfoId, source, edbCode, startDate, endDate) // if err != nil { // errmsg = "RefreshAllEdbDataByWind Err:" + err.Error() // } // } else if source == utils.DATA_SOURCE_PB { // startDate := sTime.Format(utils.FormatDateUnSpace) // endDate := time.Now().AddDate(utils.DATA_END_DATE_LIMIT, 0, 0).Format(utils.FormatDateUnSpace) // err = data_manage.RefreshAllEdbDataByPb(edbInfoId, source, edbCode, startDate, endDate) // if err != nil { // errmsg = "RefreshAllEdbDataByPb Err:" + err.Error() // } // } else if source == utils.DATA_SOURCE_MANUAL { // endDate := time.Now().AddDate(utils.DATA_END_DATE_LIMIT, 0, 0).Format(utils.FormatDate) // err = data_manage.RefreshAllEdbDataByManual(edbInfoId, source, edbCode, startDate, endDate) // if err != nil { // errmsg = "RefreshAllEdbDataByManual Err:" + err.Error() // } // } else if source == utils.DATA_SOURCE_LZ { // endDate := time.Now().AddDate(utils.DATA_END_DATE_LIMIT, 0, 0).Format(utils.FormatDate) // err = data_manage.RefreshAllEdbDataByLz(edbInfoId, source, edbCode, startDate, endDate) // if err != nil { // errmsg = "RefreshAllEdbDataByLz Err:" + err.Error() // } // } else if source == utils.DATA_SOURCE_YS { // endDate := time.Now().AddDate(utils.DATA_END_DATE_LIMIT, 0, 0).Format(utils.FormatDate) // err = data_manage.RefreshAllEdbDataByYs(edbInfoId, source, edbCode, startDate, endDate) // if err != nil { // errmsg = "RefreshAllEdbDataByYs Err:" + err.Error() // } // } else if source == utils.DATA_SOURCE_GL { // endDate := time.Now().AddDate(utils.DATA_END_DATE_LIMIT, 0, 0).Format(utils.FormatDate) // err = data_manage.RefreshAllEdbDataByGl(edbInfoId, source, edbCode, startDate, endDate) // if err != nil { // errmsg = "RefreshAllEdbDataByGl Err:" + err.Error() // } // } else if source == utils.DATA_SOURCE_ZZ { // endDate := time.Now().AddDate(utils.DATA_END_DATE_LIMIT, 0, 0).Format(utils.FormatDate) // err = data_manage.RefreshAllEdbDataByZz(edbInfoId, source, edbCode, startDate, endDate) // if err != nil { // errmsg = "RefreshAllEdbDataByZz Err:" + err.Error() // } // } else if source == utils.DATA_SOURCE_DL { // endDate := time.Now().AddDate(utils.DATA_END_DATE_LIMIT, 0, 0).Format(utils.FormatDate) // err = data_manage.RefreshAllEdbDataByDl(edbInfoId, source, edbCode, startDate, endDate) // if err != nil { // errmsg = "RefreshAllEdbDataByDl Err:" + err.Error() // } // } else if source == utils.DATA_SOURCE_SH { // endDate := time.Now().AddDate(utils.DATA_END_DATE_LIMIT, 0, 0).Format(utils.FormatDate) // err = data_manage.RefreshAllEdbDataBySh(edbInfoId, source, edbCode, startDate, endDate) // if err != nil { // errmsg = "RefreshAllEdbDataBySh Err:" + err.Error() // } // } else if source == utils.DATA_SOURCE_CFFEX { // endDate := time.Now().AddDate(utils.DATA_END_DATE_LIMIT, 0, 0).Format(utils.FormatDate) // err = data_manage.RefreshAllEdbDataByCffex(edbInfoId, source, edbCode, startDate, endDate) // if err != nil { // errmsg = "RefreshAllEdbDataByCffex Err:" + err.Error() // } // } else if source == utils.DATA_SOURCE_SHFE { // endDate := time.Now().AddDate(utils.DATA_END_DATE_LIMIT, 0, 0).Format(utils.FormatDate) // err = data_manage.RefreshAllEdbDataByShfe(edbInfoId, source, edbCode, startDate, endDate) // if err != nil { // errmsg = "RefreshAllEdbDataByShfe Err:" + err.Error() // } // } // // maxAndMinItem, err := data_manage.GetEdbInfoMaxAndMinInfo(source, edbCode) // if err != nil { // return err // } // if maxAndMinItem != nil { // err = data_manage.ModifyEdbInfoMaxAndMinInfo(edbInfoId, maxAndMinItem) // if err != nil { // return err // } // } // fmt.Println("end newBaseEdbInfoArr:", bv, time.Now()) // } // // endDate := "" //计算指标不限制日期 // //刷新相关计算指标 // for _, v := range calculateArr { // edbInfo := newCalculateMap[v] // if edbInfo == nil { // return err // } // edbInfoId := edbInfo.EdbInfoId // source := edbInfo.Source // edbCode := edbInfo.EdbCode // // if source == utils.DATA_SOURCE_CALCULATE { // startDate = edbInfo.StartDate // sTime, err := time.Parse(utils.FormatDate, edbInfo.EndDate) // if err != nil { // return err // } // startDate = sTime.Format(utils.FormatDate) // // var edbInfoIdBytes []string // calculateMap, err := data_manage.GetEdbInfoCalculateDetail(edbInfoId, edbInfo.Source) // if err != nil { // errmsg = "GetEdbInfoCalculateDetail Err:" + err.Error() // return err // } // var formulaStr string // edbInfoList := make([]*data_manage.EdbInfo, 0) // // for _, v := range calculateMap { // formulaStr += v.FromTag + "," // edbInfoIdBytes = append(edbInfoIdBytes, v.FromTag) // edbInfo, _ := data_manage.GetEdbInfoById(v.FromEdbInfoId) // edbInfoList = append(edbInfoList, edbInfo) // } // err = data_manage.RefreshAllCalculate(edbInfoList, int(edbInfoId), source, edbInfo.EdbCode, edbInfo.CalculateFormula, startDate, endDate, edbInfoIdBytes) // if err != nil { // errmsg = "RefreshCalculate Err:" + err.Error() // return err // } // } else if source == utils.DATA_SOURCE_CALCULATE_LJZZY { //刷新累计值转月值 // calculateLjzzy, err := data_manage.GetEdbInfoCalculateLjzzyDetail(edbInfoId) // if err != nil { // errmsg = "GetEdbInfoCalculateLjzzyDetail Err:" + err.Error() // return err // } // fromEdbInfo, err := data_manage.GetEdbInfoById(calculateLjzzy.FromEdbInfoId) // if err != nil { // errmsg = "GetEdbInfoById Err:" + err.Error() // return err // } // startDate = edbInfo.StartDate // endDate = time.Now().Format(utils.FormatDate) // err = data_manage.RefreshAllCalculateLjzzy(edbInfoId, source, fromEdbInfo, calculateLjzzy.EdbCode, startDate, endDate) // if err != nil { // errmsg = "RefreshAllCalculateLjzzy Err:" + err.Error() // return err // } // } else if source == utils.DATA_SOURCE_CALCULATE_TBZ { //刷新同比值 // calculateTbz, err := data_manage.GetEdbInfoCalculateTbzDetail(edbInfoId) // if err != nil { // errmsg = "GetEdbInfoCalculateTbzDetail Err:" + err.Error() // return err // } // fromEdbInfo, err := data_manage.GetEdbInfoById(calculateTbz.FromEdbInfoId) // if err != nil { // errmsg = "GetEdbInfoById Err:" + err.Error() // return err // } // startDate = edbInfo.StartDate // endDate = time.Now().Format(utils.FormatDate) // err = data_manage.RefreshAllCalculateTbz(edbInfoId, source, fromEdbInfo, calculateTbz.EdbCode, startDate, endDate) // if err != nil { // errmsg = "RefreshAllCalculateTbz Err:" + err.Error() // return err // } // } else if source == utils.DATA_SOURCE_CALCULATE_TCZ { //同差值 // calculateTcz, err := data_manage.GetEdbInfoCalculateTczDetail(edbInfoId) // if err != nil { // errmsg = "GetEdbInfoCalculateTczDetail Err:" + err.Error() // return err // } // fromEdbInfo, err := data_manage.GetEdbInfoById(calculateTcz.FromEdbInfoId) // if err != nil { // errmsg = "GetEdbInfoById Err:" + err.Error() // return err // } // startDate = edbInfo.StartDate // endDate = time.Now().Format(utils.FormatDate) // err = data_manage.RefreshAllCalculateTcz(edbInfoId, source, fromEdbInfo, calculateTcz.EdbCode, startDate, endDate) // if err != nil { // errmsg = "RefreshCalculateTcz Err:" + err.Error() // return err // } // } else if source == utils.DATA_SOURCE_CALCULATE_NSZYDPJJS { //N数值移动平均计算 // calculateNszydpjjs, err := data_manage.GetEdbInfoCalculateNszydpjjsDetail(edbInfoId) // if err != nil { // errmsg = "GetEdbInfoCalculateNszydpjjsDetail Err:" + err.Error() // return err // } // fromEdbInfo, err := data_manage.GetEdbInfoById(calculateNszydpjjs.FromEdbInfoId) // if err != nil { // errmsg = "GetEdbInfoById Err:" + err.Error() // return err // } // formulaInt, _ := strconv.Atoi(edbInfo.CalculateFormula) // startDate = edbInfo.StartDate // err = data_manage.RefreshAllCalculateNszydpjjs(edbInfoId, edbInfo.Source, formulaInt, fromEdbInfo, calculateNszydpjjs.EdbCode, startDate) // if err != nil { // errmsg = "RefreshCalculateNszydpjjs Err:" + err.Error() // return err // } // } else if source == utils.DATA_SOURCE_CALCULATE_HBZ { //刷新环比值 // calculateTbz, err := data_manage.GetEdbInfoCalculateHbzDetail(edbInfoId) // if err != nil { // errmsg = "GetEdbInfoCalculateHbzDetail Err:" + err.Error() // return err // } // fromEdbInfo, err := data_manage.GetEdbInfoById(calculateTbz.FromEdbInfoId) // if err != nil { // errmsg = "GetEdbInfoById Err:" + err.Error() // return err // } // startDate = edbInfo.StartDate // endDate = time.Now().Format(utils.FormatDate) // formulaInt, _ := strconv.Atoi(edbInfo.CalculateFormula) // err = data_manage.RefreshAllCalculateHbz(edbInfoId, source, fromEdbInfo, calculateTbz.EdbCode, startDate, endDate, formulaInt) // if err != nil { // errmsg = "RefreshAllCalculateHbz Err:" + err.Error() // return err // } // } else if source == utils.DATA_SOURCE_CALCULATE_HCZ { //刷新环差值 // calculateTbz, err := data_manage.GetEdbInfoCalculateHczDetail(edbInfoId) // if err != nil { // errmsg = "GetEdbInfoCalculateHczDetail Err:" + err.Error() // return err // } // fromEdbInfo, err := data_manage.GetEdbInfoById(calculateTbz.FromEdbInfoId) // if err != nil { // errmsg = "GetEdbInfoById Err:" + err.Error() // return err // } // startDate = edbInfo.StartDate // endDate = time.Now().Format(utils.FormatDate) // formulaInt, _ := strconv.Atoi(edbInfo.CalculateFormula) // err = data_manage.RefreshAllCalculateHcz(edbInfoId, source, fromEdbInfo, calculateTbz.EdbCode, startDate, endDate, formulaInt) // if err != nil { // errmsg = "RefreshAllCalculateHcz Err:" + err.Error() // return err // } // } else if source == utils.DATA_SOURCE_CALCULATE_BP { //刷新变频 // calculateTbz, err := data_manage.GetEdbInfoCalculateBpDetail(edbInfoId) // if err != nil { // errmsg = "GetEdbInfoCalculateTbzDetail Err:" + err.Error() // return err // } // fromEdbInfo, err := data_manage.GetEdbInfoById(calculateTbz.FromEdbInfoId) // if err != nil { // errmsg = "GetEdbInfoById Err:" + err.Error() // return err // } // startDate = edbInfo.StartDate // endDate = time.Now().Format(utils.FormatDate) // err = data_manage.RefreshAllCalculateBp(edbInfoId, source, fromEdbInfo, calculateTbz.EdbCode, startDate, endDate) // if err != nil { // errmsg = "RefreshAllCalculateBp Err:" + err.Error() // return err // } //<<<<<<< HEAD // } else if source == utils.DATA_SOURCE_CALCULATE_TIME_SHIFT { // calculate, err := data_manage.GetEdbInfoCalculateMappingDetail(edbInfoId) // if err != nil { // errmsg = "GetEdbInfoCalculateTbzDetail Err:" + err.Error() // return err // } // fromEdbInfo, err := data_manage.GetEdbInfoById(calculate.FromEdbInfoId) // if err != nil { // errmsg = "GetEdbInfoById Err:" + err.Error() // return err // } // startDate = edbInfo.StartDate // endDate = time.Now().Format(utils.FormatDate) // formulaInt, _ := strconv.Atoi(calculate.CalculateFormula) // err = data_manage.RefreshAllCalculateTimeShift(edbInfoId, source, formulaInt, calculate.MoveType, fromEdbInfo, calculate.EdbCode, startDate, endDate, calculate.MoveFrequency) // if err != nil { // errmsg = "RefreshAllCalculateTimeShift Err:" + err.Error() //======= // } else if source == utils.DATA_SOURCE_CALCULATE_ZJPJ { //刷新直接拼接 // err = data_manage.RefreshAllCalculateZjpj(edbInfo) // if err != nil { // errmsg = "RefreshAllCalculateBp Err:" + err.Error() //>>>>>>> 884fd669cc9e91923781680e1890a519c012d38e // return err // } // } // // maxAndMinItem, err := data_manage.GetEdbInfoMaxAndMinInfo(source, edbCode) // if err != nil { // return err // } // // if maxAndMinItem != nil { // err = data_manage.ModifyEdbInfoMaxAndMinInfo(edbInfoId, maxAndMinItem) // if err != nil { // return err // } // } // fmt.Println("end calculateArr:", v, time.Now()) // } // return err //} // //// AddEdbInfo 新增手工指标数据 //func AddEdbInfo(secName, unit, frequency, noticeTime, mobile string, classifyId, userId int) (err error) { // tradeCode, err := models.GetMaxTradeCode() // if err != nil { // return // } // // if tradeCode == "" { // err = errors.New("最大编码为空") // return // } // maxTradeCode, err := utils.GetMaxTradeCode(tradeCode) // if err != nil { // err = errors.New("获取编码信息失败,Err:" + err.Error()) // return // } // if maxTradeCode == "" { // err = errors.New("编码为空，请联系技术") // return // } // //判断指标名称是否为空 // if secName == "" { // err = errors.New("指标名称不能为空") // return // } // item, err := models.GetEdbinfoBySecName(secName) // if err != nil && err.Error() != utils.ErrNoRow() { // err = errors.New("获取信息失败，Err:" + err.Error()) // return // } // if item != nil { // err = errors.New("指标名称已存在") // return // } // err = models.AddEdbinfo(maxTradeCode, secName, unit, "手动", frequency, noticeTime, classifyId, userId) // if err != nil { // err = errors.New("新增失败，Err:" + err.Error()) // return // } // err = models.AddEdbinfoUser(maxTradeCode, mobile) // return //} // // // // // AddOrEditAllEdbInfoToEs 修复ES中的指标 func AddOrEditAllEdbInfoToEs() { //添加es total, itemInfoList, _ := data_manage.GetEdbInfoFilterList("", []interface{}{}, 0, 100000) for k, itemInfo := range itemInfoList { elastic.EsAddOrEditEdbInfoData(utils.DATA_INDEX_NAME, strconv.Itoa(itemInfo.EdbInfoId), itemInfo) fmt.Println("剩余", int(total)-k-1, "条数据") } } // AddOrEditEdbInfoToEs 添加/修改ES中的指标 func AddOrEditEdbInfoToEs(edbInfoId int) { //添加es itemInfo, _ := data_manage.GetEdbInfoByCondition("AND edb_info_id=?", []interface{}{edbInfoId}) go elastic.EsAddOrEditEdbInfoData(utils.DATA_INDEX_NAME, strconv.Itoa(itemInfo.EdbInfoId), itemInfo) } // DeleteEdbInfoToEs 删除ES中的指标 func DeleteEdbInfoToEs(edbInfoId int) { //添加es go elastic.EsDeleteEdbInfoData(utils.DATA_INDEX_NAME, strconv.Itoa(edbInfoId)) } // RefreshManualData 根据手工指标code刷新手工指标数据 func RefreshManualData(edbCode string) { var err error defer func() { if err != nil { go alarm_msg.SendAlarmMsg("导入手工数据后，根据手工指标code刷新手工指标数据失败提醒,Err"+err.Error(), 3) //go utils.SendEmail(utils.APPNAME+"【"+utils.RunMode+"】"+"导入手工数据后，根据手工指标code刷新手工指标数据失败提醒", "errmsg:"+err.Error(), utils.EmailSendToUsers) } }() edbInfo, err := data_manage.GetEdbInfoByEdbCode(utils.DATA_SOURCE_MANUAL, edbCode) if err != nil { if err.Error() == utils.ErrNoRow() { err = nil } return } // 刷新指标库 err, _ = EdbInfoRefreshAllFromBaseV2(edbInfo.EdbInfoId, true, true) return } // 获取频度的英文版 func GetFrequencyEn(frequency string) (frequencyEn string) { switch frequency { case "日度": frequencyEn = "day" return case "周度": frequencyEn = "week" return case "旬度": frequencyEn = "ten days" return case "月度": frequencyEn = "month" return case "季度": frequencyEn = "quarter" return case "年度": frequencyEn = "year" return } return } func GetLeadUnitEn(unit string) (unitEn string) { switch unit { case "天": unitEn = "day" return case "周": unitEn = "week" return case "月": unitEn = "month" return case "季": unitEn = "quarter" return case "年": unitEn = "year" return } return } // getRefreshEdbInfoList 获取待更新的指标（普通基础指标、普通运算指标，预测运算指标） //func getRefreshEdbInfoList(edbInfoId int) (newBaseEdbInfoArr []*data_manage.EdbInfo, newCalculateMap, newPredictCalculateMap map[int]*data_manage.EdbInfo, calculateArr, predictCalculateArr []int, err error, errMsg string) { // calculateList, err := data_manage.GetEdbInfoAllCalculate(edbInfoId) // if err != nil && err.Error() != utils.ErrNoRow() { // err = errors.New("GetEdbInfoAllCalculate Err:" + err.Error()) // return // } // // 获取指标信息 // baseEdbInfo, err := data_manage.GetEdbInfoById(edbInfoId) // if err != nil { // err = errors.New("GetEdbInfoAllCalGetEdbInfoByIdr:" + err.Error()) // return // } // // baseEdbInfoArr := make([]*data_manage.EdbInfo, 0) // 基础普通指标 // calculateInfoArr := make([]*data_manage.EdbInfo, 0) //基础运算指标 // basePredictEdbInfoArr := make([]*data_manage.EdbInfo, 0) // 预测普通指标 // predictCalculateInfoArr := make([]*data_manage.EdbInfo, 0) //预测运算指标 // // if baseEdbInfo.EdbInfoType == 0 { //普通指标 // switch baseEdbInfo.EdbType { // case 1: //基础指标 // baseEdbInfoArr = append(baseEdbInfoArr, baseEdbInfo) // case 2: //计算指标 // calculateList = append(calculateList, baseEdbInfo) // } // } else { //预测指标 // switch baseEdbInfo.EdbType { // case 1: //基础指标 // basePredictEdbInfoArr = append(basePredictEdbInfoArr, baseEdbInfo) // case 2: //计算指标 // predictCalculateInfoArr = append(predictCalculateInfoArr, baseEdbInfo) // } // } // // // 计算指标 // for _, v := range calculateList { // getBaseEdbInfoArr, getCalculateInfoArr, tmpErr := data_manage.GetRefreshEdbInfoFromBase(v.EdbInfoId, v.Source) // if tmpErr != nil { // err = tmpErr // fmt.Println("GetRefreshEdbInfoFromBase err:" + err.Error()) // errMsg = "获取需要刷新的指标失败:Err:" + err.Error() // return // } // for _, tmpEdbInfo := range getBaseEdbInfoArr { // switch tmpEdbInfo.EdbInfoType { // case 0: //普通基础指标 // baseEdbInfoArr = append(baseEdbInfoArr, tmpEdbInfo) // case 1: //预测基础指标 // basePredictEdbInfoArr = append(basePredictEdbInfoArr, tmpEdbInfo) // } // } // // for _, tmpEdbInfo := range getCalculateInfoArr { // switch tmpEdbInfo.EdbInfoType { // case 0: //普通运算指标 // calculateInfoArr = append(calculateInfoArr, tmpEdbInfo) // case 1: //预测运算指标 // predictCalculateInfoArr = append(predictCalculateInfoArr, tmpEdbInfo) // } // } // switch v.EdbInfoType { // case 0: //普通运算指标 // calculateInfoArr = append(calculateInfoArr, v) // case 1: //预测运算指标 // predictCalculateInfoArr = append(predictCalculateInfoArr, v) // } // } // // // 预测计算指标 // for _, v := range predictCalculateInfoArr { // getBaseEdbInfoArr, getCalculateInfoArr, tmpErr := data_manage.GetRefreshEdbInfoFromBase(v.EdbInfoId, v.Source) // if tmpErr != nil { // err = tmpErr // fmt.Println("GetRefreshEdbInfoFromBase err:" + err.Error()) // errMsg = "获取需要刷新的指标失败:Err:" + err.Error() // return // } // for _, tmpEdbInfo := range getBaseEdbInfoArr { // switch tmpEdbInfo.EdbInfoType { // case 0: //普通基础指标 // baseEdbInfoArr = append(baseEdbInfoArr, tmpEdbInfo) // case 1: //预测基础指标 // basePredictEdbInfoArr = append(basePredictEdbInfoArr, tmpEdbInfo) // } // } // // for _, tmpEdbInfo := range getCalculateInfoArr { // switch tmpEdbInfo.EdbInfoType { // case 0: //普通运算指标 // calculateInfoArr = append(calculateInfoArr, tmpEdbInfo) // case 1: //预测运算指标 // predictCalculateInfoArr = append(predictCalculateInfoArr, tmpEdbInfo) // } // } // switch v.EdbInfoType { // case 0: //普通运算指标 // calculateInfoArr = append(calculateInfoArr, v) // case 1: //预测运算指标 // predictCalculateInfoArr = append(predictCalculateInfoArr, v) // } // } // // // 第一次的计算指标map // newCalculateMap = make(map[int]*data_manage.EdbInfo) // for _, v := range calculateInfoArr { // if _, ok := newCalculateMap[v.EdbInfoId]; !ok { // calculateArr = append(calculateArr, v.EdbInfoId) // } // newCalculateMap[v.EdbInfoId] = v // } // // // 基础预测指标 // if len(basePredictEdbInfoArr) > 0 { // basePredictEdbInfoIdList := make([]int, 0) // for _, v := range basePredictEdbInfoArr { // basePredictEdbInfoIdList = append(basePredictEdbInfoIdList, v.EdbInfoId) // } // baseEdbInfoList, tmpErr := data_manage.GetPredictEdbInfoAllCalculate(basePredictEdbInfoIdList) // if tmpErr != nil { // err = tmpErr // return // } // // tmpCalculateList := make([]*data_manage.EdbInfo, 0) // for _, v := range baseEdbInfoList { // switch v.EdbType { // case 1: // baseEdbInfoArr = append(baseEdbInfoArr, v) // case 2: // // 如果不在第一次里面，那么在第二次 刷新预测指标的时候 需要加进去 // if _, ok := newCalculateMap[v.EdbInfoId]; !ok { // tmpCalculateList = append(tmpCalculateList, v) // } // } // } // // // 这里是查询预测指标用到的 普通计算指标 // for _, v := range tmpCalculateList { // getBaseEdbInfoArr, getCalculateInfoArr, tmpErr := data_manage.GetRefreshEdbInfoFromBase(v.EdbInfoId, v.Source) // if tmpErr != nil { // err = tmpErr // fmt.Println("GetRefreshEdbInfoFromBase err:" + err.Error()) // errMsg = "获取需要刷新的指标失败:Err:" + err.Error() // return // } // // // 基础指标 // for _, tmpEdbInfo := range getBaseEdbInfoArr { // switch tmpEdbInfo.EdbInfoType { // case 0: //普通基础指标 // baseEdbInfoArr = append(baseEdbInfoArr, tmpEdbInfo) // case 1: //预测基础指标 // basePredictEdbInfoArr = append(basePredictEdbInfoArr, tmpEdbInfo) // } // } // // // 计算指标 // for _, tmpEdbInfo := range getCalculateInfoArr { // switch tmpEdbInfo.EdbInfoType { // case 0: //普通运算指标 // calculateInfoArr = append(calculateInfoArr, tmpEdbInfo) // case 1: //预测运算指标 // predictCalculateInfoArr = append(predictCalculateInfoArr, tmpEdbInfo) // } // } // switch v.EdbInfoType { // case 0: //普通运算指标 // calculateInfoArr = append(calculateInfoArr, v) // case 1: //预测运算指标 // predictCalculateInfoArr = append(predictCalculateInfoArr, v) // } // } // // // 第二次计算指标的map // for _, v := range calculateInfoArr { // if _, ok := newCalculateMap[v.EdbInfoId]; !ok { // calculateArr = append(calculateArr, v.EdbInfoId) // } // newCalculateMap[v.EdbInfoId] = v // } // } // // // 普通基础指标去重 // newBaseEdbInfoArr = make([]*data_manage.EdbInfo, 0) // baseMap := make(map[int]int) // for _, v := range baseEdbInfoArr { // if _, ok := baseMap[v.EdbInfoId]; !ok { // newBaseEdbInfoArr = append(newBaseEdbInfoArr, v) // } // baseMap[v.EdbInfoId] = v.EdbInfoId // } // // // 普通计算指标的id // sort.Ints(calculateArr) // // // 预测计算指标去重 // newPredictCalculateMap = make(map[int]*data_manage.EdbInfo) // for _, v := range predictCalculateInfoArr { // if _, ok := newPredictCalculateMap[v.EdbInfoId]; !ok { // predictCalculateArr = append(predictCalculateArr, v.EdbInfoId) // } // newPredictCalculateMap[v.EdbInfoId] = v // } // // 预测计算指标的id // sort.Ints(predictCalculateArr) // // return //} // getRefreshEdbInfoList 获取待更新的指标（普通基础指标、普通运算指标，预测运算指标） func getRefreshEdbInfoListByIds(edbInfoIdList []int) (newBaseEdbInfoArr, newBasePredictEdbInfoArr []*data_manage.EdbInfo, newCalculateMap, newPredictCalculateMap map[int]*data_manage.EdbInfo, calculateArr, predictCalculateArr []int, err error, errMsg string) { calculateList, err := data_manage.GetEdbInfoAllCalculateByEdbInfoIdList(edbInfoIdList) if err != nil && err.Error() != utils.ErrNoRow() { err = errors.New("GetEdbInfoAllCalculate Err:" + err.Error()) return } // 获取指标信息 edbInfoList, err := data_manage.GetEdbInfoByIdList(edbInfoIdList) if err != nil { err = errors.New("GetEdbInfoAllCalGetEdbInfoByIdr:" + err.Error()) return } baseEdbInfoArr := make([]*data_manage.EdbInfo, 0) // 基础普通指标 calculateInfoArr := make([]*data_manage.EdbInfo, 0) //基础运算指标 basePredictEdbInfoArr := make([]*data_manage.EdbInfo, 0) // 预测普通指标 predictCalculateInfoArr := make([]*data_manage.EdbInfo, 0) //预测运算指标 for _, baseEdbInfo := range edbInfoList { // 指标若停更, 则忽略掉 if baseEdbInfo.NoUpdate == 1 { continue } if baseEdbInfo.EdbInfoType == 0 { //普通指标 switch baseEdbInfo.EdbType { case 1: //基础指标 baseEdbInfoArr = append(baseEdbInfoArr, baseEdbInfo) case 2: //计算指标 calculateList = append(calculateList, baseEdbInfo) } } else { //预测指标 switch baseEdbInfo.EdbType { case 1: //基础指标 basePredictEdbInfoArr = append(basePredictEdbInfoArr, baseEdbInfo) case 2: //计算指标 predictCalculateInfoArr = append(predictCalculateInfoArr, baseEdbInfo) } } } // 计算指标 for _, v := range calculateList { getBaseEdbInfoArr, getCalculateInfoArr, tmpErr := data_manage.GetRefreshEdbInfoFromBase(v.EdbInfoId, v.Source) if tmpErr != nil { err = tmpErr fmt.Println("GetRefreshEdbInfoFromBase err:" + err.Error()) errMsg = "获取需要刷新的指标失败:Err:" + err.Error() return } for _, tmpEdbInfo := range getBaseEdbInfoArr { switch tmpEdbInfo.EdbInfoType { case 0: //普通基础指标 baseEdbInfoArr = append(baseEdbInfoArr, tmpEdbInfo) case 1: //预测基础指标 basePredictEdbInfoArr = append(basePredictEdbInfoArr, tmpEdbInfo) } } for _, tmpEdbInfo := range getCalculateInfoArr { switch tmpEdbInfo.EdbInfoType { case 0: //普通运算指标 calculateInfoArr = append(calculateInfoArr, tmpEdbInfo) case 1: //预测运算指标 //predictCalculateInfoArr = append(predictCalculateInfoArr, tmpEdbInfo) if tmpEdbInfo.EdbType == 1 { //预测普通指标 basePredictEdbInfoArr = append(basePredictEdbInfoArr, tmpEdbInfo) } else { predictCalculateInfoArr = append(predictCalculateInfoArr, tmpEdbInfo) } } } switch v.EdbInfoType { case 0: //普通运算指标 calculateInfoArr = append(calculateInfoArr, v) case 1: //预测指标 if v.EdbType == 1 { //预测普通指标 basePredictEdbInfoArr = append(basePredictEdbInfoArr, v) } else { predictCalculateInfoArr = append(predictCalculateInfoArr, v) } } } // 预测计算指标 for _, v := range predictCalculateInfoArr { getBaseEdbInfoArr, getCalculateInfoArr, tmpErr := data_manage.GetRefreshEdbInfoFromBase(v.EdbInfoId, v.Source) if tmpErr != nil { err = tmpErr fmt.Println("GetRefreshEdbInfoFromBase err:" + err.Error()) errMsg = "获取需要刷新的指标失败:Err:" + err.Error() return } for _, tmpEdbInfo := range getBaseEdbInfoArr { switch tmpEdbInfo.EdbInfoType { case 0: //普通基础指标 baseEdbInfoArr = append(baseEdbInfoArr, tmpEdbInfo) case 1: //预测基础指标 if tmpEdbInfo.EdbType == 1 { //预测普通指标 basePredictEdbInfoArr = append(basePredictEdbInfoArr, tmpEdbInfo) } else { predictCalculateInfoArr = append(predictCalculateInfoArr, tmpEdbInfo) } //basePredictEdbInfoArr = append(basePredictEdbInfoArr, tmpEdbInfo) } } for _, tmpEdbInfo := range getCalculateInfoArr { switch tmpEdbInfo.EdbInfoType { case 0: //普通运算指标 calculateInfoArr = append(calculateInfoArr, tmpEdbInfo) case 1: //预测运算指标 if tmpEdbInfo.EdbType == 1 { //预测普通指标 basePredictEdbInfoArr = append(basePredictEdbInfoArr, tmpEdbInfo) } else { predictCalculateInfoArr = append(predictCalculateInfoArr, tmpEdbInfo) } //predictCalculateInfoArr = append(predictCalculateInfoArr, tmpEdbInfo) } } switch v.EdbInfoType { case 0: //普通运算指标 calculateInfoArr = append(calculateInfoArr, v) case 1: //预测指标 if v.EdbType == 1 { //预测普通指标 basePredictEdbInfoArr = append(basePredictEdbInfoArr, v) } else { predictCalculateInfoArr = append(predictCalculateInfoArr, v) } } } // 第一次的计算指标map newCalculateMap = make(map[int]*data_manage.EdbInfo) for _, v := range calculateInfoArr { if _, ok := newCalculateMap[v.EdbInfoId]; !ok { calculateArr = append(calculateArr, v.EdbInfoId) } newCalculateMap[v.EdbInfoId] = v } // 基础预测指标 if len(basePredictEdbInfoArr) > 0 { basePredictEdbInfoIdList := make([]int, 0) for _, v := range basePredictEdbInfoArr { basePredictEdbInfoIdList = append(basePredictEdbInfoIdList, v.EdbInfoId) } baseEdbInfoList, tmpErr := data_manage.GetPredictEdbInfoAllCalculate(basePredictEdbInfoIdList) if tmpErr != nil { err = tmpErr return } tmpCalculateList := make([]*data_manage.EdbInfo, 0) for _, v := range baseEdbInfoList { switch v.EdbType { case 1: baseEdbInfoArr = append(baseEdbInfoArr, v) case 2: // 如果不在第一次里面，那么在第二次 刷新预测指标的时候 需要加进去 if _, ok := newCalculateMap[v.EdbInfoId]; !ok { tmpCalculateList = append(tmpCalculateList, v) } } } // 这里是查询预测指标用到的 普通计算指标 for _, v := range tmpCalculateList { getBaseEdbInfoArr, getCalculateInfoArr, tmpErr := data_manage.GetRefreshEdbInfoFromBase(v.EdbInfoId, v.Source) if tmpErr != nil { err = tmpErr fmt.Println("GetRefreshEdbInfoFromBase err:" + err.Error()) errMsg = "获取需要刷新的指标失败:Err:" + err.Error() return } // 基础指标 for _, tmpEdbInfo := range getBaseEdbInfoArr { switch tmpEdbInfo.EdbInfoType { case 0: //普通基础指标 baseEdbInfoArr = append(baseEdbInfoArr, tmpEdbInfo) case 1: //预测基础指标 basePredictEdbInfoArr = append(basePredictEdbInfoArr, tmpEdbInfo) } } // 计算指标 for _, tmpEdbInfo := range getCalculateInfoArr { switch tmpEdbInfo.EdbInfoType { case 0: //普通运算指标 calculateInfoArr = append(calculateInfoArr, tmpEdbInfo) case 1: //预测运算指标 if v.EdbType == 1 { //预测普通指标 basePredictEdbInfoArr = append(basePredictEdbInfoArr, tmpEdbInfo) } else { predictCalculateInfoArr = append(predictCalculateInfoArr, tmpEdbInfo) } } } switch v.EdbInfoType { case 0: //普通运算指标 calculateInfoArr = append(calculateInfoArr, v) case 1: //预测指标 if v.EdbType == 1 { //预测普通指标 basePredictEdbInfoArr = append(basePredictEdbInfoArr, v) } else { //预测运算指标 predictCalculateInfoArr = append(predictCalculateInfoArr, v) } } } // 第二次计算指标的map for _, v := range calculateInfoArr { if _, ok := newCalculateMap[v.EdbInfoId]; !ok { calculateArr = append(calculateArr, v.EdbInfoId) } newCalculateMap[v.EdbInfoId] = v } } // 普通基础指标去重 newBaseEdbInfoArr = make([]*data_manage.EdbInfo, 0) baseMap := make(map[int]int) for _, v := range baseEdbInfoArr { if _, ok := baseMap[v.EdbInfoId]; !ok { newBaseEdbInfoArr = append(newBaseEdbInfoArr, v) } baseMap[v.EdbInfoId] = v.EdbInfoId } // 普通计算指标的id sort.Ints(calculateArr) // 普通预测指标去重 newBasePredictEdbInfoArr = make([]*data_manage.EdbInfo, 0) basePredictMap := make(map[int]int) for _, v := range basePredictEdbInfoArr { if _, ok := basePredictMap[v.EdbInfoId]; !ok { newBasePredictEdbInfoArr = append(newBasePredictEdbInfoArr, v) } basePredictMap[v.EdbInfoId] = v.EdbInfoId } // 预测计算指标去重 newPredictCalculateMap = make(map[int]*data_manage.EdbInfo) for _, v := range predictCalculateInfoArr { if _, ok := newPredictCalculateMap[v.EdbInfoId]; !ok { predictCalculateArr = append(predictCalculateArr, v.EdbInfoId) } newPredictCalculateMap[v.EdbInfoId] = v } // 预测计算指标的id sort.Ints(predictCalculateArr) return } type AddEdbBaseInfoReq struct { EdbCode string `description:"指标编码"` EdbName string `description:"指标名称"` Unit string `description:"单位"` ClassifyId int `description:"所属分类"` SysUserId int `description:"用户id"` SysUserRealName string `description:"用户真实名称"` } // EdbInfoAdd 添加指标到指标库 func EdbInfoAdd(source, subSource, classifyId int, edbCode, edbName, frequency, unit, startDate, endDate string, sysUserId int, sysUserRealName, lang string) (edbInfo *data_manage.EdbInfo, err error, errMsg string, isSendEmail bool) { // 有些指标的添加是走的指标公共服务 if source == utils.DATA_SOURCE_BUSINESS { req2 := AddEdbBaseInfoReq{ EdbCode: edbCode, EdbName: edbName, Unit: unit, ClassifyId: classifyId, SysUserId: sysUserId, SysUserRealName: sysUserRealName, } reqJson, tmpErr := json.Marshal(req2) if tmpErr != nil { err = tmpErr return } respItem, tmpErr := AddBaseEdbInfo(string(reqJson), source, subSource, lang) if tmpErr != nil { errMsg = "添加失败" err = tmpErr return } isSendEmail = respItem.IsSendEmail if respItem.Ret != 200 { errMsg = respItem.Msg err = errors.New(respItem.ErrMsg) } else { edbInfo, err = data_manage.GetEdbInfoById(respItem.Data.EdbInfoId) } return } isSendEmail = true //判断指标名称是否存在 { var count int var condition string var pars []interface{} //condition += " AND source=? " //pars = append(pars, source) switch lang { case utils.EnLangVersion: condition += " AND edb_name_en = ? " default: condition += " AND edb_name=? " } pars = append(pars, edbName) count, err = data_manage.GetEdbInfoCountByCondition(condition, pars) if err != nil { errMsg = "判断指标名称是否存在失败" err = errors.New("判断指标名称是否存在失败,Err:" + err.Error()) return } if count > 0 { errMsg = "指标名称已存在，请重新填写" err = errors.New(errMsg) isSendEmail = false return } } edbInfo = new(data_manage.EdbInfo) edbInfo.Source = source sourceNameMap := map[int]string{ utils.DATA_SOURCE_THS: "同花顺", utils.DATA_SOURCE_WIND: "wind", utils.DATA_SOURCE_PB: "彭博", utils.DATA_SOURCE_PB_FINANCE: "彭博财务", utils.DATA_SOURCE_MANUAL: "手工数据", utils.DATA_SOURCE_LZ: "隆众", utils.DATA_SOURCE_YS: "SMM", utils.DATA_SOURCE_GL: "钢联", utils.DATA_SOURCE_ZZ: "郑商所", utils.DATA_SOURCE_DL: "大商所", utils.DATA_SOURCE_SH: "上期所", utils.DATA_SOURCE_CFFEX: "中金所", utils.DATA_SOURCE_SHFE: "上期能源", utils.DATA_SOURCE_GIE: "欧洲天然气", utils.DATA_SOURCE_LT: "路透", utils.DATA_SOURCE_COAL: "中国煤炭市场网", utils.DATA_SOURCE_GOOGLE_TRAVEL: "our world in data", utils.DATA_SOURCE_MYSTEEL_CHEMICAL: "钢联", utils.DATA_SOURCE_EIA_STEO: "EIA STEO报告", utils.DATA_SOURCE_COM_TRADE: "UN", utils.DATA_SOURCE_SCI: "SCI", utils.DATA_SOURCE_BAIINFO: "BAIINFO", utils.DATA_SOURCE_STOCK_PLANT: "存量装置", utils.DATA_SOURCE_NATIONAL_STATISTICS: "国家统计局", utils.DATA_SOURCE_FUBAO: "富宝数据", utils.DATA_SOURCE_GFEX: "广期所", utils.DATA_SOURCE_SCI_HQ: "卓创红期", utils.DATA_SOURCE_LY: "粮油商务网", utils.DATA_SOURCE_TRADE_ANALYSIS: "持仓分析", } sourceName, ok := sourceNameMap[source] if !ok { edbSource := data_manage.EdbSourceIdMap[source] if edbSource != nil { sourceName = edbSource.SourceName } if sourceName == "" { errMsg = "指标来源异常" err = errors.New(errMsg) return } } edbInfo.SourceName = sourceName edbType := 1 //基础指标 if source == utils.DATA_SOURCE_STOCK_PLANT { edbType = 2 //计算指标 } // 从缓存中获取 terminalCode, serverUrl, sourceIndexName, e := GetEdbTerminalCodeBySource(edbInfo.Source, edbInfo.EdbCode, edbInfo.StockCode) if e != nil { errMsg = "获取可以使用的终端地址失败" err = errors.New("获取可以使用的终端地址失败，Err:" + e.Error()) return } //获取该层级下最大的排序数 maxSort, err := GetEdbClassifyMaxSort(classifyId, 0) if err != nil { errMsg = "获取失败" err = errors.New("查询排序信息失败，Err:" + err.Error()) return } edbInfo.EdbCode = edbCode edbInfo.EdbName = edbName edbInfo.EdbNameEn = edbName edbInfo.EdbNameSource = edbName edbInfo.Frequency = frequency edbInfo.Unit = unit edbInfo.UnitEn = unit edbInfo.StartDate = startDate edbInfo.EndDate = endDate edbInfo.ClassifyId = classifyId edbInfo.SysUserId = sysUserId edbInfo.SysUserRealName = sysUserRealName edbInfo.CreateTime = time.Now() edbInfo.ModifyTime = time.Now() edbInfo.ServerUrl = serverUrl edbInfo.Sort = maxSort + 1 edbInfo.DataDateType = `交易日` edbInfo.TerminalCode = terminalCode edbInfo.SourceIndexName = sourceIndexName timestamp := strconv.FormatInt(time.Now().UnixNano(), 10) edbInfo.UniqueCode = utils.MD5(utils.DATA_PREFIX + "_" + timestamp) itemVal, err := data_manage.GetEdbInfoMaxAndMinInfo(source, subSource, edbCode) if itemVal != nil && err == nil { edbInfo.MaxValue = itemVal.MaxValue edbInfo.MinValue = itemVal.MinValue } edbInfo.EdbType = edbType edbInfoId, err := data_manage.AddEdbInfo(edbInfo) if err != nil { errMsg = "保存失败" err = errors.New("保存失败，Err:" + err.Error()) return } edbInfo.EdbInfoId = int(edbInfoId) //保存数据 data_manage.ModifyEdbInfoDataStatus(edbInfoId, source, subSource, edbCode) maxAndMinItem, _ := data_manage.GetEdbInfoMaxAndMinInfo(source, subSource, edbCode) if maxAndMinItem != nil { err = data_manage.ModifyEdbInfoMaxAndMinInfo(int(edbInfoId), maxAndMinItem) } //添加es AddOrEditEdbInfoToEs(int(edbInfoId)) go handleByAddEdbInfo(edbInfo) return } // TraceEdbInfoByEdbInfoId 指标追溯 func TraceEdbInfoByEdbInfoId(edbInfoId, sysUserId int) (traceEdbInfo data_manage.TraceEdbInfoResp, err error) { edbInfo, err := data_manage.GetEdbInfoById(edbInfoId) if err != nil { return } edbInfoRuleMap := make(map[int]string, 0) edbMappingMap := make(map[int][]*data_manage.EdbInfoCalculateMappingInfo) //edbInfoRuleMap[edbInfoId] = getEdbRuleTitle(edbInfo) traceEdbInfo = data_manage.TraceEdbInfoResp{ //EdbInfoId: edbInfo.EdbInfoId, EdbInfoId: edbInfoId, EdbInfoType: edbInfo.EdbInfoType, EdbName: edbInfo.EdbName, EdbNameEn: edbInfo.EdbNameEn, EdbType: edbInfo.EdbType, //Source: edbInfo.Source, UniqueCode: edbInfo.UniqueCode, ClassifyId: edbInfo.ClassifyId, IsStop: edbInfo.NoUpdate, EdbInfo: edbInfo, } findIdMap := make(map[int]int) findIdMap[edbInfoId] = edbInfoId existMap := make(map[int]data_manage.TraceEdbInfoResp) traceEdbInfo.Child, err = traceEdbInfoByEdbInfoId(edbInfoId, traceEdbInfo, edbInfoRuleMap, findIdMap, existMap, edbMappingMap) edbInfoIdList := make([]int, 0) for _, v := range findIdMap { edbInfoIdList = append(edbInfoIdList, v) } classifyIdList := make([]int, 0) edbInfoList, err := data_manage.GetEdbInfoByIdList(edbInfoIdList) if err != nil { return } edbInfoMap := make(map[int]*data_manage.EdbInfo) for _, tmpEdbInfo := range edbInfoList { edbInfoMap[tmpEdbInfo.EdbInfoId] = tmpEdbInfo classifyIdList = append(classifyIdList, tmpEdbInfo.ClassifyId) } // 指标权限Map edbInfoIdPermissionMap := make(map[int]bool) // 指标权限 { // 当前的分类 classifyMap := make(map[int]*data_manage.EdbClassify) classifyList, tmpErr := data_manage.GetEdbClassifyByIdList(classifyIdList) if tmpErr != nil { err = tmpErr return } for _, v := range classifyList { classifyMap[v.ClassifyId] = v } // 获取所有有权限的指标和分类 permissionEdbIdList, permissionClassifyIdList, tmpErr := data_manage_permission.GetUserEdbAndClassifyPermissionList(sysUserId, 0, 0) if tmpErr != nil { err = tmpErr return } for _, v := range edbInfoMap { if currClassify, ok := classifyMap[v.ClassifyId]; ok { edbInfoIdPermissionMap[v.EdbInfoId] = data_manage_permission.CheckEdbPermissionByPermissionIdList(v.IsJoinPermission, currClassify.IsJoinPermission, v.EdbInfoId, v.ClassifyId, permissionEdbIdList, permissionClassifyIdList) } } } traceEdbInfo, err = handleTraceEdbInfo(traceEdbInfo, 0, edbInfoMap, edbMappingMap, edbInfoIdPermissionMap) // 权限校验 return } // TraceEdbInfoByEdbInfoIdList 指标追溯 func TraceEdbInfoByEdbInfoIdList(edbInfoIdList []int) (traceEdbInfoList []data_manage.TraceEdbInfoResp, err error) { traceEdbInfoList = make([]data_manage.TraceEdbInfoResp, 0) edbInfoList, err := data_manage.GetEdbInfoByIdList(edbInfoIdList) if err != nil { return } edbInfoRuleMap := make(map[int]string, 0) edbMappingMap := make(map[int][]*data_manage.EdbInfoCalculateMappingInfo) findIdMap := make(map[int]int) existMap := make(map[int]data_manage.TraceEdbInfoResp) for _, edbInfo := range edbInfoList { findIdMap[edbInfo.EdbInfoId] = edbInfo.EdbInfoId //edbInfoRuleMap[edbInfoId] = getEdbRuleTitle(edbInfo) traceEdbInfo := data_manage.TraceEdbInfoResp{ //EdbInfoId: edbInfo.EdbInfoId, EdbInfoId: edbInfo.EdbInfoId, EdbInfoType: edbInfo.EdbInfoType, EdbName: edbInfo.EdbName, EdbNameEn: edbInfo.EdbNameEn, EdbType: edbInfo.EdbType, //Source: edbInfo.Source, UniqueCode: edbInfo.UniqueCode, ClassifyId: edbInfo.ClassifyId, IsStop: edbInfo.NoUpdate, EdbInfo: edbInfo, } traceEdbInfo.Child, err = traceEdbInfoByEdbInfoId(edbInfo.EdbInfoId, traceEdbInfo, edbInfoRuleMap, findIdMap, existMap, edbMappingMap) traceEdbInfoList = append(traceEdbInfoList, traceEdbInfo) } //findEdbInfoIdList := make([]int, 0) //for _, v := range findIdMap { // findEdbInfoIdList = append(findEdbInfoIdList, v) //} //findEdbInfoList, err := data_manage.GetEdbInfoByIdList(findEdbInfoIdList) //if err != nil { // return //} //edbInfoMap := make(map[int]*data_manage.EdbInfo) //for _, tmpEdbInfo := range findEdbInfoList { // edbInfoMap[tmpEdbInfo.EdbInfoId] = tmpEdbInfo //} //for k, traceEdbInfo := range traceEdbInfoList { // traceEdbInfoList[k], err = handleTraceEdbInfo(traceEdbInfo, 0, edbInfoMap, edbMappingMap) //} return } // traceEdbInfoByEdbInfoId 指标追溯 func traceEdbInfoByEdbInfoId(edbInfoId int, traceEdbInfo data_manage.TraceEdbInfoResp, edbInfoRuleMap map[int]string, findIdMap map[int]int, existMap map[int]data_manage.TraceEdbInfoResp, edbMappingMap map[int][]*data_manage.EdbInfoCalculateMappingInfo) (child []data_manage.TraceEdbInfoResp, err error) { traceEdbInfo, ok := existMap[edbInfoId] if ok { return } child = make([]data_manage.TraceEdbInfoResp, 0) edbInfoMappingList, e := data_manage.GetEdbInfoCalculateMappingListByEdbInfoId(edbInfoId) if e != nil { err = fmt.Errorf("GetEdbInfoCalculateMappingListByEdbInfoId err: %s", e.Error()) return } // 指标信息map edbInfoMap := make(map[int]*data_manage.EdbInfo) if len(edbInfoMappingList) > 0 { fromEdbInfoIdList := make([]int, 0) for _, v := range edbInfoMappingList { fromEdbInfoIdList = append(fromEdbInfoIdList, v.FromEdbInfoId) } edbInfoList, tmpErr := data_manage.GetEdbInfoByIdList(fromEdbInfoIdList) if tmpErr != nil { err = fmt.Errorf("traceEdbInfoByEdbInfoId GetEdbInfoByIdList err: %s", tmpErr.Error()) return } for _, v := range edbInfoList { edbInfoMap[v.EdbInfoId] = v } } edbMappingMap[edbInfoId] = edbInfoMappingList for _, v := range edbInfoMappingList { tmpEdbInfoId := v.FromEdbInfoId tmpTraceEdbInfo := data_manage.TraceEdbInfoResp{ EdbInfoId: tmpEdbInfoId, EdbInfoType: v.FromEdbInfoType, EdbType: v.FromEdbType, UniqueCode: v.FromUniqueCode, ClassifyId: v.FromClassifyId, IsStop: v.NoUpdate, EdbInfo: edbInfoMap[v.FromEdbInfoId], } // 计算指标/预测指标继续溯源 if edbInfoId != v.FromEdbInfoId && (v.FromEdbType == 2 || v.FromEdbInfoType == 1) { // 查过了就不查了 if _, ok2 := findIdMap[tmpEdbInfoId]; !ok2 { tmpTraceEdbInfo.Child, e = traceEdbInfoByEdbInfoId(tmpEdbInfoId, tmpTraceEdbInfo, edbInfoRuleMap, findIdMap, existMap, edbMappingMap) if e != nil { err = fmt.Errorf("traceEdbInfoByEdbInfoId err: %s", e.Error()) return } } } child = append(child, tmpTraceEdbInfo) findIdMap[tmpEdbInfoId] = tmpEdbInfoId } existMap[edbInfoId] = traceEdbInfo return } func handleTraceEdbInfo(traceEdbInfoResp data_manage.TraceEdbInfoResp, parentEdbInfoId int, edbInfoMap map[int]*data_manage.EdbInfo, edbMappingMap map[int][]*data_manage.EdbInfoCalculateMappingInfo, edbInfoIdPermissionMap map[int]bool) (newTraceEdbInfoResp data_manage.TraceEdbInfoResp, err error) { edbInfo, ok := edbInfoMap[traceEdbInfoResp.EdbInfoId] if !ok { err = errors.New("指标异常") return } var parentEdbInfo *data_manage.EdbInfo if parentEdbInfoId > 0 { parentEdbInfo, ok = edbInfoMap[parentEdbInfoId] if !ok { err = errors.New("指标异常") return } } //traceEdbInfoResp.EdbName = edbInfo.EdbName traceEdbInfoResp.EdbName, traceEdbInfoResp.EdbNameEn, traceEdbInfoResp.RuleTitle, traceEdbInfoResp.RuleTitleEn = getEdbRuleTitle(edbInfo, parentEdbInfo, traceEdbInfoResp.Child, edbInfoMap, edbMappingMap) if traceEdbInfoResp.Child != nil && len(traceEdbInfoResp.Child) > 0 { for k, v := range traceEdbInfoResp.Child { traceEdbInfoResp.Child[k], err = handleTraceEdbInfo(v, traceEdbInfoResp.EdbInfoId, edbInfoMap, edbMappingMap, edbInfoIdPermissionMap) if err != nil { return } } } // 指标权限 traceEdbInfoResp.HaveOperaAuth, _ = edbInfoIdPermissionMap[traceEdbInfoResp.EdbInfoId] newTraceEdbInfoResp = traceEdbInfoResp return } // getEdbRule 获取规则名称 func getEdbRuleTitle(edbInfo, parentEdbInfo *data_manage.EdbInfo, childList []data_manage.TraceEdbInfoResp, edbInfoMap map[int]*data_manage.EdbInfo, edbMappingMap map[int][]*data_manage.EdbInfoCalculateMappingInfo) (edbName, edbNameEn, ruleTitle, ruleTitleEn string) { edbName = edbInfo.EdbName edbNameEn = edbInfo.EdbNameEn ruleTitle = `来源于` + edbInfo.SourceName ruleTitleEn = `Source From: ` + edbInfo.SourceName if parentEdbInfo != nil { edbMappingList, ok := edbMappingMap[parentEdbInfo.EdbInfoId] if !ok { edbMappingList = []*data_manage.EdbInfoCalculateMappingInfo{} } // 指标名称 switch parentEdbInfo.Source { case utils.DATA_SOURCE_CALCULATE, utils.DATA_SOURCE_PREDICT_CALCULATE: for _, v := range edbMappingList { if v.FromEdbInfoId == edbInfo.EdbInfoId { edbName = fmt.Sprintf("%s（%s）", edbInfo.EdbName, v.FromTag) edbNameEn = fmt.Sprintf("%s（%s）", edbInfo.EdbNameEn, v.FromTag) } } case utils.DATA_SOURCE_CALCULATE_ZJPJ, utils.DATA_SOURCE_PREDICT_CALCULATE_ZJPJ, utils.DATA_SOURCE_CALCULATE_LJZTBPJ, utils.DATA_SOURCE_PREDICT_CALCULATE_LJZTBPJ: // 直接拼接 ,累计值同比拼接 for _, v := range edbMappingList { if v.FromEdbInfoId == edbInfo.EdbInfoId { tmpName := `` tmpNameEn := `` if v.FromTag == `A` { tmpName = `拼接日期前` tmpNameEn = `Before Concatenation Date:` } else if v.FromTag == `B` { tmpName = `拼接日期后` tmpNameEn = `After Concatenation Date:` } edbName = fmt.Sprintf("%s（%s）", edbInfo.EdbName, tmpName) edbNameEn = fmt.Sprintf("%s（%s）", edbInfo.EdbNameEn, tmpNameEn) } } case utils.DATA_SOURCE_CALCULATE_NHCC, utils.DATA_SOURCE_PREDICT_CALCULATE_NHCC: //计算指标（拟合残差） for _, v := range edbMappingList { //（需对上游指标+自变量，领先10天/因变量） if v.FromEdbInfoId == edbInfo.EdbInfoId { tmpName := `` tmpNameEn := `` if v.FromTag == `A` { tmpName = fmt.Sprintf(`自变量，领先%d天`, v.MoveValue) tmpNameEn = fmt.Sprintf(`Independent Variable, Lead %d Days:`, v.MoveValue) } else if v.FromTag == `B` { tmpName = `因变量` tmpNameEn = `Dependent Variable` } edbName = fmt.Sprintf("%s（%s）", edbInfo.EdbName, tmpName) edbNameEn = fmt.Sprintf("%s（%s）", edbInfo.EdbNameEn, tmpNameEn) } } case utils.DATA_SOURCE_CALCULATE_CORRELATION: // 滚动相关性 for _, v := range edbMappingList { if v.FromEdbInfoId == edbInfo.EdbInfoId { edbName = fmt.Sprintf("%s（%s）", edbInfo.EdbName, v.FromTag) edbNameEn = fmt.Sprintf("%s（%s）", edbInfo.EdbNameEn, v.FromTag) } } } } if edbInfo.EdbType == 1 { // 基础指标的话，直接返回来源名称 //ruleTitle = `来源于`+edbInfo.SourceName return } // 规则 switch edbInfo.Source { case utils.DATA_SOURCE_CALCULATE, utils.DATA_SOURCE_PREDICT_CALCULATE: ruleTitle = "=" + edbInfo.CalculateFormula case utils.DATA_SOURCE_CALCULATE_LJZZY, utils.DATA_SOURCE_PREDICT_CALCULATE_LJZZY: ruleTitle = `累计转月值计算` ruleTitleEn = `Cumulative to Monthly Calculation` case utils.DATA_SOURCE_CALCULATE_TBZ, utils.DATA_SOURCE_PREDICT_CALCULATE_TBZ: ruleTitle = `同比值计算` ruleTitleEn = `Year-on-Year Value Calculation` case utils.DATA_SOURCE_CALCULATE_TCZ, utils.DATA_SOURCE_PREDICT_CALCULATE_TCZ: ruleTitle = `同差值计算` ruleTitleEn = `Yearly Difference Calculation` case utils.DATA_SOURCE_CALCULATE_NSZYDPJJS, utils.DATA_SOURCE_PREDICT_CALCULATE_NSZYDPJJS: ruleTitle = fmt.Sprintf("N数值移动均值计算（N=%s）", edbInfo.CalculateFormula) ruleTitleEn = fmt.Sprintf("N-Value Moving Average Calculation（N=%s）", edbInfo.CalculateFormula) case utils.DATA_SOURCE_CALCULATE_HBZ, utils.DATA_SOURCE_PREDICT_CALCULATE_HBZ: ruleTitle = fmt.Sprintf("N数值环比值计算（N=%s）", edbInfo.CalculateFormula) ruleTitleEn = fmt.Sprintf("N-Value Sequential Comparison Calculation（N=%s）", edbInfo.CalculateFormula) case utils.DATA_SOURCE_CALCULATE_HCZ, utils.DATA_SOURCE_PREDICT_CALCULATE_HCZ: ruleTitle = fmt.Sprintf("N数值环差值计算（N=%s）", edbInfo.CalculateFormula) ruleTitleEn = fmt.Sprintf("N-Value Sequential Difference Calculation（N=%s）", edbInfo.CalculateFormula) case utils.DATA_SOURCE_CALCULATE_TIME_SHIFT, utils.DATA_SOURCE_PREDICT_CALCULATE_TIME_SHIFT: moveType := `领先` moveTypeEn := `Lead` if edbInfo.MoveType == 2 { moveType = "滞后" moveTypeEn = `Lag` } ruleTitle = fmt.Sprintf("时间移位计算（%s%s%s）", moveType, edbInfo.CalculateFormula, edbInfo.MoveFrequency) ruleTitleEn = fmt.Sprintf("Time Shift Calculation（%s%s%s）", moveTypeEn, edbInfo.CalculateFormula, edbInfo.MoveFrequency) case utils.DATA_SOURCE_CALCULATE_BP, utils.DATA_SOURCE_PREDICT_CALCULATE_BP: // 变频 childFrequency := `` if len(childList) > 0 { if childEdbInfo, ok := edbInfoMap[childList[0].EdbInfoId]; ok { childFrequency = childEdbInfo.Frequency } } ruleTitle = fmt.Sprintf("升频计算（%s转%s）", childFrequency, edbInfo.Frequency) ruleTitleEn = fmt.Sprintf("Upsampling Calculation（%s转%s）", childFrequency, edbInfo.Frequency) case utils.DATA_SOURCE_CALCULATE_ZJPJ, utils.DATA_SOURCE_PREDICT_CALCULATE_ZJPJ: // 直接拼接 ruleTitle = fmt.Sprintf("直接拼接计算（%s）", edbInfo.CalculateFormula) ruleTitleEn = fmt.Sprintf("Direct Concatenation Calculation（%s）", edbInfo.CalculateFormula) case utils.DATA_SOURCE_CALCULATE_LJZTBPJ, utils.DATA_SOURCE_PREDICT_CALCULATE_LJZTBPJ: // 累计值同比拼 ruleTitle = fmt.Sprintf("累计值同比值拼接计算（%s）", edbInfo.CalculateFormula) ruleTitleEn = fmt.Sprintf("Cumulative Year-on-Year Concatenation Calculation（%s）", edbInfo.CalculateFormula) case utils.DATA_SOURCE_PYTHON: ruleTitle = `代码运算` ruleTitleEn = `Code Computation` case utils.DATA_SOURCE_CALCULATE_CJJX, utils.DATA_SOURCE_PREDICT_CALCULATE_CJJX: ruleTitle = fmt.Sprintf("超季节性计算（N=%s，%s）", edbInfo.CalculateFormula, edbInfo.Calendar) ruleTitleEn = fmt.Sprintf("Super Seasonality Calculation（N=%s，%s）", edbInfo.CalculateFormula, edbInfo.Calendar) case utils.DATA_SOURCE_CALCULATE_NHCC, utils.DATA_SOURCE_PREDICT_CALCULATE_NHCC: //计算指标（拟合残差） var startDate, endDate string dateList := strings.Split(edbInfo.CalculateFormula, ",") if len(dateList) == 2 { startDate = dateList[0] endDate = dateList[1] } ruleTitle = fmt.Sprintf("拟合残差计算（%s至%s）", startDate, endDate) ruleTitleEn = fmt.Sprintf("Fit Residual Calculation（%s至%s）", startDate, endDate) case utils.DATA_SOURCE_CALCULATE_ADJUST: ruleTitle = `数据调整` ruleTitleEn = `Data Adjustment` case utils.DATA_SOURCE_CALCULATE_NH, utils.DATA_SOURCE_PREDICT_CALCULATE_NH: ruleTitle = `年化计算` ruleTitleEn = `Annualization Calculation` case utils.DATA_SOURCE_CALCULATE_KSZS, utils.DATA_SOURCE_PREDICT_CALCULATE_KSZS: // 扩散指数->53 type KszsConfig struct { DateType int `description:"扩散指标日期；1：全部指标日期并集;2:部分指标日期并集"` CheckList []string `description:"选中的数据,A,B,C"` } var config KszsConfig err := json.Unmarshal([]byte(edbInfo.CalculateFormula), &config) if err != nil { return } var startDate, endDate time.Time childEdbInfoIdList := make([]int, 0) if config.DateType == 1 { for _, v := range childList { childEdbInfoIdList = append(childEdbInfoIdList, v.EdbInfoId) } } else { if parentEdbInfo != nil { edbMappingList, ok := edbMappingMap[parentEdbInfo.EdbInfoId] if !ok { edbMappingList = []*data_manage.EdbInfoCalculateMappingInfo{} } tagMap := make(map[string]int) for _, v := range edbMappingList { tagMap[v.FromTag] = v.FromEdbInfoId } for _, v := range config.CheckList { if tmpEdbInfoId, ok := tagMap[v]; ok { childEdbInfoIdList = append(childEdbInfoIdList, tmpEdbInfoId) } } } } for _, v := range childEdbInfoIdList { if childEdbInfo, ok := edbInfoMap[v]; ok { tmpStartDate, tmpErr := time.ParseInLocation(utils.FormatDate, childEdbInfo.StartDate, time.Local) if tmpErr != nil { return } if startDate.IsZero() || startDate.After(tmpStartDate) { startDate = tmpStartDate } tmpEndDate, tmpErr := time.ParseInLocation(utils.FormatDate, childEdbInfo.EndDate, time.Local) if tmpErr != nil { return } if endDate.IsZero() || endDate.Before(tmpEndDate) { endDate = tmpEndDate } } } ruleTitle = fmt.Sprintf("扩散指数计算（%s至%s）", startDate.Format(utils.FormatDate), endDate.Format(utils.FormatDate)) ruleTitleEn = fmt.Sprintf("Diffusion Index Calculation（%s to %s）", startDate.Format(utils.FormatDate), endDate.Format(utils.FormatDate)) case utils.DATA_SOURCE_STOCK_PLANT: ruleTitle = `来源于装置分析` ruleTitleEn = `Derived from Plant Analysis` case utils.DATA_SOURCE_CALCULATE_CORRELATION: type EdbCalculateFormula struct { BaseCalculateValue int `description:"基础计算窗口"` BaseCalculateUnit string `description:"基础计算频度"` LeadValue int `description:"领先期数"` LeadUnit string `description:"频度"` CalculateValue int `description:"计算窗口"` CalculateUnit string `description:"计算频度"` } var correlationConf EdbCalculateFormula err := json.Unmarshal([]byte(edbInfo.CalculateFormula), &correlationConf) if err != nil { return } ruleTitle = fmt.Sprintf("滚动相关性（计算窗口%d%s，B领先A%d%s）", correlationConf.CalculateValue, correlationConf.CalculateUnit, correlationConf.LeadValue, correlationConf.LeadUnit) ruleTitleEn = fmt.Sprintf("Rolling Correlation (Calculation Window %d%s, B Leads A %d%s)", correlationConf.CalculateValue, correlationConf.CalculateUnit, correlationConf.LeadValue, correlationConf.LeadUnit) case utils.DATA_SOURCE_CALCULATE_JP, utils.DATA_SOURCE_PREDICT_CALCULATE_JP: childFrequency := `` if len(childList) > 0 { if childEdbInfo, ok := edbInfoMap[childList[0].EdbInfoId]; ok { childFrequency = childEdbInfo.Frequency } } ruleTitle = fmt.Sprintf("降频计算（%s转%s，%s）", childFrequency, edbInfo.Frequency, edbInfo.CalculateFormula) ruleTitleEn = fmt.Sprintf("Downsampling Calculation（%s to %s，%s）", childFrequency, edbInfo.Frequency, edbInfo.CalculateFormula) case utils.DATA_SOURCE_CALCULATE_STANDARD_DEVIATION: ruleTitle = fmt.Sprintf("标准差（滚动%s期）", edbInfo.CalculateFormula) ruleTitleEn = fmt.Sprintf("Standard Deviation (Rolling %s Periods)", edbInfo.CalculateFormula) case utils.DATA_SOURCE_CALCULATE_PERCENTILE, utils.DATA_SOURCE_PREDICT_CALCULATE_PERCENTILE: type TempCalculate struct { CalculateValue int `description:"计算窗口"` CalculateUnit string `description:"计算频度"` } cf := TempCalculate{} if e := json.Unmarshal([]byte(edbInfo.CalculateFormula), &cf); e != nil { return } ruleTitle = fmt.Sprintf("百分位（时间长度%d%s）", cf.CalculateValue, cf.CalculateUnit) ruleTitle = fmt.Sprintf("Percentile (Time Length %d%s)", cf.CalculateValue, cf.CalculateUnit) case utils.DATA_SOURCE_CALCULATE_ZSXY, utils.DATA_SOURCE_PREDICT_CALCULATE_ZSXY: ruleTitle = `指数修匀计算` ruleTitle = `Index Smoothing Calculation` } return } // GetEdbChartAdminList // @param source 来源 ：1：手工数据指标 2：钢联化工数据库 3：ETA指标库 4：ETA预测指标 5：图库 6：ETA表格 func GetEdbChartAdminList(source int) (list []int, err error) { switch source { case 1: //手工数据指标 list, err = models.GetEdbInfoAdminList() if err != nil { return } case 2: //钢联化工数据库 list, err = data_manage.GetMysteelChemicalIndexAdminList() if err != nil { return } case 3: //ETA指标库 list, err = data_manage.GetEdbInfoAdminList(0) if err != nil { return } case 4: //ETA预测指标 list, err = data_manage.GetEdbInfoAdminList(1) if err != nil { return } case 5: //图库 list, err = data_manage.GetChartInfoAdminList() if err != nil { return } default: return } return } // EdbInfoReplace 替换指标v2版本（更换时间：2022年01月05日15:33:42） func EdbInfoReplace(oldEdbInfo, newEdbInfo *data_manage.EdbInfo, sysAdminId int, sysAdminRealName string) (replaceChartTotal, replaceCalculateTotal int, err error) { defer func() { if err != nil { fmt.Println("ERR:", err.Error()) } }() // 替换关联表中的source_edb_info_id relationEdbInfoIdList, replaceChartTotal, replaceCalculateTotal, err := data_manage.ReplaceChartEdb(oldEdbInfo, newEdbInfo) if err != nil { return } //加入到缓存队列中处理 go cache.AddReplaceEdbInfo(oldEdbInfo, newEdbInfo) // 更新所有的关联指标 err, _ = EdbInfoRefreshAllFromBaseV3(relationEdbInfoIdList, true, true, true) return } func GetTerminalFromBaseIndex(source int, edbCode string) (terminalCode string, sourceIndexName string, err error) { tbName := GetBaseIndexTableName(source) if tbName != "" { var item data_manage.BaseIndexTerminalCode item, err = data_manage.GetBaseIndexTerminalCode(edbCode, tbName) if err != nil { err = fmt.Errorf("查询数据源指标信息失败， Err： %s", err) return } terminalCode = item.TerminalCode sourceIndexName = item.IndexName } return } func GetBaseIndexTableName(source int) (tableName string) { switch source { case utils.DATA_SOURCE_YS: tableName = "base_from_smm_index" case utils.DATA_SOURCE_MYSTEEL_CHEMICAL: tableName = "base_from_mysteel_chemical_index" case utils.DATA_SOURCE_SCI: tableName = "base_from_sci_index" case utils.DATA_SOURCE_BAIINFO: tableName = "base_from_baiinfo_index" default: } return } // EdbInfoWsdAdd 添加指标到指标库 func EdbInfoWsdAdd(item *data_manage.EdbInfo) (edbInfo *data_manage.EdbInfo, err error, errMsg string, isSendEmail bool) { isSendEmail = true //判断指标名称是否存在 var condition string var pars []interface{} condition += " AND edb_name=? " pars = append(pars, item.EdbName) count, err := data_manage.GetEdbInfoCountByCondition(condition, pars) if err != nil { errMsg = "判断指标名称是否存在失败" err = errors.New("判断指标名称是否存在失败,Err:" + err.Error()) return } if count > 0 { errMsg = "指标名称已存在，请重新填写" err = errors.New(errMsg) isSendEmail = false return } source := item.Source edbInfo = new(data_manage.EdbInfo) edbInfo.Source = source sourceNameMap := map[int]string{ utils.DATA_SOURCE_THS: "同花顺", utils.DATA_SOURCE_WIND: "wind", utils.DATA_SOURCE_PB: "彭博", utils.DATA_SOURCE_PB_FINANCE: "彭博财务", utils.DATA_SOURCE_MANUAL: "手工数据", utils.DATA_SOURCE_LZ: "隆众", utils.DATA_SOURCE_YS: "SMM", utils.DATA_SOURCE_GL: "钢联", utils.DATA_SOURCE_ZZ: "郑商所", utils.DATA_SOURCE_DL: "大商所", utils.DATA_SOURCE_SH: "上期所", utils.DATA_SOURCE_CFFEX: "中金所", utils.DATA_SOURCE_SHFE: "上期能源", utils.DATA_SOURCE_GIE: "欧洲天然气", utils.DATA_SOURCE_LT: "路透", utils.DATA_SOURCE_COAL: "中国煤炭市场网", utils.DATA_SOURCE_GOOGLE_TRAVEL: "our world in data", utils.DATA_SOURCE_MYSTEEL_CHEMICAL: "钢联", utils.DATA_SOURCE_EIA_STEO: "EIA STEO报告", utils.DATA_SOURCE_COM_TRADE: "UN", utils.DATA_SOURCE_SCI: "SCI", utils.DATA_SOURCE_BAIINFO: "BAIINFO", utils.DATA_SOURCE_STOCK_PLANT: "存量装置", utils.DATA_SOURCE_NATIONAL_STATISTICS: "国家统计局", utils.DATA_SOURCE_FUBAO: "富宝数据", } sourceName, ok := sourceNameMap[source] if !ok { edbSource := data_manage.EdbSourceIdMap[source] if edbSource != nil { sourceName = edbSource.SourceName } if sourceName == "" { errMsg = "指标来源异常" err = errors.New(errMsg) return } } edbInfo.SourceName = sourceName edbType := 1 //基础指标 if source == utils.DATA_SOURCE_STOCK_PLANT { edbType = 2 //计算指标 } //获取该层级下最大的排序数 maxSort, err := GetEdbClassifyMaxSort(item.ClassifyId, 0) if err != nil { errMsg = "获取失败" err = errors.New("查询排序信息失败，Err:" + err.Error()) return } edbInfo.EdbCode = item.EdbCode edbInfo.EdbName = item.EdbName edbInfo.EdbNameSource = item.EdbName edbInfo.EdbNameEn = item.EdbNameEn edbInfo.Frequency = item.Frequency edbInfo.Unit = item.Unit edbInfo.UnitEn = item.UnitEn edbInfo.ClassifyId = item.ClassifyId edbInfo.SysUserId = item.SysUserId edbInfo.SysUserRealName = item.SysUserRealName edbInfo.CreateTime = time.Now() edbInfo.ModifyTime = time.Now() edbInfo.ServerUrl = item.ServerUrl edbInfo.Sort = maxSort + 1 edbInfo.DataDateType = `交易日` timestamp := strconv.FormatInt(time.Now().UnixNano(), 10) edbInfo.UniqueCode = utils.MD5(utils.DATA_PREFIX + "_" + timestamp) edbInfo.EdbType = edbType edbInfo.SubSource = item.SubSource edbInfo.SubSourceName = "日期序列" edbInfo.IndicatorCode = item.IndicatorCode edbInfo.StockCode = item.StockCode edbInfo.TerminalCode = item.TerminalCode edbInfo.Extra = item.Extra edbInfoId, err := data_manage.AddEdbInfo(edbInfo) if err != nil { errMsg = "保存失败" err = errors.New("保存失败，Err:" + err.Error()) return } edbInfo.EdbInfoId = int(edbInfoId) //保存数据 err = data_manage.ModifyEdbInfoWindWsdDataStatus(source, item.SubSource, edbInfoId, item.EdbCode) if err != nil { errMsg = "保存失败" err = errors.New("修改数据对应指标ID失败，Err:" + err.Error()) return } maxAndMinItem, _ := data_manage.GetEdbInfoWsdMaxAndMinInfo(source, item.SubSource, item.EdbCode) if maxAndMinItem != nil { err = data_manage.ModifyEdbInfoMaxAndMinInfo(int(edbInfoId), maxAndMinItem) } //添加es AddOrEditEdbInfoToEs(int(edbInfoId)) return } // BatchRefreshEdbByEdbIds 批量刷新指标 func BatchRefreshEdbByEdbIds(edbIdList []int, redisKey string, refreshKeys []string) (syncing bool, err error) { if len(edbIdList) <= 0 { return } // 设置刷新缓存 if redisKey != `` { // 设置最多10分钟缓存 utils.Rc.SetNX(redisKey, 1, time.Minute*10) } // 获取需要刷新的指标列表 newBaseEdbInfoArr, newBasePredictEdbInfoArr, newCalculateMap, newPredictCalculateMap, calculateArr, predictCalculateArr := getEdbInfoIdList(edbIdList) // 需要刷新的指标数量 totalEdbInfo := len(newBaseEdbInfoArr) + len(calculateArr) + len(predictCalculateArr) // 关联指标过多的时候, 异步刷新 if totalEdbInfo > 20 { syncing = true //go func() { // defer func() { // if err != nil { // alarm_msg.SendAlarmMsg("BatchRefreshEdbByEdbIds, ErrMsg: "+err.Error(), 3) // } // }() // // err = edbInfoRefreshAll(false, newBaseEdbInfoArr, newBasePredictEdbInfoArr, newCalculateMap, newPredictCalculateMap, calculateArr, predictCalculateArr) // // if redisKey != `` { // _ = utils.Rc.Delete(redisKey) // } // if len(refreshKeys) > 0 { // for _, v := range refreshKeys { // _ = utils.Rc.Delete(v) // } // } //}() // 加入队列刷新 var refreshItem EdbQueueRefreshReq refreshItem.RefreshKey = redisKey refreshItem.ItemRefreshKeys = refreshKeys refreshItem.BaseEdbInfoArr = newBaseEdbInfoArr refreshItem.BasePredictEdbInfoArr = newBasePredictEdbInfoArr refreshItem.CalculateMap = newCalculateMap refreshItem.PredictCalculateMap = newPredictCalculateMap refreshItem.CalculateArr = calculateArr refreshItem.PredictCalculateArr = predictCalculateArr go PushEdb2Refresh(refreshItem) return } // 同步刷新 err = edbInfoRefreshAll(false, newBaseEdbInfoArr, newBasePredictEdbInfoArr, newCalculateMap, newPredictCalculateMap, calculateArr, predictCalculateArr) if redisKey != `` { _ = utils.Rc.Delete(redisKey) } if len(refreshKeys) > 0 { for _, v := range refreshKeys { _ = utils.Rc.Delete(v) } } return } // EdbInfoSmmApiAdd 添加指标到指标库 func EdbInfoSmmApiAdd(item *data_manage.EdbInfo) (edbInfo *data_manage.EdbInfo, err error, errMsg string, isSendEmail bool) { isSendEmail = true //判断指标名称是否存在 var condition string var pars []interface{} condition += " AND edb_name=? " pars = append(pars, item.EdbName) count, err := data_manage.GetEdbInfoCountByCondition(condition, pars) if err != nil { errMsg = "判断指标名称是否存在失败" err = errors.New("判断指标名称是否存在失败,Err:" + err.Error()) return } if count > 0 { errMsg = "指标名称已存在，请重新填写" err = errors.New(errMsg) isSendEmail = false return } source := item.Source edbInfo = new(data_manage.EdbInfo) edbInfo.Source = source sourceNameMap := map[int]string{ utils.DATA_SOURCE_THS: "同花顺", utils.DATA_SOURCE_WIND: "wind", utils.DATA_SOURCE_PB: "彭博", utils.DATA_SOURCE_PB_FINANCE: "彭博财务", utils.DATA_SOURCE_MANUAL: "手工数据", utils.DATA_SOURCE_LZ: "隆众", utils.DATA_SOURCE_YS: "SMM", utils.DATA_SOURCE_GL: "钢联", utils.DATA_SOURCE_ZZ: "郑商所", utils.DATA_SOURCE_DL: "大商所", utils.DATA_SOURCE_SH: "上期所", utils.DATA_SOURCE_CFFEX: "中金所", utils.DATA_SOURCE_SHFE: "上期能源", utils.DATA_SOURCE_GIE: "欧洲天然气", utils.DATA_SOURCE_LT: "路透", utils.DATA_SOURCE_COAL: "中国煤炭市场网", utils.DATA_SOURCE_GOOGLE_TRAVEL: "our world in data", utils.DATA_SOURCE_MYSTEEL_CHEMICAL: "钢联", utils.DATA_SOURCE_EIA_STEO: "EIA STEO报告", utils.DATA_SOURCE_COM_TRADE: "UN", utils.DATA_SOURCE_SCI: "SCI", utils.DATA_SOURCE_BAIINFO: "BAIINFO", utils.DATA_SOURCE_STOCK_PLANT: "存量装置", utils.DATA_SOURCE_NATIONAL_STATISTICS: "国家统计局", utils.DATA_SOURCE_FUBAO: "富宝数据", } sourceName, ok := sourceNameMap[source] if !ok { edbSource := data_manage.EdbSourceIdMap[source] if edbSource != nil { sourceName = edbSource.SourceName } if sourceName == "" { errMsg = "指标来源异常" err = errors.New(errMsg) return } } edbInfo.SourceName = sourceName edbType := 1 //基础指标 if source == utils.DATA_SOURCE_STOCK_PLANT { edbType = 2 //计算指标 } //从缓存中获取 serverUrl := `` if edbInfo.Source == utils.DATA_SOURCE_WIND { windCacheKey := utils.CACHE_WIND_URL + ":" + item.EdbCode serverUrl, _ = utils.Rc.RedisString(windCacheKey) if serverUrl == `` { if len(utils.Hz_Data_WIND_Url_List) >= 1 { serverUrl = utils.Hz_Data_WIND_Url_List[len(utils.Hz_Data_WIND_Url_List)-1] //默认是最后一个服务器地址 } } } //获取该层级下最大的排序数 maxSort, err := GetEdbClassifyMaxSort(item.ClassifyId, 0) if err != nil { errMsg = "获取失败" err = errors.New("查询排序信息失败，Err:" + err.Error()) return } edbInfo.EdbCode = item.EdbCode edbInfo.EdbName = item.EdbName edbInfo.EdbNameSource = item.EdbName edbInfo.Frequency = item.Frequency edbInfo.Unit = item.Unit edbInfo.ClassifyId = item.ClassifyId edbInfo.SysUserId = item.SysUserId edbInfo.SysUserRealName = item.SysUserRealName edbInfo.CreateTime = time.Now() edbInfo.ModifyTime = time.Now() edbInfo.ServerUrl = serverUrl edbInfo.Sort = maxSort + 1 edbInfo.DataDateType = `交易日` timestamp := strconv.FormatInt(time.Now().UnixNano(), 10) edbInfo.UniqueCode = utils.MD5(utils.DATA_PREFIX + "_" + timestamp) edbInfo.EdbType = edbType edbInfo.SubSource = item.SubSource edbInfo.SubSourceName = "" edbInfo.IndicatorCode = item.IndicatorCode edbInfo.StockCode = item.StockCode edbInfoId, err := data_manage.AddEdbInfo(edbInfo) if err != nil { errMsg = "保存失败" err = errors.New("保存失败，Err:" + err.Error()) return } edbInfo.EdbInfoId = int(edbInfoId) //保存数据 err = data_manage.ModifyEdbInfoWindWsdDataStatus(source, item.SubSource, edbInfoId, item.EdbCode) if err != nil { errMsg = "保存失败" err = errors.New("修改数据对应指标ID失败，Err:" + err.Error()) return } maxAndMinItem, _ := data_manage.GetEdbInfoWsdMaxAndMinInfo(source, item.SubSource, item.EdbCode) if maxAndMinItem != nil { err = data_manage.ModifyEdbInfoMaxAndMinInfo(int(edbInfoId), maxAndMinItem) } //添加es AddOrEditEdbInfoToEs(int(edbInfoId)) return } // GetEdbTerminal 获取终端信息 func GetEdbTerminal(source int, oldTerminalCode string) (edbTerminal *data_manage.EdbTerminal, err error) { if source == utils.DATA_SOURCE_WIND && oldTerminalCode == "" { tmpConfig := new(data_manage.EdbConfig) tmpConfig, err = data_manage.GetConfigDetailByCode("wind_terminal_code") if err != nil && err.Error() != utils.ErrNoRow() { err = errors.New("获取配置的wind地址失败，err:" + err.Error()) return } err = nil if tmpConfig != nil { oldTerminalCode = tmpConfig.ConfigValue //return } } if oldTerminalCode != "" { edbTerminal, err = data_manage.GetEdbTerminalByCode(oldTerminalCode) if err != nil { return } return } // 配置中没有的话，那么就从分组获取, list, err := data_manage.GetEdbCountGroupByTerminal(source) if err != nil { return } //windUrlNum := 0 //获取对应的配置url terminalNumMap := make(map[string]int) for _, v := range list { terminalNumMap[v.TerminalCode] = v.Total } terminalList, err := data_manage.GetEdbTerminalListBySource(source) if err != nil { return } if len(terminalList) == 0 { err = errors.New("终端地址未配置") return } num := 0 for _, v := range terminalList { tmpNum := terminalNumMap[v.TerminalCode] if edbTerminal == nil { edbTerminal = v num = tmpNum } else if tmpNum < num { edbTerminal = v num = tmpNum } } if edbTerminal == nil { err = errors.New("获取配置的终端地址失败") return } /*if edbTerminal != nil { windUrl = edbTerminal.ServerUrl terminalCode = edbTerminal.TerminalCode }*/ return } func GetEdbTerminalCodeBySource(source int, edbCode, stockCode string) (terminalCode, serverUrl, sourceIndexName string, err error) { var e error if stockCode != "" { terminalCodeCacheKey := utils.CACHE_EDB_TERMINAL_CODE_URL + stockCode terminalCode, e = utils.Rc.RedisString(terminalCodeCacheKey) if e != nil { utils.FileLog.Info(fmt.Sprintf("从缓存中获取数据源终端信息失败, Err: %s", e)) } terminalInfo, e := GetEdbTerminal(source, terminalCode) if e != nil { err = errors.New("获取可以使用的终端失败，Err:" + e.Error()) return } serverUrl = terminalInfo.ServerUrl terminalCode = terminalInfo.TerminalCode return } // 获取终端信息 terminalCodeCacheKey := utils.CACHE_EDB_TERMINAL_CODE_URL + edbCode terminalCode, tE := utils.Rc.RedisString(terminalCodeCacheKey) if tE != nil { utils.FileLog.Info(fmt.Sprintf("从缓存中获取数据源终端信息失败, Err: %s", tE)) } var terminalCodeOrigin string terminalCodeOrigin, sourceIndexName, tE = GetTerminalFromBaseIndex(source, edbCode) if tE != nil { utils.FileLog.Info(fmt.Sprintf("获取数据源终端信息失败, Err: %s", tE)) } if terminalCodeOrigin != "" { terminalCode = terminalCodeOrigin } if source == utils.DATA_SOURCE_WIND || source == utils.DATA_SOURCE_THS { terminalInfo, e := GetEdbTerminal(source, terminalCode) if e != nil { err = errors.New("获取可以使用的终端失败，Err:" + e.Error()) return } serverUrl = terminalInfo.ServerUrl terminalCode = terminalInfo.TerminalCode } return } // handleByAddEdbInfo // @Description: 添加指标后的处理操作 // @author: Roc // @datetime 2024-07-22 13:06:36 // @param edbInfo *data_manage.EdbInfo func handleByAddEdbInfo(edbInfo *data_manage.EdbInfo) { // 更新钢联化工状态为启用 if edbInfo.Source == utils.DATA_SOURCE_MYSTEEL_CHEMICAL { // 启动钢联的刷新 _ = data_manage.UpdateMysteelChemicalRefreshStatus(edbInfo.EdbCode, 0) } // 如果是手工数据，那么需要标记手工数据为已加入指标库 if edbInfo.Source == utils.DATA_SOURCE_MANUAL { _ = models.UpdateManualIsJoinEdbStatus(edbInfo.EdbCode, 1) } } // handleByDelEdbInfo // @Description: 删除指标后的处理操作 // @author: Roc // @datetime 2024-07-22 13:06:36 // @param edbInfo *data_manage.EdbInfo func handleByDelEdbInfo(edbInfo *data_manage.EdbInfo) { // 如果删除的指标是自定义分析的来源，那么还需要删除指标与excel的关系 if edbInfo.Source == utils.DATA_SOURCE_CALCULATE_ZDYFX { err := excel.DeleteCustomAnalysisExcelEdbMappingByEdbInfoId(edbInfo.EdbInfoId) if err != nil { utils.FileLog.Error(fmt.Sprintf("删除指标时，需要删除与自定义分析的关系失败，指标ID：%d，Err:%s", edbInfo.EdbInfoId, err.Error()), 3) } } // 如果是手工数据，那么需要标记手工数据为未加入指标库 if edbInfo.Source == utils.DATA_SOURCE_MANUAL { _ = models.UpdateManualIsJoinEdbStatus(edbInfo.EdbCode, 0) } } // GetIsSupplierStop // @Description: 获取是否供应商停更 // @author: Roc // @datetime 2024-08-27 10:28:19 // @param source int // @param edbCode string // @return isSupplierStop int func GetIsSupplierStop(source int, edbCode string) (isSupplierStop int) { switch source { case utils.DATA_SOURCE_MYSTEEL_CHEMICAL: item, err := data_manage.GetBaseFromMysteelChemicalIndexByCode(edbCode) if err != nil { return } isSupplierStop = item.IsSupplierStop } return } //IsSupplierStop int `description:"是否供应商停更:1:停更,0:未停更"`