package data import ( "errors" "eta_gn/eta_chart_lib/models" "eta_gn/eta_chart_lib/models/data_manage" "eta_gn/eta_chart_lib/services/alarm_msg" "eta_gn/eta_chart_lib/utils" "fmt" "math" "sort" "time" "github.com/shopspring/decimal" ) func EdbInfoRefreshAllFromBaseBak(edbInfoIdList []int, refreshAll bool) (err error, errmsg string) { defer func() { if err != nil { fmt.Println("EdbInfoRefreshAllFromBaseV2 Err:" + err.Error() + ";errmsg:" + errmsg) go alarm_msg.SendAlarmMsg("EdbInfoRefreshFromBaseV2,Err"+err.Error()+";errMsg:"+errmsg, 3) } }() newBaseEdbInfoArr, newBasePredictEdbInfoArr, newCalculateMap, newPredictCalculateMap, calculateArr, predictCalculateArr, err, errmsg := getRefreshEdbInfoListByIds(edbInfoIdList) if err != nil { return } var startDate string for _, bv := range newBaseEdbInfoArr { startDate = utils.BaseEdbRefreshStartDate if !refreshAll { //不是刷新所有数据,用结束时间减去对应期数作为起始日期去刷新 sDate := bv.EndDate if sDate == `` || sDate == `0000-00-00` { sDate = utils.BaseEdbRefreshStartDate } sTime, tmpErr := time.ParseInLocation(utils.FormatDate, sDate, time.Local) if tmpErr != nil { err = tmpErr errmsg = err.Error() return } frequency := bv.Frequency var limitDay int switch frequency { case "日度": limitDay = utils.DATA_REFRESH case "周度": limitDay = utils.DATA_REFRESH * 7 case "月度": limitDay = utils.DATA_REFRESH * 30 case "季度": limitDay = utils.DATA_REFRESH * 90 case "年度": limitDay = utils.DATA_REFRESH * 365 default: limitDay = utils.DATA_REFRESH } startDate = sTime.AddDate(0, 0, -limitDay).Format(utils.FormatDate) } result, tmpErr := RefreshEdbData(bv.EdbInfoId, bv.Source, bv.SubSource, bv.EdbCode, startDate) if tmpErr != nil { err = tmpErr fmt.Println(bv.EdbInfoId, "RefreshBaseEdbData err", time.Now()) errmsg = "RefreshBaseEdbData Err:" + tmpErr.Error() return } if result.Ret != 200 { fmt.Println(bv.EdbInfoId, "RefreshBaseEdbData err;msg:", result.Msg, ";errMsg:", result.ErrMsg) errmsg = fmt.Sprint(bv.EdbInfoId, "RefreshBaseEdbData err;msg:", result.Msg, ";errMsg:", result.ErrMsg) err = fmt.Errorf("刷新失败, err:", errmsg) return } fmt.Println("end newBaseEdbInfoArr:", bv, time.Now()) } for _, v := range calculateArr { edbInfo := newCalculateMap[v] if edbInfo == nil { return } startDate = edbInfo.StartDate source := edbInfo.Source if startDate == "" || startDate == "0000-00-00" { //如果没有开始日期,说明还没有计算出来数据,那么就往前面推40年吧(也意味着重新计算了) startDate = time.Now().AddDate(-40, 0, 0).Format(utils.FormatDate) } else { if source == utils.DATA_SOURCE_CALCULATE { startDate = `` if refreshAll { //刷新所有数据,用开始时间作为起始日期去刷新 startDate = edbInfo.StartDate } else { sTime, tmpErr := time.Parse(utils.FormatDate, edbInfo.EndDate) if tmpErr != nil { err = tmpErr errmsg = tmpErr.Error() return } frequency := edbInfo.Frequency var limitDay int switch frequency { case "日度": limitDay = utils.DATA_REFRESH case "周度": limitDay = utils.DATA_REFRESH * 7 case "月度": limitDay = utils.DATA_REFRESH * 30 case "季度": limitDay = utils.DATA_REFRESH * 90 case "年度": limitDay = utils.DATA_REFRESH * 365 default: limitDay = utils.DATA_REFRESH } startDate = sTime.AddDate(0, 0, -limitDay).Format(utils.FormatDate) } } } result, tmpErr := RefreshEdbCalculateData(edbInfo.EdbInfoId, edbInfo.EdbCode, startDate) if tmpErr != nil { err = tmpErr fmt.Println(v, "RefreshEdbCalculateData err", time.Now()) errmsg = "RefreshEdbCalculateData Err:" + tmpErr.Error() return } if result.Ret != 200 { fmt.Println(v, "RefreshEdbCalculateData err;msg:", result.Msg, ";errMsg:", result.ErrMsg) errmsg = fmt.Sprint(v, "RefreshEdbCalculateData err;msg:", result.Msg, ";errMsg:", result.ErrMsg) err = fmt.Errorf("刷新失败") return } } for _, v := range predictCalculateArr { edbInfo := newPredictCalculateMap[v] if edbInfo == nil { return } startDate = edbInfo.StartDate source := edbInfo.Source if startDate == "" || startDate == "0000-00-00" { //如果没有开始日期,说明还没有计算出来数据,那么就往前面推40年吧(也意味着重新计算了) startDate = time.Now().AddDate(-40, 0, 0).Format(utils.FormatDate) } else { if source == utils.DATA_SOURCE_PREDICT_CALCULATE { startDate = `` if refreshAll { //刷新所有数据,用开始时间作为起始日期去刷新 startDate = edbInfo.StartDate } else { sTime, tmpErr := time.Parse(utils.FormatDate, edbInfo.EndDate) if tmpErr != nil { err = tmpErr errmsg = tmpErr.Error() return } frequency := edbInfo.Frequency var limitDay int switch frequency { case "日度": limitDay = utils.DATA_REFRESH case "周度": limitDay = utils.DATA_REFRESH * 7 case "月度": limitDay = utils.DATA_REFRESH * 30 case "季度": limitDay = utils.DATA_REFRESH * 90 case "年度": limitDay = utils.DATA_REFRESH * 365 default: limitDay = utils.DATA_REFRESH } startDate = sTime.AddDate(0, 0, -limitDay).Format(utils.FormatDate) } } } result, tmpErr := RefreshPredictEdbCalculateData(edbInfo.EdbInfoId, edbInfo.EdbCode, startDate) if tmpErr != nil { err = tmpErr fmt.Println(v, "RefreshPredictEdbCalculateData err", time.Now()) errmsg = "RefreshPredictEdbCalculateData Err:" + tmpErr.Error() return } if result.Ret != 200 { fmt.Println(v, "RefreshPredictEdbCalculateData err;msg:", result.Msg, ";errMsg:", result.ErrMsg) errmsg = fmt.Sprint(v, "RefreshPredictEdbCalculateData err;msg:", result.Msg, ";errMsg:", result.ErrMsg) err = fmt.Errorf("刷新失败") return } } { predictEdbInfoIdList := make([]int, 0) for _, v := range newBasePredictEdbInfoArr { predictEdbInfoIdList = append(predictEdbInfoIdList, v.EdbInfoId) } fmt.Println("predictEdbConfCalculateMappingDetailList:", predictEdbInfoIdList) predictEdbConfCalculateMappingDetailList, tmpErr := data_manage.GetPredictEdbConfCalculateMappingDetailListByEdbInfoId(predictEdbInfoIdList) if tmpErr != nil { err = tmpErr errmsg = fmt.Sprint("更新动态环差值时,获取数据失败:" + err.Error()) return } for _, bv := range predictEdbConfCalculateMappingDetailList { result, tmpErr := RefreshEdbData(bv.EdbInfoId, utils.DATA_SOURCE_PREDICT, utils.DATA_SUB_SOURCE_EDB, bv.EdbCode, "") if tmpErr != nil { err = tmpErr fmt.Println(bv.EdbInfoId, "RefreshBasePredictEdbData err", time.Now()) errmsg = "RefreshBasePredictEdbData Err:" + err.Error() return } if result.Ret != 200 { fmt.Println(bv.EdbInfoId, "RefreshBasePredictEdbData err;msg:", result.Msg, ";errMsg:", result.ErrMsg) errmsg = fmt.Sprint(bv.EdbInfoId, "RefreshBasePredictEdbData err;msg:", result.Msg, ";errMsg:", result.ErrMsg) err = errors.New("刷新失败:" + errmsg) return } fmt.Println("end predictEdbConfCalculateMappingDetailList:", bv, time.Now()) } } for _, v := range predictCalculateArr { edbInfo := newPredictCalculateMap[v] if edbInfo == nil { return } startDate = edbInfo.StartDate source := edbInfo.Source if startDate == "" || startDate == "0000-00-00" { //如果没有开始日期,说明还没有计算出来数据,那么就往前面推40年吧(也意味着重新计算了) startDate = time.Now().AddDate(-40, 0, 0).Format(utils.FormatDate) } else { if source == utils.DATA_SOURCE_PREDICT_CALCULATE { startDate = `` if refreshAll { //刷新所有数据,用开始时间作为起始日期去刷新 startDate = edbInfo.StartDate } else { sTime, tmpErr := time.Parse(utils.FormatDate, edbInfo.EndDate) if tmpErr != nil { err = tmpErr errmsg = tmpErr.Error() return } frequency := edbInfo.Frequency var limitDay int switch frequency { case "日度": limitDay = utils.DATA_REFRESH case "周度": limitDay = utils.DATA_REFRESH * 7 case "月度": limitDay = utils.DATA_REFRESH * 30 case "季度": limitDay = utils.DATA_REFRESH * 90 case "年度": limitDay = utils.DATA_REFRESH * 365 default: limitDay = utils.DATA_REFRESH } startDate = sTime.AddDate(0, 0, -limitDay).Format(utils.FormatDate) } } } result, tmpErr := RefreshPredictEdbCalculateData(edbInfo.EdbInfoId, edbInfo.EdbCode, startDate) if tmpErr != nil { err = tmpErr fmt.Println(v, "RefreshPredictEdbCalculateData err", time.Now()) errmsg = "RefreshPredictEdbCalculateData Err:" + tmpErr.Error() return } if result.Ret != 200 { fmt.Println(v, "RefreshPredictEdbCalculateData err;msg:", result.Msg, ";errMsg:", result.ErrMsg) errmsg = fmt.Sprint(v, "RefreshPredictEdbCalculateData err;msg:", result.Msg, ";errMsg:", result.ErrMsg) err = fmt.Errorf("刷新失败") return } } return } func getRefreshEdbInfoListByIds(edbInfoIdList []int) (newBaseEdbInfoArr, newBasePredictEdbInfoArr []*data_manage.EdbInfo, newCalculateMap, newPredictCalculateMap map[int]*data_manage.EdbInfo, calculateArr, predictCalculateArr []int, err error, errMsg string) { calculateList, err := data_manage.GetEdbInfoAllCalculateByEdbInfoIdList(edbInfoIdList) if err != nil && !utils.IsErrNoRow(err) { err = errors.New("GetEdbInfoAllCalculate Err:" + err.Error()) return } edbInfoList, err := data_manage.GetEdbInfoByIdList(edbInfoIdList) if err != nil { err = errors.New("GetEdbInfoAllCalGetEdbInfoByIdr:" + err.Error()) return } baseEdbInfoArr := make([]*data_manage.EdbInfo, 0) // 基础普通指标 calculateInfoArr := make([]*data_manage.EdbInfo, 0) //基础运算指标 basePredictEdbInfoArr := make([]*data_manage.EdbInfo, 0) // 预测普通指标 predictCalculateInfoArr := make([]*data_manage.EdbInfo, 0) //预测运算指标 for _, baseEdbInfo := range edbInfoList { if baseEdbInfo.EdbInfoType == 0 { //普通指标 switch baseEdbInfo.EdbType { case 1: //基础指标 baseEdbInfoArr = append(baseEdbInfoArr, baseEdbInfo) case 2: //计算指标 calculateList = append(calculateList, baseEdbInfo) } } else { //预测指标 switch baseEdbInfo.EdbType { case 1: //基础指标 basePredictEdbInfoArr = append(basePredictEdbInfoArr, baseEdbInfo) case 2: //计算指标 predictCalculateInfoArr = append(predictCalculateInfoArr, baseEdbInfo) } } } for _, v := range calculateList { getBaseEdbInfoArr, getCalculateInfoArr, tmpErr := data_manage.GetRefreshEdbInfoFromBase(v.EdbInfoId, v.Source) if tmpErr != nil { err = tmpErr fmt.Println("GetRefreshEdbInfoFromBase err:" + err.Error()) errMsg = "获取需要刷新的指标失败:Err:" + err.Error() return } for _, tmpEdbInfo := range getBaseEdbInfoArr { switch tmpEdbInfo.EdbInfoType { case 0: //普通基础指标 baseEdbInfoArr = append(baseEdbInfoArr, tmpEdbInfo) case 1: //预测基础指标 basePredictEdbInfoArr = append(basePredictEdbInfoArr, tmpEdbInfo) } } for _, tmpEdbInfo := range getCalculateInfoArr { switch tmpEdbInfo.EdbInfoType { case 0: //普通运算指标 calculateInfoArr = append(calculateInfoArr, tmpEdbInfo) case 1: //预测运算指标 if tmpEdbInfo.EdbType == 1 { //预测普通指标 basePredictEdbInfoArr = append(basePredictEdbInfoArr, tmpEdbInfo) } else { predictCalculateInfoArr = append(predictCalculateInfoArr, tmpEdbInfo) } } } switch v.EdbInfoType { case 0: //普通运算指标 calculateInfoArr = append(calculateInfoArr, v) case 1: //预测运算指标 if v.EdbType == 1 { //预测普通指标 basePredictEdbInfoArr = append(basePredictEdbInfoArr, v) } else { predictCalculateInfoArr = append(predictCalculateInfoArr, v) } } } for _, v := range predictCalculateInfoArr { getBaseEdbInfoArr, getCalculateInfoArr, tmpErr := data_manage.GetRefreshEdbInfoFromBase(v.EdbInfoId, v.Source) if tmpErr != nil { err = tmpErr fmt.Println("GetRefreshEdbInfoFromBase err:" + err.Error()) errMsg = "获取需要刷新的指标失败:Err:" + err.Error() return } for _, tmpEdbInfo := range getBaseEdbInfoArr { switch tmpEdbInfo.EdbInfoType { case 0: //普通基础指标 baseEdbInfoArr = append(baseEdbInfoArr, tmpEdbInfo) case 1: //预测基础指标 if tmpEdbInfo.EdbType == 1 { //预测普通指标 basePredictEdbInfoArr = append(basePredictEdbInfoArr, tmpEdbInfo) } else { predictCalculateInfoArr = append(predictCalculateInfoArr, tmpEdbInfo) } } } for _, tmpEdbInfo := range getCalculateInfoArr { switch tmpEdbInfo.EdbInfoType { case 0: //普通运算指标 calculateInfoArr = append(calculateInfoArr, tmpEdbInfo) case 1: //预测运算指标 if tmpEdbInfo.EdbType == 1 { //预测普通指标 basePredictEdbInfoArr = append(basePredictEdbInfoArr, tmpEdbInfo) } else { predictCalculateInfoArr = append(predictCalculateInfoArr, tmpEdbInfo) } } } switch v.EdbInfoType { case 0: //普通运算指标 calculateInfoArr = append(calculateInfoArr, v) case 1: //预测运算指标 if v.EdbType == 1 { //预测普通指标 basePredictEdbInfoArr = append(basePredictEdbInfoArr, v) } else { predictCalculateInfoArr = append(predictCalculateInfoArr, v) } } } newCalculateMap = make(map[int]*data_manage.EdbInfo) for _, v := range calculateInfoArr { if _, ok := newCalculateMap[v.EdbInfoId]; !ok { calculateArr = append(calculateArr, v.EdbInfoId) } newCalculateMap[v.EdbInfoId] = v } if len(basePredictEdbInfoArr) > 0 { basePredictEdbInfoIdList := make([]int, 0) for _, v := range basePredictEdbInfoArr { basePredictEdbInfoIdList = append(basePredictEdbInfoIdList, v.EdbInfoId) } baseEdbInfoList, tmpErr := data_manage.GetPredictEdbInfoAllCalculate(basePredictEdbInfoIdList) if tmpErr != nil { err = tmpErr return } tmpCalculateList := make([]*data_manage.EdbInfo, 0) for _, v := range baseEdbInfoList { switch v.EdbType { case 1: baseEdbInfoArr = append(baseEdbInfoArr, v) case 2: if _, ok := newCalculateMap[v.EdbInfoId]; !ok { tmpCalculateList = append(tmpCalculateList, v) } } } for _, v := range tmpCalculateList { getBaseEdbInfoArr, getCalculateInfoArr, tmpErr := data_manage.GetRefreshEdbInfoFromBase(v.EdbInfoId, v.Source) if tmpErr != nil { err = tmpErr fmt.Println("GetRefreshEdbInfoFromBase err:" + err.Error()) errMsg = "获取需要刷新的指标失败:Err:" + err.Error() return } for _, tmpEdbInfo := range getBaseEdbInfoArr { switch tmpEdbInfo.EdbInfoType { case 0: //普通基础指标 baseEdbInfoArr = append(baseEdbInfoArr, tmpEdbInfo) case 1: //预测基础指标 basePredictEdbInfoArr = append(basePredictEdbInfoArr, tmpEdbInfo) } } for _, tmpEdbInfo := range getCalculateInfoArr { switch tmpEdbInfo.EdbInfoType { case 0: //普通运算指标 calculateInfoArr = append(calculateInfoArr, tmpEdbInfo) case 1: //预测运算指标 if tmpEdbInfo.EdbType == 1 { //预测普通指标 basePredictEdbInfoArr = append(basePredictEdbInfoArr, tmpEdbInfo) } else { predictCalculateInfoArr = append(predictCalculateInfoArr, tmpEdbInfo) } } } switch v.EdbInfoType { case 0: //普通运算指标 calculateInfoArr = append(calculateInfoArr, v) case 1: //预测运算指标 if v.EdbType == 1 { //预测普通指标 basePredictEdbInfoArr = append(basePredictEdbInfoArr, v) } else { predictCalculateInfoArr = append(predictCalculateInfoArr, v) } } } for _, v := range calculateInfoArr { if _, ok := newCalculateMap[v.EdbInfoId]; !ok { calculateArr = append(calculateArr, v.EdbInfoId) } newCalculateMap[v.EdbInfoId] = v } } newBaseEdbInfoArr = make([]*data_manage.EdbInfo, 0) baseMap := make(map[int]int) for _, v := range baseEdbInfoArr { if _, ok := baseMap[v.EdbInfoId]; !ok { newBaseEdbInfoArr = append(newBaseEdbInfoArr, v) } baseMap[v.EdbInfoId] = v.EdbInfoId } sort.Ints(calculateArr) newBasePredictEdbInfoArr = make([]*data_manage.EdbInfo, 0) basePredictMap := make(map[int]int) for _, v := range basePredictEdbInfoArr { if _, ok := basePredictMap[v.EdbInfoId]; !ok { newBasePredictEdbInfoArr = append(newBasePredictEdbInfoArr, v) } basePredictMap[v.EdbInfoId] = v.EdbInfoId } newPredictCalculateMap = make(map[int]*data_manage.EdbInfo) for _, v := range predictCalculateInfoArr { if _, ok := newPredictCalculateMap[v.EdbInfoId]; !ok { predictCalculateArr = append(predictCalculateArr, v.EdbInfoId) } newPredictCalculateMap[v.EdbInfoId] = v } sort.Ints(predictCalculateArr) return } func GetEdbSourceByEdbInfoIdList(chartEdbInfoMappingList []*models.ChartEdbInfoMapping) (sourceNameList, sourceNameEnList []string) { sourceNameList = make([]string, 0) sourceNameEnList = make([]string, 0) sourceMap := make(map[int]string) for _, v := range chartEdbInfoMappingList { if v.EdbType == 2 || v.EdbInfoCategoryType == 1 { baseEdbInfoArr, _, _ := data_manage.GetRefreshEdbInfoFromBase(v.EdbInfoId, v.Source) for _, baseEdbInfo := range baseEdbInfoArr { if baseEdbInfo.EdbInfoType == 0 { //普通指标才参与,预测指标不参与 sourceMap[baseEdbInfo.Source] = baseEdbInfo.SourceName } } } else { sourceMap[v.Source] = v.SourceName } } for source, sourceName := range sourceMap { if utils.InArrayByInt([]int{utils.DATA_SOURCE_MANUAL}, source) { continue } sourceNameList = append(sourceNameList, sourceName) sourceNameEn, ok := utils.DataSourceEnMap[source] if !ok { sourceNameEn = sourceName } sourceNameEnList = append(sourceNameEnList, sourceNameEn) } conf, e := models.GetBusinessConf() if e != nil { return } if conf[models.BusinessConfCompanyName] != "" { sourceNameList = append(sourceNameList, conf[models.BusinessConfCompanyName]) sourceNameEnList = append(sourceNameEnList, conf[models.BusinessConfCompanyName]) } return } func edbInfoRefreshAll(refreshAll bool, newBaseEdbInfoArr, newBasePredictEdbInfoArr []*data_manage.EdbInfo, newCalculateMap, newPredictCalculateMap map[int]*data_manage.EdbInfo, calculateArr, predictCalculateArr []int) (err error, errmsg string) { defer func() { if err != nil { fmt.Println("EdbInfoRefreshAllFromBaseV2 Err:" + err.Error() + ";errmsg:" + errmsg) go alarm_msg.SendAlarmMsg("EdbInfoRefreshFromBaseV2,Err"+err.Error()+";errMsg:"+errmsg, 3) } }() var startDate string for _, bv := range newBaseEdbInfoArr { if bv.StartDate == "0000-00-00" { continue } startDate = utils.BaseEdbRefreshStartDate if !refreshAll { // 非刷新所有数据,用结束时间减去对应期数作为起始日期去刷新 sDate := bv.EndDate if sDate == `` || sDate == `0000-00-00` { sDate = utils.BaseEdbRefreshStartDate } sTime, tmpErr := time.Parse(utils.FormatDate, sDate) if tmpErr != nil { err = tmpErr errmsg = "时间格式转换失败2" return } frequency := bv.Frequency var limitDay int switch frequency { case "日度": limitDay = utils.DATA_REFRESH case "周度": limitDay = utils.DATA_REFRESH * 7 case "月度": limitDay = utils.DATA_REFRESH * 30 case "季度": limitDay = utils.DATA_REFRESH * 90 case "年度": limitDay = utils.DATA_REFRESH * 365 default: limitDay = utils.DATA_REFRESH } startDate = sTime.AddDate(0, 0, -limitDay).Format(utils.FormatDate) } result, tmpErr := RefreshEdbData(bv.EdbInfoId, bv.Source, bv.SubSource, bv.EdbCode, startDate) if tmpErr != nil { err = tmpErr fmt.Println(bv.EdbInfoId, "RefreshBaseEdbData err", time.Now()) errmsg = "RefreshBaseEdbData Err:" + tmpErr.Error() return } if result.Ret != 200 { fmt.Println(bv.EdbInfoId, "RefreshBaseEdbData err;msg:", result.Msg, ";errMsg:", result.ErrMsg) errmsg = fmt.Sprint(bv.EdbInfoId, "RefreshBaseEdbData err;msg:", result.Msg, ";errMsg:", result.ErrMsg) err = fmt.Errorf("刷新失败, err:", errmsg) return } fmt.Println("end newBaseEdbInfoArr:", bv, time.Now()) } for _, v := range calculateArr { edbInfo := newCalculateMap[v] if edbInfo == nil { return } startDate = edbInfo.StartDate source := edbInfo.Source if startDate == "" || startDate == "0000-00-00" { //如果没有开始日期,说明还没有计算出来数据,那么就往前面推40年吧(也意味着重新计算了) startDate = time.Now().AddDate(-40, 0, 0).Format(utils.FormatDate) } else { if source == utils.DATA_SOURCE_CALCULATE { startDate = `` if refreshAll { //刷新所有数据,用开始时间作为起始日期去刷新 startDate = edbInfo.StartDate } else { sTime, tmpErr := time.Parse(utils.FormatDate, edbInfo.EndDate) if tmpErr != nil { err = tmpErr errmsg = tmpErr.Error() return } frequency := edbInfo.Frequency var limitDay int switch frequency { case "日度": limitDay = utils.DATA_REFRESH case "周度": limitDay = utils.DATA_REFRESH * 7 case "月度": limitDay = utils.DATA_REFRESH * 30 case "季度": limitDay = utils.DATA_REFRESH * 90 case "年度": limitDay = utils.DATA_REFRESH * 365 default: limitDay = utils.DATA_REFRESH } startDate = sTime.AddDate(0, 0, -limitDay).Format(utils.FormatDate) } } } result, tmpErr := RefreshEdbCalculateData(edbInfo.EdbInfoId, edbInfo.EdbCode, startDate) if tmpErr != nil { err = tmpErr fmt.Println(v, "RefreshEdbCalculateData err", time.Now()) errmsg = "RefreshEdbCalculateData Err:" + tmpErr.Error() return } if result.Ret != 200 { fmt.Println(v, "RefreshEdbCalculateData err;msg:", result.Msg, ";errMsg:", result.ErrMsg) errmsg = fmt.Sprint(v, "RefreshEdbCalculateData err;msg:", result.Msg, ";errMsg:", result.ErrMsg) err = fmt.Errorf("刷新失败") return } } for _, v := range predictCalculateArr { edbInfo := newPredictCalculateMap[v] if edbInfo == nil { return } startDate = edbInfo.StartDate source := edbInfo.Source if startDate == "" || startDate == "0000-00-00" { //如果没有开始日期,说明还没有计算出来数据,那么就往前面推40年吧(也意味着重新计算了) startDate = time.Now().AddDate(-40, 0, 0).Format(utils.FormatDate) } else { if source == utils.DATA_SOURCE_PREDICT_CALCULATE { startDate = `` if refreshAll { //刷新所有数据,用开始时间作为起始日期去刷新 startDate = edbInfo.StartDate } else { sTime, tmpErr := time.Parse(utils.FormatDate, edbInfo.EndDate) if tmpErr != nil { err = tmpErr errmsg = tmpErr.Error() return } frequency := edbInfo.Frequency var limitDay int switch frequency { case "日度": limitDay = utils.DATA_REFRESH case "周度": limitDay = utils.DATA_REFRESH * 7 case "月度": limitDay = utils.DATA_REFRESH * 30 case "季度": limitDay = utils.DATA_REFRESH * 90 case "年度": limitDay = utils.DATA_REFRESH * 365 default: limitDay = utils.DATA_REFRESH } startDate = sTime.AddDate(0, 0, -limitDay).Format(utils.FormatDate) } } } result, tmpErr := RefreshPredictEdbCalculateData(edbInfo.EdbInfoId, edbInfo.EdbCode, startDate) if tmpErr != nil { err = tmpErr fmt.Println(v, "RefreshPredictEdbCalculateData err", time.Now()) errmsg = "RefreshPredictEdbCalculateData Err:" + tmpErr.Error() return } if result.Ret != 200 { fmt.Println(v, "RefreshPredictEdbCalculateData err;msg:", result.Msg, ";errMsg:", result.ErrMsg) errmsg = fmt.Sprint(v, "RefreshPredictEdbCalculateData err;msg:", result.Msg, ";errMsg:", result.ErrMsg) err = fmt.Errorf("刷新失败") return } } { predictEdbInfoIdList := make([]int, 0) for _, v := range newBasePredictEdbInfoArr { predictEdbInfoIdList = append(predictEdbInfoIdList, v.EdbInfoId) } fmt.Println("predictEdbConfCalculateMappingDetailList:", predictEdbInfoIdList) predictEdbConfCalculateMappingDetailList, tmpErr := data_manage.GetPredictEdbConfCalculateMappingDetailListByEdbInfoId(predictEdbInfoIdList) if tmpErr != nil { err = tmpErr errmsg = fmt.Sprint("更新动态环差值时,获取数据失败:" + err.Error()) return } for _, bv := range predictEdbConfCalculateMappingDetailList { result, tmpErr := RefreshEdbData(bv.EdbInfoId, utils.DATA_SOURCE_PREDICT, utils.DATA_SUB_SOURCE_EDB, bv.EdbCode, "") if tmpErr != nil { err = tmpErr fmt.Println(bv.EdbInfoId, "RefreshBasePredictEdbData err", time.Now()) errmsg = "RefreshBasePredictEdbData Err:" + err.Error() return } if result.Ret != 200 { fmt.Println(bv.EdbInfoId, "RefreshBasePredictEdbData err;msg:", result.Msg, ";errMsg:", result.ErrMsg) errmsg = fmt.Sprint(bv.EdbInfoId, "RefreshBasePredictEdbData err;msg:", result.Msg, ";errMsg:", result.ErrMsg) err = errors.New("刷新失败:" + errmsg) return } fmt.Println("end predictEdbConfCalculateMappingDetailList:", bv, time.Now()) } } for _, v := range predictCalculateArr { edbInfo := newPredictCalculateMap[v] if edbInfo == nil { return } startDate = edbInfo.StartDate source := edbInfo.Source if startDate == "" || startDate == "0000-00-00" { //如果没有开始日期,说明还没有计算出来数据,那么就往前面推40年吧(也意味着重新计算了) startDate = time.Now().AddDate(-40, 0, 0).Format(utils.FormatDate) } else { if source == utils.DATA_SOURCE_PREDICT_CALCULATE { startDate = `` if refreshAll { //刷新所有数据,用开始时间作为起始日期去刷新 startDate = edbInfo.StartDate } else { sTime, tmpErr := time.Parse(utils.FormatDate, edbInfo.EndDate) if tmpErr != nil { err = tmpErr errmsg = tmpErr.Error() return } frequency := edbInfo.Frequency var limitDay int switch frequency { case "日度": limitDay = utils.DATA_REFRESH case "周度": limitDay = utils.DATA_REFRESH * 7 case "月度": limitDay = utils.DATA_REFRESH * 30 case "季度": limitDay = utils.DATA_REFRESH * 90 case "年度": limitDay = utils.DATA_REFRESH * 365 default: limitDay = utils.DATA_REFRESH } startDate = sTime.AddDate(0, 0, -limitDay).Format(utils.FormatDate) } } } result, tmpErr := RefreshPredictEdbCalculateData(edbInfo.EdbInfoId, edbInfo.EdbCode, startDate) if tmpErr != nil { err = tmpErr fmt.Println(v, "RefreshPredictEdbCalculateData err", time.Now()) errmsg = "RefreshPredictEdbCalculateData Err:" + tmpErr.Error() return } if result.Ret != 200 { fmt.Println(v, "RefreshPredictEdbCalculateData err;msg:", result.Msg, ";errMsg:", result.ErrMsg) errmsg = fmt.Sprint(v, "RefreshPredictEdbCalculateData err;msg:", result.Msg, ";errMsg:", result.ErrMsg) err = fmt.Errorf("刷新失败") return } } return } func EdbInfoRefreshAllFromBase(edbInfoIdList []int, refreshAll bool) (err error, errMsg string) { defer func() { if err != nil { fmt.Println("EdbInfoRefreshAllFromBaseV3 Err:" + err.Error() + ";errMsg:" + errMsg) go alarm_msg.SendAlarmMsg("EdbInfoRefreshAllFromBaseV3,Err"+err.Error()+";errMsg:"+errMsg, 3) } }() traceEdbInfoList, err := TraceEdbInfoByEdbInfoIdList(edbInfoIdList) if err != nil { return } existEdbInfoIdMap := make(map[int]int) newBaseEdbInfoArr := make([]*data_manage.EdbInfo, 0) newBasePredictEdbInfoArr := make([]*data_manage.EdbInfo, 0) newBaseMap := make(map[int]*data_manage.EdbInfo) newPredictBaseMap := make(map[int]*data_manage.EdbInfo) newCalculateMap := make(map[int]*data_manage.EdbInfo) newPredictCalculateMap := make(map[int]*data_manage.EdbInfo) calculateArr := make([]int, 0) predictCalculateArr := make([]int, 0) for _, traceEdbInfo := range traceEdbInfoList { tmpBaseEdbInfoArr, tmpBasePredictEdbInfoArr, tmpCalculateMap, tmpPredictCalculateMap, _, _ := getRefreshEdbInfoListByTraceEdbInfo(traceEdbInfo, existEdbInfoIdMap) for _, edbInfo := range tmpBaseEdbInfoArr { if _, ok := newBaseMap[edbInfo.EdbInfoId]; !ok { if edbInfo.NoUpdate == 1 { continue } newBaseMap[edbInfo.EdbInfoId] = edbInfo newBaseEdbInfoArr = append(newBaseEdbInfoArr, edbInfo) } } for _, edbInfo := range tmpBasePredictEdbInfoArr { if _, ok := newPredictBaseMap[edbInfo.EdbInfoId]; !ok { newPredictBaseMap[edbInfo.EdbInfoId] = edbInfo newBasePredictEdbInfoArr = append(newBasePredictEdbInfoArr, edbInfo) } } for _, edbInfo := range tmpCalculateMap { if _, ok := newCalculateMap[edbInfo.EdbInfoId]; !ok { if edbInfo.NoUpdate == 1 { continue } newCalculateMap[edbInfo.EdbInfoId] = edbInfo calculateArr = append(calculateArr, edbInfo.EdbInfoId) } } for _, edbInfo := range tmpPredictCalculateMap { if _, ok := newPredictCalculateMap[edbInfo.EdbInfoId]; !ok { newPredictCalculateMap[edbInfo.EdbInfoId] = edbInfo predictCalculateArr = append(predictCalculateArr, edbInfo.EdbInfoId) } } } sort.Ints(calculateArr) sort.Ints(predictCalculateArr) totalEdbInfo := len(newBaseEdbInfoArr) + len(calculateArr) + len(predictCalculateArr) + len(newBasePredictEdbInfoArr) if totalEdbInfo == 0 { return } err, errMsg = edbInfoRefreshAll(refreshAll, newBaseEdbInfoArr, newBasePredictEdbInfoArr, newCalculateMap, newPredictCalculateMap, calculateArr, predictCalculateArr) return } func getRefreshEdbInfoListByTraceEdbInfo(traceEdbInfo data_manage.TraceEdbInfoResp, existEdbInfoIdMap map[int]int) (newBaseEdbInfoArr, newBasePredictEdbInfoArr []*data_manage.EdbInfo, newCalculateMap, newPredictCalculateMap map[int]*data_manage.EdbInfo, calculateArr, predictCalculateArr []int) { newBaseEdbInfoArr = make([]*data_manage.EdbInfo, 0) newBasePredictEdbInfoArr = make([]*data_manage.EdbInfo, 0) newCalculateMap = make(map[int]*data_manage.EdbInfo) newPredictCalculateMap = make(map[int]*data_manage.EdbInfo) calculateArr = make([]int, 0) predictCalculateArr = make([]int, 0) _, ok := existEdbInfoIdMap[traceEdbInfo.EdbInfoId] if ok { return } existEdbInfoIdMap[traceEdbInfo.EdbInfoId] = traceEdbInfo.EdbInfoId switch traceEdbInfo.EdbInfoType { case 0: // 0-普通指标 if traceEdbInfo.EdbType == 1 { //1-基础指标 newBaseEdbInfoArr = append(newBaseEdbInfoArr, traceEdbInfo.EdbInfo) } else if traceEdbInfo.EdbType == 2 { //2-计算指标 newCalculateMap[traceEdbInfo.EdbInfoId] = traceEdbInfo.EdbInfo calculateArr = append(calculateArr, traceEdbInfo.EdbInfoId) } case 1: // 1-预测指标 if traceEdbInfo.EdbType == 1 { //1-基础指标 newBasePredictEdbInfoArr = append(newBasePredictEdbInfoArr, traceEdbInfo.EdbInfo) } else if traceEdbInfo.EdbType == 2 { //2-计算指标 newPredictCalculateMap[traceEdbInfo.EdbInfoId] = traceEdbInfo.EdbInfo predictCalculateArr = append(predictCalculateArr, traceEdbInfo.EdbInfoId) } } if traceEdbInfo.Child != nil && len(traceEdbInfo.Child) > 0 { for _, v := range traceEdbInfo.Child { tmpBaseEdbInfoArr, tmpPredictEdbInfoArr, tmpCalculateMap, tmpPredictCalculateMap, tmpCalculateArr, tmpPredictCalculateArr := getRefreshEdbInfoListByTraceEdbInfo(v, existEdbInfoIdMap) newBaseEdbInfoArr = append(newBaseEdbInfoArr, tmpBaseEdbInfoArr...) newBasePredictEdbInfoArr = append(newBasePredictEdbInfoArr, tmpPredictEdbInfoArr...) for k, tmpEdbInfo := range tmpCalculateMap { newCalculateMap[k] = tmpEdbInfo } for k, tmpEdbInfo := range tmpPredictCalculateMap { newPredictCalculateMap[k] = tmpEdbInfo } calculateArr = append(calculateArr, tmpCalculateArr...) predictCalculateArr = append(predictCalculateArr, tmpPredictCalculateArr...) } } return } func TraceEdbInfoByEdbInfoIdList(edbInfoIdList []int) (traceEdbInfoList []data_manage.TraceEdbInfoResp, err error) { traceEdbInfoList = make([]data_manage.TraceEdbInfoResp, 0) edbInfoList, err := data_manage.GetEdbInfoByIdList(edbInfoIdList) if err != nil { return } edbInfoRuleMap := make(map[int]string, 0) edbMappingMap := make(map[int][]*data_manage.EdbInfoCalculateMappingInfo) findIdMap := make(map[int]int) existMap := make(map[int]data_manage.TraceEdbInfoResp) for _, edbInfo := range edbInfoList { findIdMap[edbInfo.EdbInfoId] = edbInfo.EdbInfoId traceEdbInfo := data_manage.TraceEdbInfoResp{ EdbInfoId: edbInfo.EdbInfoId, EdbInfoType: edbInfo.EdbInfoType, EdbName: edbInfo.EdbName, EdbType: edbInfo.EdbType, UniqueCode: edbInfo.UniqueCode, ClassifyId: edbInfo.ClassifyId, EdbInfo: edbInfo, } traceEdbInfo.Child, err = traceEdbInfoByEdbInfoId(edbInfo.EdbInfoId, traceEdbInfo, edbInfoRuleMap, findIdMap, existMap, edbMappingMap) traceEdbInfoList = append(traceEdbInfoList, traceEdbInfo) } return } func traceEdbInfoByEdbInfoId(edbInfoId int, traceEdbInfo data_manage.TraceEdbInfoResp, edbInfoRuleMap map[int]string, findIdMap map[int]int, existMap map[int]data_manage.TraceEdbInfoResp, edbMappingMap map[int][]*data_manage.EdbInfoCalculateMappingInfo) (child []data_manage.TraceEdbInfoResp, err error) { traceEdbInfo, ok := existMap[edbInfoId] if ok { return } child = make([]data_manage.TraceEdbInfoResp, 0) edbInfoMappingList, e := data_manage.GetEdbInfoCalculateMappingListByEdbInfoId(edbInfoId) if e != nil { err = fmt.Errorf("GetEdbInfoCalculateMappingListByEdbInfoId err: %s", e.Error()) return } edbInfoMap := make(map[int]*data_manage.EdbInfo) if len(edbInfoMappingList) > 0 { fromEdbInfoIdList := make([]int, 0) for _, v := range edbInfoMappingList { fromEdbInfoIdList = append(fromEdbInfoIdList, v.FromEdbInfoId) } edbInfoList, tmpErr := data_manage.GetEdbInfoByIdList(fromEdbInfoIdList) if tmpErr != nil { err = fmt.Errorf("traceEdbInfoByEdbInfoId GetEdbInfoByIdList err: %s", tmpErr.Error()) return } for _, v := range edbInfoList { edbInfoMap[v.EdbInfoId] = v } } edbMappingMap[edbInfoId] = edbInfoMappingList for _, v := range edbInfoMappingList { tmpEdbInfoId := v.FromEdbInfoId tmpTraceEdbInfo := data_manage.TraceEdbInfoResp{ EdbInfoId: tmpEdbInfoId, EdbInfoType: v.FromEdbInfoType, EdbType: v.FromEdbType, UniqueCode: v.FromUniqueCode, ClassifyId: v.FromClassifyId, EdbInfo: edbInfoMap[v.FromEdbInfoId], } if edbInfoId != v.FromEdbInfoId && (v.FromEdbType == 2 || v.FromEdbInfoType == 1) { if _, ok2 := findIdMap[tmpEdbInfoId]; !ok2 { tmpTraceEdbInfo.Child, e = traceEdbInfoByEdbInfoId(tmpEdbInfoId, tmpTraceEdbInfo, edbInfoRuleMap, findIdMap, existMap, edbMappingMap) if e != nil { err = fmt.Errorf("traceEdbInfoByEdbInfoId err: %s", e.Error()) return } } } child = append(child, tmpTraceEdbInfo) findIdMap[tmpEdbInfoId] = tmpEdbInfoId } existMap[edbInfoId] = traceEdbInfo return } func GetEdbSourceByEdbInfoIdListForExcel(edbInfoIdList []int) (sourceNameList, sourceNameEnList []string, err error) { sourceNameList = make([]string, 0) sourceNameEnList = make([]string, 0) sourceMap := make(map[int]string) edbInfoList, tmpErr := data_manage.GetEdbInfoByIdList(edbInfoIdList) if tmpErr != nil { err = tmpErr return } for _, v := range edbInfoList { if v.EdbType == 2 { baseEdbInfoArr, _, _ := data_manage.GetRefreshEdbInfoFromBase(v.EdbInfoId, v.Source) for _, baseEdbInfo := range baseEdbInfoArr { if baseEdbInfo.EdbInfoType == 0 { //普通指标才参与,预测指标不参与 sourceMap[baseEdbInfo.Source] = baseEdbInfo.SourceName } } } else { sourceMap[v.Source] = v.SourceName } } for source, sourceName := range sourceMap { if utils.InArrayByInt([]int{utils.DATA_SOURCE_MANUAL, utils.DATA_SOURCE_MYSTEEL_CHEMICAL}, source) { continue } sourceNameList = append(sourceNameList, sourceName) sourceNameEn, ok := utils.DataSourceEnMap[source] if !ok { sourceNameEn = sourceName } sourceNameEnList = append(sourceNameEnList, sourceNameEn) } conf, e := models.GetBusinessConf() if e != nil { return } if conf[models.BusinessConfCompanyName] != "" { sourceNameList = append(sourceNameList, conf[models.BusinessConfCompanyName]) sourceNameEnList = append(sourceNameEnList, conf[models.BusinessConfCompanyName]) } return } func HandleDataByLinearRegressionToList(edbInfoDataList []*models.EdbDataList, handleDataMap map[string]float64) (dataTimeList []string, valueList []float64, err error) { if len(edbInfoDataList) < 2 { return } var startEdbInfoData *models.EdbDataList for _, v := range edbInfoDataList { handleDataMap[v.DataTime] = v.Value dataTimeList = append(dataTimeList, v.DataTime) if startEdbInfoData == nil { startEdbInfoData = v continue } startDataTime, _ := time.ParseInLocation(utils.FormatDate, startEdbInfoData.DataTime, time.Local) currDataTime, _ := time.ParseInLocation(utils.FormatDate, v.DataTime, time.Local) betweenHour := int(currDataTime.Sub(startDataTime).Hours()) betweenDay := betweenHour / 24 if betweenDay <= 1 { startEdbInfoData = v continue } var a, b float64 { coordinateData := make([]utils.Coordinate, 0) tmpCoordinate1 := utils.Coordinate{ X: 1, Y: startEdbInfoData.Value, } coordinateData = append(coordinateData, tmpCoordinate1) tmpCoordinate2 := utils.Coordinate{ X: float64(betweenDay) + 1, Y: v.Value, } coordinateData = append(coordinateData, tmpCoordinate2) a, b = utils.GetLinearResult(coordinateData) if math.IsNaN(a) || math.IsNaN(b) { err = errors.New("线性方程公式生成失败") return } } { for i := 1; i < betweenDay; i++ { tmpDataTime := startDataTime.AddDate(0, 0, i) aDecimal := decimal.NewFromFloat(a) xDecimal := decimal.NewFromInt(int64(i) + 1) bDecimal := decimal.NewFromFloat(b) val, _ := aDecimal.Mul(xDecimal).Add(bDecimal).Round(4).Float64() handleDataMap[tmpDataTime.Format(utils.FormatDate)] = val dataTimeList = append(dataTimeList, tmpDataTime.Format(utils.FormatDate)) valueList = append(valueList, val) } } startEdbInfoData = v } return } func HandleDataByLinearRegressionToListV2(edbInfoDataList []*models.EdbDataList, handleDataMap map[string]float64) (dataTimeList []string, valueList []float64, err error) { if len(edbInfoDataList) < 2 { return } if len(edbInfoDataList) < 2 { err = errors.New("至少需要两天的数据来执行线性插值") return } sort.Slice(edbInfoDataList, func(i, j int) bool { t1, _ := time.ParseInLocation(utils.FormatDate, edbInfoDataList[i].DataTime, time.Local) t2, _ := time.ParseInLocation(utils.FormatDate, edbInfoDataList[j].DataTime, time.Local) return t1.Before(t2) }) startEdbInfoData := edbInfoDataList[0] endEdbInfoData := edbInfoDataList[len(edbInfoDataList)-1] startDate, _ := time.ParseInLocation(utils.FormatDate, startEdbInfoData.DataTime, time.Local) endDate, _ := time.ParseInLocation(utils.FormatDate, endEdbInfoData.DataTime, time.Local) actualDays := endDate.Sub(startDate).Hours() / 24 for _, v := range edbInfoDataList { handleDataMap[v.DataTime] = v.Value dataTimeList = append(dataTimeList, v.DataTime) valueList = append(valueList, v.Value) } if actualDays < 365 { var a, b float64 coordinateData := []utils.Coordinate{ {X: 1, Y: startEdbInfoData.Value}, {X: float64(len(edbInfoDataList)), Y: endEdbInfoData.Value}, } a, b = utils.GetLinearResult(coordinateData) if math.IsNaN(a) || math.IsNaN(b) { err = errors.New("线性方程公式生成失败") return } for i := 1; i < 365; i++ { day := startDate.AddDate(0, 0, i) if _, exists := handleDataMap[day.Format(utils.FormatDate)]; !exists { aDecimal := decimal.NewFromFloat(a) xDecimal := decimal.NewFromInt(int64(i) + 1) bDecimal := decimal.NewFromFloat(b) val, _ := aDecimal.Mul(xDecimal).Add(bDecimal).Round(4).Float64() handleDataMap[day.Format(utils.FormatDate)] = val dataTimeList = append(dataTimeList, day.Format(utils.FormatDate)) valueList = append(valueList, val) } } } return } func HandleDataByLinearRegressionToListV3(edbInfoDataList []*models.EdbDataList, handleDataMap map[string]float64) (newEdbInfoDataList []*models.EdbDataList, dataTimeList []string, valueList []float64, err error) { if len(edbInfoDataList) < 2 { return } var startEdbInfoData *models.EdbDataList for _, v := range edbInfoDataList { handleDataMap[v.DataTime] = v.Value newEdbInfoDataList = append(newEdbInfoDataList, v) dataTimeList = append(dataTimeList, v.DataTime) if startEdbInfoData == nil { startEdbInfoData = v continue } startDataTime, _ := time.ParseInLocation(utils.FormatDate, startEdbInfoData.DataTime, time.Local) currDataTime, _ := time.ParseInLocation(utils.FormatDate, v.DataTime, time.Local) betweenHour := int(currDataTime.Sub(startDataTime).Hours()) betweenDay := betweenHour / 24 if betweenDay <= 1 { startEdbInfoData = v continue } var a, b float64 { coordinateData := make([]utils.Coordinate, 0) tmpCoordinate1 := utils.Coordinate{ X: 1, Y: startEdbInfoData.Value, } coordinateData = append(coordinateData, tmpCoordinate1) tmpCoordinate2 := utils.Coordinate{ X: float64(betweenDay) + 1, Y: v.Value, } coordinateData = append(coordinateData, tmpCoordinate2) a, b = utils.GetLinearResult(coordinateData) if math.IsNaN(a) || math.IsNaN(b) { err = errors.New("线性方程公式生成失败") return } } { for i := 1; i < betweenDay; i++ { tmpDataTime := startDataTime.AddDate(0, 0, i) aDecimal := decimal.NewFromFloat(a) xDecimal := decimal.NewFromInt(int64(i) + 1) bDecimal := decimal.NewFromFloat(b) val, _ := aDecimal.Mul(xDecimal).Add(bDecimal).Round(4).Float64() handleDataMap[tmpDataTime.Format(utils.FormatDate)] = val dataTimeList = append(dataTimeList, tmpDataTime.Format(utils.FormatDate)) valueList = append(valueList, val) newEdbInfoDataList = append(newEdbInfoDataList, &models.EdbDataList{ EdbDataId: v.EdbDataId, EdbInfoId: v.EdbInfoId, DataTime: tmpDataTime.Format(utils.FormatDate), DataTimestamp: tmpDataTime.UnixNano() / 1e6, Value: val, }) } } startEdbInfoData = v } return }