package index_data import ( "eta_gn/eta_bridge/global" "eta_gn/eta_bridge/models/gn" "eta_gn/eta_bridge/models/index" "eta_gn/eta_bridge/services/alarm_msg" "eta_gn/eta_bridge/utils" "fmt" "github.com/rdlucklib/rdluck_tools/paging" "github.com/shopspring/decimal" "strings" "time" ) // TODO 过滤前后空格 // GetPageIndexesFromGn 分页获取指标数据 func GetPageIndexesFromGn(pageIndex, pageSize int, modifyTime, indexId string) (list []gn.IndexInfo, page *paging.PagingItem, err error) { defer func() { if err != nil { global.LOG.Info("GetPageIndexesFromGn Err: " + err.Error()) go alarm_msg.SendAlarmMsg("GetPageIndexesFromGn Err: "+err.Error(), 3) return } }() req := IndexInfoReq{ UpdateTime: "", Size: pageSize, PageNo: pageIndex, } if indexId != "" { req.IndexId = indexId } resp, err := GetIndexInfoList(req) if err != nil { return } page = paging.GetPaging(resp.Data.PageNo, resp.Data.Size, resp.Data.Total) list = make([]gn.IndexInfo, 0) for _, v := range resp.Data.Records { //if v.INDEXLEVELTHREE == `` { // continue //} // 必须保证有一级目录 if v.INDEXLEVELONE == `` { v.INDEXLEVELONE = `其它指标` } //if v.INDEXLEVELTWO == `` { // v.INDEXLEVELTWO = `其它指标` //} //if v.INDEXLEVELTHREE == `` { // v.INDEXLEVELTHREE = `其它指标` //} list = append(list, gn.IndexInfo{ ClassifyNameOne: strings.TrimSpace(v.INDEXLEVELONE), ClassifyNameTwo: strings.TrimSpace(v.INDEXLEVELTWO), ClassifyNameThree: strings.TrimSpace(v.INDEXLEVELTHREE), DataIndexCode: fmt.Sprint(v.UNIQUE_ID), SourceEdbCode: strings.TrimSpace(v.INDEXORIGINCODE), EdbName: strings.TrimSpace(v.INDEXNAME), Frequency: strings.TrimSpace(v.DATAFREQ), Unit: strings.TrimSpace(v.DATAUNIT), SourceName: strings.TrimSpace(v.DATASOURCE), SourceCode: strings.TrimSpace(v.DATASOURCECODE), }) } return } // GetIndexAndDataFromGn 获取指标和数据 func GetIndexAndDataFromGn(indexCode, startDate, endDate string) (list []gn.IndexData, err error) { defer func() { if err != nil { global.LOG.Info("GetIndexAndDataFromGn Err: " + err.Error()) go alarm_msg.SendAlarmMsg("GetIndexAndDataFromGn Err: "+err.Error(), 3) return } }() var dataDate string dateList := make([]string, 0) // 没有指定开始日期的话,那就默认当前日期 if startDate == `` { startDate = `1901-01-01` } dateList = append(dateList, startDate) // 没有的话,就默认往后找一个月 if endDate == `` { endDate = time.Now().AddDate(0, 1, 0).Format(utils.FormatDate) } dateList = append(dateList, endDate) if len(dateList) > 0 { dataDate = strings.Join(dateList, ",") } req := IndexValueReq{ UniqueId: indexCode, DataDate: dataDate, Size: 500, PageNo: 1, } list = make([]gn.IndexData, 0) isEnd := false for i := 1; !isEnd; i++ { req.PageNo = i tmpList, page, tmpErr := getIndexAndDataFromGn(req) if tmpErr != nil { err = tmpErr return } list = append(list, tmpList...) isEnd = page.IsEnd } return } // getIndexAndDataFromGn // @Description: 获取数据 // @param req // @return list // @return page // @return err func getIndexAndDataFromGn(req IndexValueReq) (list []gn.IndexData, page *paging.PagingItem, err error) { defer func() { if err != nil { global.LOG.Info("GetIndexAndDataFromGn Err: " + err.Error()) go alarm_msg.SendAlarmMsg("GetIndexAndDataFromGn Err: "+err.Error(), 3) return } }() resp, err := GetIndexValueList(req) if err != nil { return } page = paging.GetPaging(resp.Data.PageNo, resp.Data.Size, resp.Data.Total) list = make([]gn.IndexData, 0) for _, v := range resp.Data.Records { // 非数值数据,过滤 //脏数据: // 停产 // TODO、 // 1-2 // 空字符串 // 空数据,过滤 if v.VALUE == `` { continue } valF, tmpErr := decimal.NewFromString(strings.TrimSpace(v.VALUE)) if tmpErr != nil { //err = tmpErr continue } val, _ := valF.Float64() dataTime, tmpErr := time.ParseInLocation(utils.FormatDateTime, v.DATADATE, time.Local) if tmpErr != nil { continue } list = append(list, gn.IndexData{ Val: val, UpdateTime: v.UPDATETIME, DataTime: dataTime.Format(utils.FormatDate), }) } return } func FixData() { edbInfoList, err := index.GetAllBaseEdbInfo() if err != nil { fmt.Println("获取指标列表失败:", err) return } sourceMap := map[int]string{ 96: `S02`, 97: `S04`, 98: `S03`, 99: `S05`, 100: `S08`, 101: `S01`, } count := len(edbInfoList) for k, v := range edbInfoList { fmt.Println("剩余:", count-k, "条数据") list, _, err := getData(1, 100, v.EdbCode, sourceMap[v.Source]) if err != nil { fmt.Println("获取指标数据失败:", err) continue } if len(list) > 0 { uniqueCode := list[0].DataIndexCode fmt.Println(v.EdbCode, " === ", uniqueCode) v.OriginalEdbCode = uniqueCode err = v.Update([]string{"OriginalEdbCode"}) if err != nil { fmt.Println("修改指标原始编码失败:", err) continue } } } } func getData(pageIndex, pageSize int, edbCode, dataSource string) (list []gn.IndexInfo, page *paging.PagingItem, err error) { defer func() { if err != nil { global.LOG.Info("GetPageIndexesFromGn Err: " + err.Error()) go alarm_msg.SendAlarmMsg("GetPageIndexesFromGn Err: "+err.Error(), 3) return } }() req := IndexInfoV2Req{ UpdateTime: "", Size: pageSize, PageNo: pageIndex, EdbCode: edbCode, SourceCode: dataSource, } resp, err := GetIndexInfoListV2(req) if err != nil { return } page = paging.GetPaging(resp.Data.PageNo, resp.Data.Size, resp.Data.Total) list = make([]gn.IndexInfo, 0) for _, v := range resp.Data.Records { //if v.INDEXLEVELTHREE == `` { // continue //} // 必须保证有一级目录 if v.INDEXLEVELONE == `` { v.INDEXLEVELONE = `其它指标` } //if v.INDEXLEVELTWO == `` { // v.INDEXLEVELTWO = `其它指标` //} //if v.INDEXLEVELTHREE == `` { // v.INDEXLEVELTHREE = `其它指标` //} list = append(list, gn.IndexInfo{ ClassifyNameOne: strings.TrimSpace(v.INDEXLEVELONE), ClassifyNameTwo: strings.TrimSpace(v.INDEXLEVELTWO), ClassifyNameThree: strings.TrimSpace(v.INDEXLEVELTHREE), DataIndexCode: fmt.Sprint(v.UNIQUE_ID), SourceEdbCode: strings.TrimSpace(v.INDEXORIGINCODE), EdbName: strings.TrimSpace(v.INDEXORIGINNAME), Frequency: strings.TrimSpace(v.DATAFREQ), Unit: strings.TrimSpace(v.DATAUNIT), SourceName: strings.TrimSpace(v.DATASOURCE), SourceCode: strings.TrimSpace(v.DATASOURCECODE), }) } return }