package services import ( "context" "encoding/json" "eta/eta_crawler/models" "fmt" "github.com/rdlucklib/rdluck_tools/http" "reflect" "strconv" "time" ) func SyncEiaSteoData(cont context.Context) (err error) { err = syncEiaSteoData() return } func syncEiaSteoData() (err error) { // 获取数据 //官网地址：https://www.eia.gov/outlooks/steo/data/browser/#/?v=6&f=M&s=0&start=201701&end=202312&linechart=~T3_STCHANGE_WORLD&maptype=0&ctype=linechart&map= // 这是获取数据的链接（月度的） eiaSteoUrl := "https://www.eia.gov/outlooks/steo/data/browser/data/index.php?v=6&f=M&s=0&id=&linechart=PAPR_OECD~PAPR_NONOPEC&maptype=0&ctype=linechart&map=&method=getData" eiaSteoData, err := queryData(eiaSteoUrl) if err != nil { fmt.Println("读取失败", err) return } // 获取分类列表 classifyList, err := models.GetBaseFromEiaSteoClassifyAll() if err != nil { fmt.Println("获取分类失败：", err) return } classifyMap := make(map[string]*models.BaseFromEiaSteoClassify) for _, v := range classifyList { classifyMap[v.ClassifyNameOriginal] = v } // 获取指标列表 indexList, err := models.GetBaseFromEiaSteoIndexAll() if err != nil { fmt.Println("获取分类失败：", err) return } indexMap := make(map[string]*models.BaseFromEiaSteoIndex) for _, v := range indexList { indexMap[v.IndexCode] = v } var nowClassify *models.BaseFromEiaSteoClassify for _, v := range eiaSteoData.VIEWSDATA.ROWS { // 如果没有数据，那么就返回 if v.HASDATA != 1 { continue } if v.LEVEL == 1 { tmpNowClassify, ok := classifyMap[v.CHARTNAME] if !ok { nowClassify = &models.BaseFromEiaSteoClassify{ BaseFromEiaSteoClassifyId: 0, ClassifyName: v.CHARTNAME, ClassifyNameOriginal: v.CHARTNAME, ModifyTime: time.Now(), CreateTime: time.Now(), } // 新增指标 err = nowClassify.AddBaseFromEiaSteoClassify() if err != nil { return } classifyMap[v.CHARTNAME] = nowClassify } else { nowClassify = tmpNowClassify } //continue } // 如果系列名称为空的话，那么也返回 if v.SERIESID == `` { continue } if v.LEVEL > 1 && v.SERIESID == `` { continue } var eiaSteoIndex *models.BaseFromEiaSteoIndex eiaSteoIndex, ok := indexMap[v.SERIESID] // 指标名称（中文） indexName := EiaSteoNameMap[v.SERIESID] if indexName == `` { indexName = v.CHARTNAME } if !ok { eiaSteoIndex = &models.BaseFromEiaSteoIndex{ //BaseFromEiaSteoIndexId: 0, BaseFromEiaSteoClassifyId: nowClassify.BaseFromEiaSteoClassifyId, IndexCode: v.SERIESID, IndexName: indexName, IndexNameOriginal: v.CHARTNAME, Frequency: "月度", Level: v.LEVEL, Unit: v.UNITS, Super: v.SUPER, Precision: v.PRECISION, LastHistorical: strconv.Itoa(v.LASTHISTORICAL), Description: v.DESCRIPTION, IsMappable: v.ISMAPPABLE, StartDate: time.Now(), EndDate: time.Now(), ModifyTime: time.Now(), CreateTime: time.Now(), } // 新增指标 err = eiaSteoIndex.Add() if err != nil { return } indexMap[v.SERIESID] = eiaSteoIndex } else { updateCol := make([]string, 0) if eiaSteoIndex.BaseFromEiaSteoClassifyId != nowClassify.BaseFromEiaSteoClassifyId { eiaSteoIndex.BaseFromEiaSteoClassifyId = nowClassify.BaseFromEiaSteoClassifyId updateCol = append(updateCol, "BaseFromEiaSteoClassifyId") } if eiaSteoIndex.IndexName != indexName { eiaSteoIndex.IndexName = indexName updateCol = append(updateCol, "IndexName") } if eiaSteoIndex.IndexNameOriginal != v.CHARTNAME { eiaSteoIndex.IndexNameOriginal = v.CHARTNAME updateCol = append(updateCol, "IndexNameOriginal") } // Frequency: "月度", // Level: v.LEVEL, // Unit: v.UNITS, // Super: v.SUPER, // Precision: v.PRECISION, // LastHistorical: strconv.Itoa(v.LASTHISTORICAL), // Description: v.DESCRIPTION, // IsMappable: v.ISMAPPABLE, if len(updateCol) > 0 { eiaSteoIndex.Update(updateCol) } } //校验数据类型对不对 valType := reflect.TypeOf(v.DATA) //if valType.String() == "map[string]interface{}"{ // fmt.Println("匹配上了") //} switch valType.String() { case "[]interface {}": // 没有数据 case "map[string]interface {}": // 有数据 data := v.DATA.(map[string]interface{}) err = models.HandleEiaSteoData(data, eiaSteoIndex) if err != nil { return } //if eiaSteoIndex.IndexCode == "PATC_OECD_EUROPE" { // os.Exit(0) //} } fmt.Println(v.CHARTNAME, v.SERIESID, "==", v.PRECISION, "===========") } return } // queryData 接口请求网站数据 func queryData(eiaSteoUrl string) (eiaSteoData EiaSteoData, err error) { body, err := http.Get(eiaSteoUrl) if err != nil { fmt.Println("err:", err) } //utils.FileLog.Info("eia steo 报告数据：" + string(body)) err = json.Unmarshal(body, &eiaSteoData) return } type EiaSteoData struct { SERIESDATA struct { DATACOLUMNS []interface{} `json:"DATACOLUMNS"` ROWS []interface{} `json:"ROWS"` } `json:"SERIESDATA"` VIEWSDATA struct { DATACOLUMNS []int `json:"DATACOLUMNS"` ROWS []struct { CHARTNAME string `json:"CHART_NAME"` DESCRIPTION string `json:"DESCRIPTION"` HASDATA int `json:"HAS_DATA"` PRECISION int `json:"PRECISION"` SERIESID string `json:"SERIES_ID"` UNITS string `json:"UNITS"` SUPER string `json:"SUPER"` LEVEL int `json:"LEVEL"` //DATA map[int]float64 `json:"DATA"` DATA interface{} `json:"DATA"` LASTHISTORICAL int `json:"LAST_HISTORICAL,omitempty"` ISMAPPABLE int `json:"IS_MAPPABLE,omitempty"` } `json:"ROWS"` } `json:"VIEWSDATA"` } // EiaSteoNameMap 中英文互换 var EiaSteoNameMap = map[string]string{ "PAPR_OECD": "OECD石油产量", "PAPR_US": "美国五十州石油产量", "PAPR_CA": "加拿大石油产量", "PAPR_MX": "墨西哥石油产量", "PAPR_OTHEROECD": "其他OECD石油产量", "PAPR_NONOECD": "非OECD石油产量", "PAPR_OPEC": "OPEC13国石油产量", "COPR_OPEC": "OPEC13国原油产量", "OPEC_NC": "OPEC其他石油液体产量", "PAPR_FSU": "欧亚石油产量", "PAPR_CH": "中国石油产量", "PAPR_OTHER_NONOECD": "其他非OECD石油产量", "PAPR_WORLD": "全球石油产量", "PAPR_NONOPEC": "非OPEC石油产量", "PATC_OECD": "OECD石油需求", "PATC_US": "美国五十州石油需求", "PATC_UST": "美国领地石油需求", "PATC_CA": "加拿大石油需求", "PATC_OECD_EUROPE": "欧洲石油需求", "PATC_JA": "日本石油需求", "PATC_OTHER_OECD": "其他OECD石油需求", "PATC_NON_OECD": "非OECD石油需求", "PATC_FSU": "欧亚石油需求", "PATC_NONOECD_EUROPE": "非OECD欧洲石油需求", "PATC_CH": "中国石油需求", "PATC_OTHER_ASIA": "其他亚洲石油需求", "PATC_OTHER_NONOECD": "其他非OECD石油需求", "PATC_WORLD": "全球石油需求", "T3_STCHANGE_US": "美国50州石油库存去化（需求-供给）", "T3_STCHANGE_OOECD": "其他OECD石油库存去化（需求-供给）", "T3_STCHANGE_NOECD": "非OECD石油库存去化（需求-供给）", "T3_STCHANGE_WORLD": "全球石油库存去化（需求-供给）", "PASC_US": "美国商业库存", "PASC_OECD_T3": "OECD商业库存", }