eta_crawler/services/eia_steo.go

package services

import (
	"context"
	"encoding/json"
	"fmt"
	"github.com/rdlucklib/rdluck_tools/http"
	"hongze/hongze_data_crawler/models"
	"reflect"
	"strconv"
	"time"
)

func SyncEiaSteoData(cont context.Context) (err error) {
	err = syncEiaSteoData()
	return
}

func syncEiaSteoData() (err error) {
	// 获取数据

	//官网地址：https://www.eia.gov/outlooks/steo/data/browser/#/?v=6&f=M&s=0&start=201701&end=202312&linechart=~T3_STCHANGE_WORLD&maptype=0&ctype=linechart&map=
	// 这是获取数据的链接（月度的）
	eiaSteoUrl := "https://www.eia.gov/outlooks/steo/data/browser/data/index.php?v=6&f=M&s=0&id=&linechart=PAPR_OECD~PAPR_NONOPEC&maptype=0&ctype=linechart&map=&method=getData"
	eiaSteoData, err := queryData(eiaSteoUrl)
	if err != nil {
		fmt.Println("读取失败", err)
		return
	}

	// 获取分类列表
	classifyList, err := models.GetBaseFromEiaSteoClassifyAll()
	if err != nil {
		fmt.Println("获取分类失败：", err)
		return
	}
	classifyMap := make(map[string]*models.BaseFromEiaSteoClassify)
	for _, v := range classifyList {
		classifyMap[v.ClassifyNameOriginal] = v
	}

	// 获取指标列表
	indexList, err := models.GetBaseFromEiaSteoIndexAll()
	if err != nil {
		fmt.Println("获取分类失败：", err)
		return
	}
	indexMap := make(map[string]*models.BaseFromEiaSteoIndex)
	for _, v := range indexList {
		indexMap[v.IndexCode] = v
	}

	var nowClassify *models.BaseFromEiaSteoClassify
	for _, v := range eiaSteoData.VIEWSDATA.ROWS {
		if v.LEVEL == 1 {
			tmpNowClassify, ok := classifyMap[v.CHARTNAME]
			if !ok {
				nowClassify = &models.BaseFromEiaSteoClassify{
					BaseFromEiaSteoClassifyId: 0,
					ClassifyName:              v.CHARTNAME,
					ClassifyNameOriginal:      v.CHARTNAME,
					ModifyTime:                time.Now(),
					CreateTime:                time.Now(),
				}
				// 新增指标
				err = nowClassify.AddBaseFromEiaSteoClassify()
				if err != nil {
					return
				}
				classifyMap[v.CHARTNAME] = nowClassify
			} else {
				nowClassify = tmpNowClassify
			}
			continue
		}
		if v.LEVEL > 1 && v.SERIESID == `` {
			continue
		}

		var eiaSteoIndex *models.BaseFromEiaSteoIndex

		eiaSteoIndex, ok := indexMap[v.SERIESID]

		// 指标名称（中文）
		indexName := EiaSteoNameMap[v.SERIESID]
		if indexName == `` {
			indexName = v.CHARTNAME
		}
		if !ok {
			eiaSteoIndex = &models.BaseFromEiaSteoIndex{
				//BaseFromEiaSteoIndexId:    0,
				BaseFromEiaSteoClassifyId: nowClassify.BaseFromEiaSteoClassifyId,
				IndexCode:                 v.SERIESID,
				IndexName:                 indexName,
				IndexNameOriginal:         v.CHARTNAME,
				Frequency:                 "月度",
				Level:                     v.LEVEL,
				Unit:                      v.UNITS,
				Super:                     v.SUPER,
				Precision:                 v.PRECISION,
				LastHistorical:            strconv.Itoa(v.LASTHISTORICAL),
				Description:               v.DESCRIPTION,
				IsMappable:                v.ISMAPPABLE,
				StartDate:                 time.Now(),
				EndDate:                   time.Now(),
				ModifyTime:                time.Now(),
				CreateTime:                time.Now(),
			}
			// 新增指标
			err = eiaSteoIndex.Add()
			if err != nil {
				return
			}
			indexMap[v.SERIESID] = eiaSteoIndex
		} else {
			updateCol := make([]string, 0)
			if eiaSteoIndex.BaseFromEiaSteoClassifyId != nowClassify.BaseFromEiaSteoClassifyId {
				eiaSteoIndex.BaseFromEiaSteoClassifyId = nowClassify.BaseFromEiaSteoClassifyId
				updateCol = append(updateCol, "BaseFromEiaSteoClassifyId")
			}
			if eiaSteoIndex.IndexName != indexName {
				eiaSteoIndex.IndexName = indexName
				updateCol = append(updateCol, "IndexName")
			}
			if eiaSteoIndex.IndexNameOriginal != v.CHARTNAME {
				eiaSteoIndex.IndexNameOriginal = v.CHARTNAME
				updateCol = append(updateCol, "IndexNameOriginal")
			}
			//	Frequency:                  "月度",
			//	Level:                      v.LEVEL,
			//	Unit:                       v.UNITS,
			//	Super:                      v.SUPER,
			//	Precision:                  v.PRECISION,
			//	LastHistorical:             strconv.Itoa(v.LASTHISTORICAL),
			//	Description:                v.DESCRIPTION,
			//	IsMappable:                 v.ISMAPPABLE,
			if len(updateCol) > 0 {
				eiaSteoIndex.Update(updateCol)
			}

		}

		//校验数据类型对不对
		valType := reflect.TypeOf(v.DATA)
		//if valType.String() == "map[string]interface{}"{
		//	fmt.Println("匹配上了")
		//}
		switch valType.String() {
		case "[]interface {}": // 没有数据
		case "map[string]interface {}": // 有数据
			data := v.DATA.(map[string]interface{})
			err = models.HandleEiaSteoData(data, eiaSteoIndex)
			if err != nil {
				return
			}
			//if eiaSteoIndex.IndexCode == "PATC_OECD_EUROPE" {
			//	os.Exit(0)
			//}
		}

		fmt.Println(v.CHARTNAME, v.SERIESID, "==", v.PRECISION, "===========")
	}
	return
}

// queryData 接口请求网站数据
func queryData(eiaSteoUrl string) (eiaSteoData EiaSteoData, err error) {
	body, err := http.Get(eiaSteoUrl)
	if err != nil {
		fmt.Println("err:", err)
	}
	//utils.FileLog.Info("eia steo 报告数据：" + string(body))
	err = json.Unmarshal(body, &eiaSteoData)
	return
}

type EiaSteoData struct {
	SERIESDATA struct {
		DATACOLUMNS []interface{} `json:"DATACOLUMNS"`
		ROWS        []interface{} `json:"ROWS"`
	} `json:"SERIESDATA"`
	VIEWSDATA struct {
		DATACOLUMNS []int `json:"DATACOLUMNS"`
		ROWS        []struct {
			CHARTNAME   string `json:"CHART_NAME"`
			DESCRIPTION string `json:"DESCRIPTION"`
			HASDATA     int    `json:"HAS_DATA"`
			PRECISION   int    `json:"PRECISION"`
			SERIESID    string `json:"SERIES_ID"`
			UNITS       string `json:"UNITS"`
			SUPER       string `json:"SUPER"`
			LEVEL       int    `json:"LEVEL"`
			//DATA           map[int]float64 `json:"DATA"`
			DATA           interface{} `json:"DATA"`
			LASTHISTORICAL int         `json:"LAST_HISTORICAL,omitempty"`
			ISMAPPABLE     int         `json:"IS_MAPPABLE,omitempty"`
		} `json:"ROWS"`
	} `json:"VIEWSDATA"`
}

// EiaSteoNameMap 中英文互换
var EiaSteoNameMap = map[string]string{
	"PAPR_OECD":           "OECD石油产量",
	"PAPR_US":             "美国五十州石油产量",
	"PAPR_CA":             "加拿大石油产量",
	"PAPR_MX":             "墨西哥石油产量",
	"PAPR_OTHEROECD":      "其他OECD石油产量",
	"PAPR_NONOECD":        "非OECD石油产量",
	"PAPR_OPEC":           "OPEC13国石油产量",
	"COPR_OPEC":           "OPEC13国原油产量",
	"OPEC_NC":             "OPEC其他石油液体产量",
	"PAPR_FSU":            "欧亚石油产量",
	"PAPR_CH":             "中国石油产量",
	"PAPR_OTHER_NONOECD":  "其他非OECD石油产量",
	"PAPR_WORLD":          "全球石油产量",
	"PAPR_NONOPEC":        "非OPEC石油产量",
	"PATC_OECD":           "OECD石油需求",
	"PATC_US":             "美国五十州石油需求",
	"PATC_UST":            "美国领地石油需求",
	"PATC_CA":             "加拿大石油需求",
	"PATC_OECD_EUROPE":    "欧洲石油需求",
	"PATC_JA":             "日本石油需求",
	"PATC_OTHER_OECD":     "其他OECD石油需求",
	"PATC_NON_OECD":       "非OECD石油需求",
	"PATC_FSU":            "欧亚石油需求",
	"PATC_NONOECD_EUROPE": "非OECD欧洲石油需求",
	"PATC_CH":             "中国石油需求",
	"PATC_OTHER_ASIA":     "其他亚洲石油需求",
	"PATC_OTHER_NONOECD":  "其他非OECD石油需求",
	"PATC_WORLD":          "全球石油需求",
	"T3_STCHANGE_US":      "美国50州石油库存去化（需求-供给）",
	"T3_STCHANGE_OOECD":   "其他OECD石油库存去化（需求-供给）",
	"T3_STCHANGE_NOECD":   "非OECD石油库存去化（需求-供给）",
	"T3_STCHANGE_WORLD":   "全球石油库存去化（需求-供给）",
	"PASC_US":             "美国商业库存",
	"PASC_OECD_T3":        "OECD商业库存",
}