eta_crawler/services/eia_steo.go

package services

import (
	"context"
	"encoding/json"
	"eta/eta_crawler/models"
	"eta/eta_crawler/utils"
	"fmt"
	"reflect"
	"strconv"
	"strings"
	"time"

	"github.com/rdlucklib/rdluck_tools/http"
)

var noCrawlerName = map[string]struct{}{"Real Gross Domestic Product": {}, "Nominal U.S. Dollar Exchange Rate": {}}

func SyncEiaSteoData(cont context.Context) (err error) {
	// err = syncEiaSteoData()
	eiaSteoUrls := map[string]string{
		"International Petroleum and Other Liquids Production、Consumption、Inventories": `https://www.eia.gov/outlooks/steo/data/browser/data/index.php?v=6&f=M&s=0&id=&linechart=PAPR_OECD~PAPR_NONOPEC&maptype=0&ctype=linechart&map=&method=getData`,
		"Non-OPEC Petroleum and Other Liquids Production":                              `https://www.eia.gov/outlooks/steo/data/browser/data/index.php?v=29&f=M&s=0&start=201901&end=202512&id=&ctype=linechart&maptype=0&method=getData`,
		"Total Liquid Fuels Production":                                                `https://www.eia.gov/outlooks/steo/data/browser/data/index.php?v=7&f=M&s=0&start=201901&end=202512&maptype=0&ctype=linechart&id=&method=getData`,
		"Total Crude Oil Production":                                                   `https://www.eia.gov/outlooks/steo/data/browser/data/index.php?v=30&f=M&s=0&start=201901&end=202512&id=&ctype=linechart&maptype=0&method=getData`,
		"World Petroleum and Other Liquid Fuels Consumption":                           `https://www.eia.gov/outlooks/steo/data/browser/data/index.php?v=31&f=M&s=0&start=201901&end=202512&maptype=0&ctype=linechart&id=&method=getData`,
		"U.S. Petroleum and Other Liquids Supply、Consumption、Inventories":              `https://www.eia.gov/outlooks/steo/data/browser/data/index.php?v=9&f=M&s=0&start=201901&end=202512&maptype=0&ctype=linechart&method=getData`,
		"Drilling Productivity Metrics":                                                `https://www.eia.gov/outlooks/steo/data/browser/data/index.php?v=32&f=M&s=0&start=201901&end=202512&ctype=linechart&maptype=0&method=getData`,
		"Crude Oil and Natural Gas Production from Shale and Tight Formations":         `https://www.eia.gov/outlooks/steo/data/browser/data/index.php?v=33&f=M&s=0&start=201901&end=202512&maptype=0&ctype=linechart&method=getData`,
	}
	var eiaIndexName = []string{
		"International Petroleum and Other Liquids Production、Consumption、Inventories",
		"Non-OPEC Petroleum and Other Liquids Production",
		"Total Liquid Fuels Production",
		"Total Crude Oil Production",
		"World Petroleum and Other Liquid Fuels Consumption",
		"U.S. Petroleum and Other Liquids Supply、Consumption、Inventories",
		"Drilling Productivity Metrics",
		"Crude Oil and Natural Gas Production from Shale and Tight Formations",
	}

	for _, name := range eiaIndexName {
		url := eiaSteoUrls[name]
		err = syncEiaSteoDataV2(name, url)
		if err != nil {
			fmt.Println("同步失败", err)
			return
		}
	}
	return
}

func syncEiaSteoData(eiaSteoUrl string) (err error) {
	// 获取数据

	//官网地址：https://www.eia.gov/outlooks/steo/data/browser/#/?v=6&f=M&s=0&start=201701&end=202312&linechart=~T3_STCHANGE_WORLD&maptype=0&ctype=linechart&map=
	// 这是获取数据的链接（月度的）
	// eiaSteoUrl := "https://www.eia.gov/outlooks/steo/data/browser/data/index.php?v=6&f=M&s=0&id=&linechart=PAPR_OECD~PAPR_NONOPEC&maptype=0&ctype=linechart&map=&method=getData"
	eiaSteoData, err := queryData(eiaSteoUrl)
	if err != nil {
		fmt.Println("读取失败", err)
		return
	}

	// 获取分类列表
	classifyList, err := models.GetBaseFromEiaSteoClassifyAll()
	if err != nil {
		fmt.Println("获取分类失败：", err)
		return
	}
	classifyMap := make(map[string]*models.BaseFromEiaSteoClassify)
	for _, v := range classifyList {
		classifyMap[v.ClassifyNameOriginal] = v
	}

	// 获取指标列表
	indexList, err := models.GetBaseFromEiaSteoIndexAll()
	if err != nil {
		fmt.Println("获取分类失败：", err)
		return
	}
	indexMap := make(map[string]*models.BaseFromEiaSteoIndex)
	for _, v := range indexList {
		indexMap[v.IndexCode] = v
	}

	var nowClassify *models.BaseFromEiaSteoClassify
	for _, v := range eiaSteoData.VIEWSDATA.ROWS {
		// 如果没有数据，那么就返回
		if v.HASDATA != 1 {
			continue
		}

		if v.LEVEL == 1 {
			tmpNowClassify, ok := classifyMap[v.CHARTNAME]
			if !ok {
				nowClassify = &models.BaseFromEiaSteoClassify{
					BaseFromEiaSteoClassifyId: 0,
					ClassifyName:              v.CHARTNAME,
					ClassifyNameOriginal:      v.CHARTNAME,
					ModifyTime:                time.Now(),
					CreateTime:                time.Now(),
				}
				// 新增指标
				err = nowClassify.AddBaseFromEiaSteoClassify()
				if err != nil {
					return
				}
				classifyMap[v.CHARTNAME] = nowClassify
			} else {
				nowClassify = tmpNowClassify
			}
			//continue
		}
		// 如果系列名称为空的话，那么也返回
		if v.SERIESID == `` {
			continue
		}

		if v.LEVEL > 1 && v.SERIESID == `` {
			continue
		}

		var eiaSteoIndex *models.BaseFromEiaSteoIndex

		eiaSteoIndex, ok := indexMap[v.SERIESID]

		if !ok {
			eiaSteoIndex = &models.BaseFromEiaSteoIndex{
				//BaseFromEiaSteoIndexId:    0,
				BaseFromEiaSteoClassifyId: nowClassify.BaseFromEiaSteoClassifyId,
				IndexCode:                 v.SERIESID,
				IndexName:                 v.CHARTNAME, // 不做中文名称的转换
				IndexNameOriginal:         v.CHARTNAME,
				Frequency:                 "月度",
				Level:                     v.LEVEL,
				Unit:                      v.UNITS,
				Super:                     v.SUPER,
				Precision:                 v.PRECISION,
				LastHistorical:            strconv.Itoa(v.LASTHISTORICAL),
				Description:               v.DESCRIPTION,
				IsMappable:                v.ISMAPPABLE,
				StartDate:                 time.Now(),
				EndDate:                   time.Now(),
				ModifyTime:                time.Now(),
				CreateTime:                time.Now(),
			}
			// 新增指标
			err = eiaSteoIndex.Add()
			if err != nil {
				return
			}
			indexMap[v.SERIESID] = eiaSteoIndex
		} else {
			updateCol := make([]string, 0)
			if eiaSteoIndex.BaseFromEiaSteoClassifyId != nowClassify.BaseFromEiaSteoClassifyId {
				eiaSteoIndex.BaseFromEiaSteoClassifyId = nowClassify.BaseFromEiaSteoClassifyId
				updateCol = append(updateCol, "BaseFromEiaSteoClassifyId")
			}
			if eiaSteoIndex.IndexName != v.CHARTNAME {
				eiaSteoIndex.IndexName = v.CHARTNAME
				updateCol = append(updateCol, "IndexName")
			}
			if eiaSteoIndex.IndexNameOriginal != v.CHARTNAME {
				eiaSteoIndex.IndexNameOriginal = v.CHARTNAME
				updateCol = append(updateCol, "IndexNameOriginal")
			}
			//	Frequency:                  "月度",
			//	Level:                      v.LEVEL,
			//	Unit:                       v.UNITS,
			//	Super:                      v.SUPER,
			//	Precision:                  v.PRECISION,
			//	LastHistorical:             strconv.Itoa(v.LASTHISTORICAL),
			//	Description:                v.DESCRIPTION,
			//	IsMappable:                 v.ISMAPPABLE,
			if len(updateCol) > 0 {
				eiaSteoIndex.Update(updateCol)
			}

		}

		//校验数据类型对不对
		valType := reflect.TypeOf(v.DATA)
		//if valType.String() == "map[string]interface{}"{
		//	fmt.Println("匹配上了")
		//}
		switch valType.String() {
		case "[]interface {}": // 没有数据
		case "map[string]interface {}": // 有数据
			data := v.DATA.(map[string]interface{})
			err = models.HandleEiaSteoData(data, eiaSteoIndex)
			if err != nil {
				return
			}
			//if eiaSteoIndex.IndexCode == "PATC_OECD_EUROPE" {
			//	os.Exit(0)
			//}
		}

		fmt.Println(v.CHARTNAME, v.SERIESID, "==", v.PRECISION, "===========")
	}
	return
}

func syncEiaSteoDataV2(eiaSteoName, eiaSteoUrl string) (err error) {
	defer func() {
		if r := recover(); r != nil {
			fmt.Println("异常：", r)
			fmt.Println("异常的eiaSteo名字", eiaSteoName)
		}
	}()
	// 获取数据

	//官网地址：https://www.eia.gov/outlooks/steo/data/browser/#/?v=6&f=M&s=0&start=201701&end=202312&linechart=~T3_STCHANGE_WORLD&maptype=0&ctype=linechart&map=
	// 这是获取数据的链接（月度的）
	// eiaSteoUrl := "https://www.eia.gov/outlooks/steo/data/browser/data/index.php?v=6&f=M&s=0&id=&linechart=PAPR_OECD~PAPR_NONOPEC&maptype=0&ctype=linechart&map=&method=getData"
	eiaSteoData, err := queryData(eiaSteoUrl)
	if err != nil {
		fmt.Println("读取失败", err)
		return
	}

	parentClassify, err := models.GetBaseFromEiaSteoClassifyByName(eiaSteoName)
	if err != nil && err.Error() != utils.ErrNoRow() {
		fmt.Println("添加eiaSteo指标异常, err: ", err.Error())
		return
	}
	if parentClassify == nil {
		tmpClassify := &models.BaseFromEiaSteoClassify{
			ClassifyName:         eiaSteoName,
			ClassifyNameOriginal: eiaSteoName,
			Level:                1,
			ModifyTime:           time.Now(),
			CreateTime:           time.Now(),
		}
		// 新增指标分类
		err = tmpClassify.AddBaseFromEiaSteoClassify()
		if err != nil {
			return
		}
		parentClassify = tmpClassify
	}

	// 获取分类列表
	classifyList, err := models.GetChildBaseFromEiaSteoClassifyById(parentClassify.BaseFromEiaSteoClassifyId)
	if err != nil {
		fmt.Println("获取分类失败：", err)
		return
	}
	classifyMap := make(map[string]*models.BaseFromEiaSteoClassify)
	for _, v := range classifyList {
		classifyMap[v.ClassifyNameOriginal] = v
	}
	// 获取指标列表
	indexList, err := models.GetBaseFromEiaSteoIndexAll()
	if err != nil {
		fmt.Println("获取分类失败：", err)
		return
	}
	indexMap := make(map[string]*models.BaseFromEiaSteoIndex)
	for _, v := range indexList {
		indexMap[v.IndexCode] = v
	}

	var hasClassify *models.BaseFromEiaSteoClassify
	var noDataClassify *models.BaseFromEiaSteoClassify
	for i, v := range eiaSteoData.VIEWSDATA.ROWS {
		// 如果没有数据，那么就返回
		if v.HASDATA != 1 && strings.TrimSpace(v.CHARTNAME) == "" {
			curRow := eiaSteoData.VIEWSDATA.ROWS
			length := len(eiaSteoData.VIEWSDATA.ROWS)
			if length > i+1 {
				if noDataClassify != nil {
					if curRow[i+1].LEVEL == 1 && strings.TrimSpace(curRow[i+1].CHARTNAME) != "" && curRow[i+1].HASDATA != 1 {
						noDataClassify = nil
					}
				}
				if hasClassify != nil {
					if curRow[i+1].LEVEL == 1 && strings.TrimSpace(curRow[i+1].CHARTNAME) != "" && curRow[i+1].HASDATA == 1 {
						hasClassify = nil
					}
				}
			}
			continue
		}
		if noDataClassify == nil && hasClassify == nil && strings.TrimSpace(v.CHARTNAME) != "" && v.LEVEL == 1 {
			classify, ok := classifyMap[v.CHARTNAME]
			if !ok {
				tmpClassify := &models.BaseFromEiaSteoClassify{
					BaseFromEiaSteoClassifyId: 0,
					ClassifyName:              v.CHARTNAME,
					ClassifyNameOriginal:      v.CHARTNAME,
					ParentId:                  parentClassify.BaseFromEiaSteoClassifyId,
					Level:                     2,
					ModifyTime:                time.Now(),
					CreateTime:                time.Now(),
				}
				classifyMap[v.CHARTNAME] = tmpClassify
				if v.HASDATA == 1 {
					hasClassify = tmpClassify
				} else {
					noDataClassify = tmpClassify
				}
				if _, ok := noCrawlerName[tmpClassify.ClassifyName]; ok {
					continue
				}
				// 新增指标分类
				err = tmpClassify.AddBaseFromEiaSteoClassify()
				if err != nil {
					return
				}
			} else {
				if v.HASDATA == 1 {
					hasClassify = classify
				} else {
					noDataClassify = classify
				}
			}
		}
		// 如果系列名称为空的话，那么也返回
		if v.SERIESID == `` {
			continue
		}
		if v.HASDATA != 1 {
			continue
		}

		var curClassify *models.BaseFromEiaSteoClassify
		if noDataClassify != nil {
			curClassify = noDataClassify
		} else {
			curClassify = hasClassify
		}
		if _, ok := noCrawlerName[curClassify.ClassifyName]; ok {
			continue
		}

		eiaSteoIndex, ok := indexMap[v.SERIESID]
		if !ok {
			eiaSteoIndex = &models.BaseFromEiaSteoIndex{
				//BaseFromEiaSteoIndexId:    0,
				BaseFromEiaSteoClassifyId: curClassify.BaseFromEiaSteoClassifyId,
				IndexCode:                 v.SERIESID,
				IndexName:                 v.CHARTNAME, // 不做中文名称的转换
				IndexNameOriginal:         v.CHARTNAME,
				Frequency:                 "月度",
				Level:                     v.LEVEL,
				Unit:                      v.UNITS,
				Super:                     v.SUPER,
				Precision:                 v.PRECISION,
				LastHistorical:            strconv.Itoa(v.LASTHISTORICAL),
				Description:               v.DESCRIPTION,
				IsMappable:                v.ISMAPPABLE,
				StartDate:                 time.Now(),
				EndDate:                   time.Now(),
				ModifyTime:                time.Now(),
				CreateTime:                time.Now(),
			}
			// 新增指标
			err = eiaSteoIndex.Add()
			if err != nil {
				return
			}
			indexMap[v.SERIESID] = eiaSteoIndex
		} else {
			updateCol := make([]string, 0)
			if eiaSteoIndex.BaseFromEiaSteoClassifyId != curClassify.BaseFromEiaSteoClassifyId {
				eiaSteoIndex.BaseFromEiaSteoClassifyId = curClassify.BaseFromEiaSteoClassifyId
				updateCol = append(updateCol, "BaseFromEiaSteoClassifyId")
			}
			if eiaSteoIndex.IndexName != v.CHARTNAME {
				eiaSteoIndex.IndexName = v.CHARTNAME
				updateCol = append(updateCol, "IndexName")
			}
			if eiaSteoIndex.IndexNameOriginal != v.CHARTNAME {
				eiaSteoIndex.IndexNameOriginal = v.CHARTNAME
				updateCol = append(updateCol, "IndexNameOriginal")
			}
			if len(updateCol) > 0 {
				eiaSteoIndex.Update(updateCol)
			}

		}

		//校验数据类型对不对
		valType := reflect.TypeOf(v.DATA)
		switch valType.String() {
		case "[]interface {}": // 没有数据
		case "map[string]interface {}": // 有数据
			data := v.DATA.(map[string]interface{})
			err = models.HandleEiaSteoData(data, eiaSteoIndex)
			if err != nil {
				return
			}
		}

		fmt.Println(v.CHARTNAME, v.SERIESID, "==", v.PRECISION, "===========")
	}
	return
}

// queryData 接口请求网站数据
func queryData(eiaSteoUrl string) (eiaSteoData EiaSteoData, err error) {
	body, err := http.Get(eiaSteoUrl)
	if err != nil {
		fmt.Println("err:", err)
	}
	//utils.FileLog.Info("eia steo 报告数据：" + string(body))
	err = json.Unmarshal(body, &eiaSteoData)
	return
}

type EiaSteoData struct {
	SERIESDATA struct {
		DATACOLUMNS []interface{} `json:"DATACOLUMNS"`
		ROWS        []interface{} `json:"ROWS"`
	} `json:"SERIESDATA"`
	VIEWSDATA struct {
		DATACOLUMNS []int `json:"DATACOLUMNS"`
		ROWS        []struct {
			CHARTNAME   string `json:"CHART_NAME"`
			DESCRIPTION string `json:"DESCRIPTION"`
			HASDATA     int    `json:"HAS_DATA"`
			PRECISION   int    `json:"PRECISION"`
			SERIESID    string `json:"SERIES_ID"`
			UNITS       string `json:"UNITS"`
			SUPER       string `json:"SUPER"`
			LEVEL       int    `json:"LEVEL"`
			//DATA           map[int]float64 `json:"DATA"`
			DATA           interface{} `json:"DATA"`
			LASTHISTORICAL int         `json:"LAST_HISTORICAL,omitempty"`
			ISMAPPABLE     int         `json:"IS_MAPPABLE,omitempty"`
		} `json:"ROWS"`
	} `json:"VIEWSDATA"`
}

// EiaSteoNameMap 中英文互换
var EiaSteoNameMap = map[string]string{
	"PAPR_OECD":           "OECD石油产量",
	"PAPR_US":             "美国五十州石油产量",
	"PAPR_CA":             "加拿大石油产量",
	"PAPR_MX":             "墨西哥石油产量",
	"PAPR_OTHEROECD":      "其他OECD石油产量",
	"PAPR_NONOECD":        "非OECD石油产量",
	"PAPR_OPEC":           "OPEC13国石油产量",
	"COPR_OPEC":           "OPEC13国原油产量",
	"OPEC_NC":             "OPEC其他石油液体产量",
	"PAPR_FSU":            "欧亚石油产量",
	"PAPR_CH":             "中国石油产量",
	"PAPR_OTHER_NONOECD":  "其他非OECD石油产量",
	"PAPR_WORLD":          "全球石油产量",
	"PAPR_NONOPEC":        "非OPEC石油产量",
	"PATC_OECD":           "OECD石油需求",
	"PATC_US":             "美国五十州石油需求",
	"PATC_UST":            "美国领地石油需求",
	"PATC_CA":             "加拿大石油需求",
	"PATC_OECD_EUROPE":    "欧洲石油需求",
	"PATC_JA":             "日本石油需求",
	"PATC_OTHER_OECD":     "其他OECD石油需求",
	"PATC_NON_OECD":       "非OECD石油需求",
	"PATC_FSU":            "欧亚石油需求",
	"PATC_NONOECD_EUROPE": "非OECD欧洲石油需求",
	"PATC_CH":             "中国石油需求",
	"PATC_OTHER_ASIA":     "其他亚洲石油需求",
	"PATC_OTHER_NONOECD":  "其他非OECD石油需求",
	"PATC_WORLD":          "全球石油需求",
	"T3_STCHANGE_US":      "美国50州石油库存去化（需求-供给）",
	"T3_STCHANGE_OOECD":   "其他OECD石油库存去化（需求-供给）",
	"T3_STCHANGE_NOECD":   "非OECD石油库存去化（需求-供给）",
	"T3_STCHANGE_WORLD":   "全球石油库存去化（需求-供给）",
	"PASC_US":             "美国商业库存",
	"PASC_OECD_T3":        "OECD商业库存",
}