eta_crawler/services/sso_eic.go

package services

import (
	"encoding/json"
	"fmt"
	"hongze/hongze_data_crawler/models"
	"rdluck_tools/http"
	"strconv"
	"time"
)

type EicListing struct {
	Name       string       `json:"name"`
	ShortName  string       `json:"short_name"`
	Type       string       `json:"type"`
	Eic        string       `json:"eic"`
	Country    string       `json:"country"`
	Url        string       `json:"url"`
	Facilities []Facilities `json:"facilities"`
}

type Facilities struct {
	Name    string `json:"name"`
	Type    string `json:"type"`
	Eic     string `json:"eic"`
	Country string `json:"country"`
	Company string `json:"company"`
	Url     string `json:"url"`
}
type Storage struct {
	Status             string `json:"status"`
	GasDayStartedOn    string `json:"gasDayStartedOn"`
	GasInStorage       string `json:"gasInStorage"`
	Full               string `json:"full"`
	Trend              string `json:"trend"`
	Injection          string `json:"injection"`
	Withdrawal         string `json:"withdrawal"`
	WorkingGasVolume   string `json:"workingGasVolume"`
	InjectionCapacity  string `json:"injectionCapacity"`
	WithdrawalCapacity string `json:"withdrawalCapacity"`
	Info               string `json:"info"`
}

var eicIndexCodeMap = make(map[string]string)

func SyncStorageFromEic() {
	allCode, err := models.GetIndexCodeFromMapping("Eic")
	if err != nil {
		fmt.Println("select Code err:", err)
	}
	for _, item := range allCode {
		eicIndexCodeMap[item.IndexName] = item.IndexCode
	}
	baseUrl := "https://agsi.gie.eu/api/eic-listing/SSO/view"
	body, err := http.Get(baseUrl)
	if err != nil {
		fmt.Println("GetData Err:" + err.Error())
		return
	}

	var eicListing []EicListing
	err = json.Unmarshal(body, &eicListing)
	for _, sso := range eicListing {
		var codeMapList []*models.BaseFromTradeMapping

		existIndexMap := make(map[string]*models.BaseFromTradeEicIndex)
		//获取所有指标信息
		allIndex, err := models.GetSSOFromEicIndexAll(sso.Name)
		if err != nil {
			fmt.Println("select err:", err)
		}
		for _, v := range allIndex {
			existIndexMap[v.GasDayStartedOn+v.Name] = v
		}

		ssoUrl := sso.Url
		ssoBody, err := http.Get(ssoUrl)
		fmt.Println("ssoUrl:", ssoUrl)
		if err != nil {
			fmt.Println("GetData Err:" + err.Error())
			continue
		}
		var ssoStorages []Storage
		err = json.Unmarshal(ssoBody, &ssoStorages)
		//SSO item
		var ssoItems []*models.BaseFromTradeEicIndex
		for _, storageItem := range ssoStorages {
			gasInStorage, _ := strconv.ParseFloat(storageItem.GasInStorage, 64)
			full, _ := strconv.ParseFloat(storageItem.Full, 64)
			trend, _ := strconv.ParseFloat(storageItem.Trend, 64)
			injection, _ := strconv.ParseFloat(storageItem.Injection, 64)
			withdrawal, _ := strconv.ParseFloat(storageItem.Withdrawal, 64)
			workingGasVolume, _ := strconv.ParseFloat(storageItem.WorkingGasVolume, 64)
			injectionCapacity, _ := strconv.ParseFloat(storageItem.InjectionCapacity, 64)
			withdrawalCapacity, _ := strconv.ParseFloat(storageItem.WithdrawalCapacity, 64)
			ssoItem := models.BaseFromTradeEicIndex{
				BaseFromEicIndexId: 0,
				Country:            sso.Country,
				Type:               sso.Type,
				EicCode:            sso.Eic + sso.Country,
				ShortName:          sso.ShortName,
				Name:               sso.Name,
				Status:             storageItem.Status,
				GasDayStartedOn:    storageItem.GasDayStartedOn,
				GasInStorage:       gasInStorage,
				Full:               full,
				Trend:              trend,
				Injection:          injection,
				Withdrawal:         withdrawal,
				WorkingGasVolume:   workingGasVolume,
				InjectionCapacity:  injectionCapacity,
				WithdrawalCapacity: withdrawalCapacity,
				Info:               storageItem.Info,
				CreateTime:         time.Now(),
				ModifyTime:         time.Now(),
			}
			itemVerifyCode := ssoItem.GasInStorage + ssoItem.Full + ssoItem.Trend + ssoItem.Injection + ssoItem.Withdrawal

			if existIndex, ok := existIndexMap[ssoItem.GasDayStartedOn+ssoItem.Name]; !ok {
				ssoItems = append(ssoItems, &ssoItem)

				//newID, err := models.AddBaseFromEicIndex(&ssoItem)
				//if err != nil {
				//	fmt.Println("insert error:", err)
				//}
				//fmt.Println("insert new indexID:", newID)
				existIndexMap[ssoItem.GasDayStartedOn+ssoItem.Name] = &ssoItem
			} else if existIndex != nil && itemVerifyCode != (existIndex.GasInStorage+existIndex.Full+existIndex.Trend+existIndex.Injection+existIndex.Withdrawal) {
				//更新
				err := models.ModifyBaseFromEicIndex(ssoItem.GasInStorage, ssoItem.Full, ssoItem.Trend, ssoItem.Injection, ssoItem.Withdrawal, existIndex.BaseFromEicIndexId)
				if err != nil {
					fmt.Println("data update err:", err)
				}
			}
		}
		if _, ok := eicIndexCodeMap[sso.Country+sso.Name]; !ok {
			codeMappingItem := models.BaseFromTradeMapping{
				BaseFromTradeMappingId: 0,
				IndexName:              sso.Country+sso.Name,
				IndexCode:              sso.Country + sso.Eic,
				Exchange:               "EIC",
			}
			codeMapList = append(codeMapList, &codeMappingItem)
			eicIndexCodeMap[sso.Country+sso.Name] = sso.Country + sso.Eic
		}

		if len(ssoItems) != 0 {
			successNums, err := models.AddEicDataMulti(ssoItems)
			fmt.Println("SSO successNums:", successNums)
			if err != nil {
				fmt.Println("AddSSODataMulti err:", err)
			}
		}

		facilities := sso.Facilities
		var facItems []*models.BaseFromTradeEicIndex
		for _, facility := range facilities {
			//获取所有指标信息
			allFacIndex, err := models.GetFacFromEicIndexAll(facility.Name)
			if err != nil {
				fmt.Println("select err:", err)
			}
			existFacIndexMap := make(map[string]*models.BaseFromTradeEicIndex)
			for _, v := range allFacIndex {
				existFacIndexMap[v.Name+v.GasDayStartedOn] = v
			}
			facUrl := facility.Url
			facBody, err := http.Get(ssoUrl)
			fmt.Println("facUrl:", facUrl)
			if err != nil {
				fmt.Println("GetData Err:" + err.Error())
				continue
			}
			var facStorages []Storage
			err = json.Unmarshal(facBody, &facStorages)

			for _, storageItem := range facStorages {
				gasInStorage, _ := strconv.ParseFloat(storageItem.GasInStorage, 64)
				full, _ := strconv.ParseFloat(storageItem.Full, 64)
				trend, _ := strconv.ParseFloat(storageItem.Trend, 64)
				injection, _ := strconv.ParseFloat(storageItem.Injection, 64)
				withdrawal, _ := strconv.ParseFloat(storageItem.Withdrawal, 64)
				workingGasVolume, _ := strconv.ParseFloat(storageItem.WorkingGasVolume, 64)
				injectionCapacity, _ := strconv.ParseFloat(storageItem.InjectionCapacity, 64)
				withdrawalCapacity, _ := strconv.ParseFloat(storageItem.WithdrawalCapacity, 64)
				facItem := models.BaseFromTradeEicIndex{
					BaseFromEicIndexId: 0,
					Country:            facility.Country,
					Type:               facility.Type,
					EicCode:            facility.Eic + facility.Country,
					ShortName:          sso.ShortName,
					Name:               facility.Name,
					Status:             storageItem.Status,
					GasDayStartedOn:    storageItem.GasDayStartedOn,
					GasInStorage:       gasInStorage,
					Full:               full,
					Trend:              trend,
					Injection:          injection,
					Withdrawal:         withdrawal,
					WorkingGasVolume:   workingGasVolume,
					InjectionCapacity:  injectionCapacity,
					WithdrawalCapacity: withdrawalCapacity,
					Info:               storageItem.Info,
					CreateTime:         time.Now(),
					ModifyTime:         time.Now(),
				}
				itemVerifyCode := facItem.GasInStorage + facItem.Full + facItem.Trend + facItem.Injection + facItem.Withdrawal

				if existIndex, ok := existFacIndexMap[facItem.Name+facItem.GasDayStartedOn]; !ok {
					facItems = append(facItems, &facItem)

					//newID, err := models.AddBaseFromEicIndex(&facItem)
					//if err != nil {
					//	fmt.Println("insert error:", err)
					//}
					//fmt.Println("insert new indexID:", newID)
					existFacIndexMap[facItem.Name+facItem.GasDayStartedOn] = &facItem
				} else if existIndex != nil && itemVerifyCode != (existIndex.GasInStorage+existIndex.Full+existIndex.Trend+existIndex.Injection+existIndex.Withdrawal) {
					//更新
					err := models.ModifyBaseFromEicIndex(facItem.GasInStorage, facItem.Full, facItem.Trend, facItem.Injection, facItem.Withdrawal, existIndex.BaseFromEicIndexId)
					if err != nil {
						fmt.Println("data update err:", err)
					}
				}
			}
			if _, ok := eicIndexCodeMap[facility.Country+facility.Name]; !ok {
				codeMappingItem := models.BaseFromTradeMapping{
					BaseFromTradeMappingId: 0,
					IndexName:              facility.Country+facility.Name,
					IndexCode:              facility.Country+facility.Eic,
					Exchange:               "EIC",
				}
				codeMapList = append(codeMapList, &codeMappingItem)
				eicIndexCodeMap[facility.Country+facility.Name] = facility.Country+facility.Eic
			}
		}
		if len(facItems) != 0 {
			successNums, err := models.AddEicDataMulti(facItems)
			fmt.Println("fac successNums:", successNums)
			if err != nil {
				fmt.Println("AddFacDataMulti err:", err)
			}
		}
		successNums, err := models.AddEicCodeMulti(codeMapList)
		fmt.Println("codeMapping successNums:", successNums)
		if err != nil {
			fmt.Println("AddEicCodeMulti err:", err)
		}
	}
	//国家级，洲际级统计
	var codeMapList []*models.BaseFromTradeMapping

	Countries := []string{"eu", "AT", "BE", "BG", "HR", "CZ",
		"DK", "FR", "DE", "HU", "IE", "IT", "LV", "NL", "PL",
		"PT", "RO", "SK", "ES", "SE", "GB", "ne", "RS", "UA"}
	for _, country := range Countries {
		var countryItems []*models.BaseFromTradeEicIndex
		//获取所有指标信息
		allFacIndex, err := models.GetCountryFromEicIndexAll(country)
		if err != nil {
			fmt.Println("select err:", err)
		}
		existFacIndexMap := make(map[string]*models.BaseFromTradeEicIndex)
		for _, v := range allFacIndex {
			existFacIndexMap[v.Name+v.GasDayStartedOn] = v
		}
		countryUrl := fmt.Sprintf("https://agsi.gie.eu/api/data/%s", country)
		fmt.Println("countryUrl:", countryUrl)
		body, err := http.Get(countryUrl)
		if err != nil {
			fmt.Println("GetData Err:" + err.Error())
			continue
		}
		var countryStorages []Storage
		err = json.Unmarshal(body, &countryStorages)

		for _, countryStorage := range countryStorages {
			gasInStorage, _ := strconv.ParseFloat(countryStorage.GasInStorage, 64)
			full, _ := strconv.ParseFloat(countryStorage.Full, 64)
			trend, _ := strconv.ParseFloat(countryStorage.Trend, 64)
			injection, _ := strconv.ParseFloat(countryStorage.Injection, 64)
			withdrawal, _ := strconv.ParseFloat(countryStorage.Withdrawal, 64)
			workingGasVolume, _ := strconv.ParseFloat(countryStorage.WorkingGasVolume, 64)
			injectionCapacity, _ := strconv.ParseFloat(countryStorage.InjectionCapacity, 64)
			withdrawalCapacity, _ := strconv.ParseFloat(countryStorage.WithdrawalCapacity, 64)
			countryItem := models.BaseFromTradeEicIndex{
				BaseFromEicIndexId: 0,
				Country:            country,
				Type:               "country",
				EicCode:            country,
				ShortName:          country,
				Name:               country,
				Status:             countryStorage.Status,
				GasDayStartedOn:    countryStorage.GasDayStartedOn,
				GasInStorage:       gasInStorage,
				Full:               full,
				Trend:              trend,
				Injection:          injection,
				Withdrawal:         withdrawal,
				WorkingGasVolume:   workingGasVolume,
				InjectionCapacity:  injectionCapacity,
				WithdrawalCapacity: withdrawalCapacity,
				Info:               countryStorage.Info,
				CreateTime:         time.Now(),
				ModifyTime:         time.Now(),
			}
			if country == "eu" || country == "ne" {
				countryItem.Type = "continent"
			}
			itemVerifyCode := countryItem.GasInStorage + countryItem.Full + countryItem.Trend + countryItem.Injection + countryItem.Withdrawal

			if existIndex, ok := existFacIndexMap[countryItem.Name+countryItem.GasDayStartedOn]; !ok {
				countryItems = append(countryItems, &countryItem)

				//newID, err := models.AddBaseFromEicIndex(&countryItem)
				//if err != nil {
				//	fmt.Println("insert error:", err)
				//}
				//fmt.Println("insert new indexID:", newID)
				existFacIndexMap[countryItem.Name+countryItem.GasDayStartedOn] = &countryItem
			} else if existIndex != nil && itemVerifyCode != (existIndex.GasInStorage+existIndex.Full+existIndex.Trend+existIndex.Injection+existIndex.Withdrawal) {
				//更新
				err := models.ModifyBaseFromEicIndex(countryItem.GasInStorage, countryItem.Full, countryItem.Trend, countryItem.Injection, countryItem.Withdrawal, existIndex.BaseFromEicIndexId)
				if err != nil {
					fmt.Println("data update err:", err)
				}
			}
		}
		if len(countryItems) != 0 {
			successNums, err := models.AddEicDataMulti(countryItems)
			fmt.Println("Country successNums:", successNums)
			if err != nil {
				fmt.Println("AddCountryDataMulti err:", err)
			}

		}

		if _, ok := eicIndexCodeMap[country]; !ok {
			codeMappingItem := models.BaseFromTradeMapping{
				BaseFromTradeMappingId: 0,
				IndexName:              country,
				IndexCode:              country,
				Exchange:               "EIC",
			}
			codeMapList = append(codeMapList, &codeMappingItem)
			eicIndexCodeMap[country] = country
		}
	}
	successNums, err := models.AddEicCodeMulti(codeMapList)
	fmt.Println("codeMapping successNums:", successNums)
	if err != nil {
		fmt.Println("AddEicCodeMulti err:", err)
	}
}