|
|
@@ -0,0 +1,354 @@
|
|
|
+package services
|
|
|
+
|
|
|
+import (
|
|
|
+ "encoding/json"
|
|
|
+ "fmt"
|
|
|
+ "hongze/hongze_data_crawler/models"
|
|
|
+ "rdluck_tools/http"
|
|
|
+ "strconv"
|
|
|
+ "time"
|
|
|
+)
|
|
|
+
|
|
|
+type EicListing struct {
|
|
|
+ Name string `json:"name"`
|
|
|
+ ShortName string `json:"short_name"`
|
|
|
+ Type string `json:"type"`
|
|
|
+ Eic string `json:"eic"`
|
|
|
+ Country string `json:"country"`
|
|
|
+ Url string `json:"url"`
|
|
|
+ Facilities []Facilities `json:"facilities"`
|
|
|
+}
|
|
|
+
|
|
|
+type Facilities struct {
|
|
|
+ Name string `json:"name"`
|
|
|
+ Type string `json:"type"`
|
|
|
+ Eic string `json:"eic"`
|
|
|
+ Country string `json:"country"`
|
|
|
+ Company string `json:"company"`
|
|
|
+ Url string `json:"url"`
|
|
|
+}
|
|
|
+type Storage struct {
|
|
|
+ Status string `json:"status"`
|
|
|
+ GasDayStartedOn string `json:"gasDayStartedOn"`
|
|
|
+ GasInStorage string `json:"gasInStorage"`
|
|
|
+ Full string `json:"full"`
|
|
|
+ Trend string `json:"trend"`
|
|
|
+ Injection string `json:"injection"`
|
|
|
+ Withdrawal string `json:"withdrawal"`
|
|
|
+ WorkingGasVolume string `json:"workingGasVolume"`
|
|
|
+ InjectionCapacity string `json:"injectionCapacity"`
|
|
|
+ WithdrawalCapacity string `json:"withdrawalCapacity"`
|
|
|
+ Info string `json:"info"`
|
|
|
+}
|
|
|
+
|
|
|
+var eicIndexCodeMap = make(map[string]string)
|
|
|
+
|
|
|
+func SyncStorageFromEic() {
|
|
|
+ allCode, err := models.GetIndexCodeFromMapping("Eic")
|
|
|
+ if err != nil {
|
|
|
+ fmt.Println("select Code err:", err)
|
|
|
+ }
|
|
|
+ for _, item := range allCode {
|
|
|
+ eicIndexCodeMap[item.IndexName] = item.IndexCode
|
|
|
+ }
|
|
|
+ baseUrl := "https://agsi.gie.eu/api/eic-listing/SSO/view"
|
|
|
+ body, err := http.Get(baseUrl)
|
|
|
+ if err != nil {
|
|
|
+ fmt.Println("GetData Err:" + err.Error())
|
|
|
+ return
|
|
|
+ }
|
|
|
+
|
|
|
+ var eicListing []EicListing
|
|
|
+ err = json.Unmarshal(body, &eicListing)
|
|
|
+ for _, sso := range eicListing {
|
|
|
+ var codeMapList []*models.BaseFromTradeMapping
|
|
|
+
|
|
|
+ existIndexMap := make(map[string]*models.BaseFromTradeEicIndex)
|
|
|
+ //获取所有指标信息
|
|
|
+ allIndex, err := models.GetSSOFromEicIndexAll(sso.Name)
|
|
|
+ if err != nil {
|
|
|
+ fmt.Println("select err:", err)
|
|
|
+ }
|
|
|
+ for _, v := range allIndex {
|
|
|
+ existIndexMap[v.GasDayStartedOn+v.Name] = v
|
|
|
+ }
|
|
|
+
|
|
|
+ ssoUrl := sso.Url
|
|
|
+ ssoBody, err := http.Get(ssoUrl)
|
|
|
+ fmt.Println("ssoUrl:", ssoUrl)
|
|
|
+ if err != nil {
|
|
|
+ fmt.Println("GetData Err:" + err.Error())
|
|
|
+ continue
|
|
|
+ }
|
|
|
+ var ssoStorages []Storage
|
|
|
+ err = json.Unmarshal(ssoBody, &ssoStorages)
|
|
|
+ //SSO item
|
|
|
+ var ssoItems []*models.BaseFromTradeEicIndex
|
|
|
+ for _, storageItem := range ssoStorages {
|
|
|
+ gasInStorage, _ := strconv.ParseFloat(storageItem.GasInStorage, 64)
|
|
|
+ full, _ := strconv.ParseFloat(storageItem.Full, 64)
|
|
|
+ trend, _ := strconv.ParseFloat(storageItem.Trend, 64)
|
|
|
+ injection, _ := strconv.ParseFloat(storageItem.Injection, 64)
|
|
|
+ withdrawal, _ := strconv.ParseFloat(storageItem.Withdrawal, 64)
|
|
|
+ workingGasVolume, _ := strconv.ParseFloat(storageItem.WorkingGasVolume, 64)
|
|
|
+ injectionCapacity, _ := strconv.ParseFloat(storageItem.InjectionCapacity, 64)
|
|
|
+ withdrawalCapacity, _ := strconv.ParseFloat(storageItem.WithdrawalCapacity, 64)
|
|
|
+ ssoItem := models.BaseFromTradeEicIndex{
|
|
|
+ BaseFromEicIndexId: 0,
|
|
|
+ Country: sso.Country,
|
|
|
+ Type: sso.Type,
|
|
|
+ EicCode: sso.Eic + sso.Country,
|
|
|
+ ShortName: sso.ShortName,
|
|
|
+ Name: sso.Name,
|
|
|
+ Status: storageItem.Status,
|
|
|
+ GasDayStartedOn: storageItem.GasDayStartedOn,
|
|
|
+ GasInStorage: gasInStorage,
|
|
|
+ Full: full,
|
|
|
+ Trend: trend,
|
|
|
+ Injection: injection,
|
|
|
+ Withdrawal: withdrawal,
|
|
|
+ WorkingGasVolume: workingGasVolume,
|
|
|
+ InjectionCapacity: injectionCapacity,
|
|
|
+ WithdrawalCapacity: withdrawalCapacity,
|
|
|
+ Info: storageItem.Info,
|
|
|
+ CreateTime: time.Now(),
|
|
|
+ ModifyTime: time.Now(),
|
|
|
+ }
|
|
|
+ itemVerifyCode := ssoItem.GasInStorage + ssoItem.Full + ssoItem.Trend + ssoItem.Injection + ssoItem.Withdrawal
|
|
|
+
|
|
|
+ if existIndex, ok := existIndexMap[ssoItem.GasDayStartedOn+ssoItem.Name]; !ok {
|
|
|
+ ssoItems = append(ssoItems, &ssoItem)
|
|
|
+
|
|
|
+ //newID, err := models.AddBaseFromEicIndex(&ssoItem)
|
|
|
+ //if err != nil {
|
|
|
+ // fmt.Println("insert error:", err)
|
|
|
+ //}
|
|
|
+ //fmt.Println("insert new indexID:", newID)
|
|
|
+ existIndexMap[ssoItem.GasDayStartedOn+ssoItem.Name] = &ssoItem
|
|
|
+ } else if existIndex != nil && itemVerifyCode != (existIndex.GasInStorage+existIndex.Full+existIndex.Trend+existIndex.Injection+existIndex.Withdrawal) {
|
|
|
+ //更新
|
|
|
+ err := models.ModifyBaseFromEicIndex(ssoItem.GasInStorage, ssoItem.Full, ssoItem.Trend, ssoItem.Injection, ssoItem.Withdrawal, existIndex.BaseFromEicIndexId)
|
|
|
+ if err != nil {
|
|
|
+ fmt.Println("data update err:", err)
|
|
|
+ }
|
|
|
+ }
|
|
|
+ }
|
|
|
+ if _, ok := eicIndexCodeMap[sso.Country+sso.Name]; !ok {
|
|
|
+ codeMappingItem := models.BaseFromTradeMapping{
|
|
|
+ BaseFromTradeMappingId: 0,
|
|
|
+ IndexName: sso.Country+sso.Name,
|
|
|
+ IndexCode: sso.Country + sso.Eic,
|
|
|
+ Exchange: "EIC",
|
|
|
+ }
|
|
|
+ codeMapList = append(codeMapList, &codeMappingItem)
|
|
|
+ eicIndexCodeMap[sso.Country+sso.Name] = sso.Country + sso.Eic
|
|
|
+ }
|
|
|
+
|
|
|
+ if len(ssoItems) != 0 {
|
|
|
+ successNums, err := models.AddEicDataMulti(ssoItems)
|
|
|
+ fmt.Println("SSO successNums:", successNums)
|
|
|
+ if err != nil {
|
|
|
+ fmt.Println("AddSSODataMulti err:", err)
|
|
|
+ }
|
|
|
+ }
|
|
|
+
|
|
|
+ facilities := sso.Facilities
|
|
|
+ var facItems []*models.BaseFromTradeEicIndex
|
|
|
+ for _, facility := range facilities {
|
|
|
+ //获取所有指标信息
|
|
|
+ allFacIndex, err := models.GetFacFromEicIndexAll(facility.Name)
|
|
|
+ if err != nil {
|
|
|
+ fmt.Println("select err:", err)
|
|
|
+ }
|
|
|
+ existFacIndexMap := make(map[string]*models.BaseFromTradeEicIndex)
|
|
|
+ for _, v := range allFacIndex {
|
|
|
+ existFacIndexMap[v.Name+v.GasDayStartedOn] = v
|
|
|
+ }
|
|
|
+ facUrl := facility.Url
|
|
|
+ facBody, err := http.Get(ssoUrl)
|
|
|
+ fmt.Println("facUrl:", facUrl)
|
|
|
+ if err != nil {
|
|
|
+ fmt.Println("GetData Err:" + err.Error())
|
|
|
+ continue
|
|
|
+ }
|
|
|
+ var facStorages []Storage
|
|
|
+ err = json.Unmarshal(facBody, &facStorages)
|
|
|
+
|
|
|
+ for _, storageItem := range facStorages {
|
|
|
+ gasInStorage, _ := strconv.ParseFloat(storageItem.GasInStorage, 64)
|
|
|
+ full, _ := strconv.ParseFloat(storageItem.Full, 64)
|
|
|
+ trend, _ := strconv.ParseFloat(storageItem.Trend, 64)
|
|
|
+ injection, _ := strconv.ParseFloat(storageItem.Injection, 64)
|
|
|
+ withdrawal, _ := strconv.ParseFloat(storageItem.Withdrawal, 64)
|
|
|
+ workingGasVolume, _ := strconv.ParseFloat(storageItem.WorkingGasVolume, 64)
|
|
|
+ injectionCapacity, _ := strconv.ParseFloat(storageItem.InjectionCapacity, 64)
|
|
|
+ withdrawalCapacity, _ := strconv.ParseFloat(storageItem.WithdrawalCapacity, 64)
|
|
|
+ facItem := models.BaseFromTradeEicIndex{
|
|
|
+ BaseFromEicIndexId: 0,
|
|
|
+ Country: facility.Country,
|
|
|
+ Type: facility.Type,
|
|
|
+ EicCode: facility.Eic + facility.Country,
|
|
|
+ ShortName: sso.ShortName,
|
|
|
+ Name: facility.Name,
|
|
|
+ Status: storageItem.Status,
|
|
|
+ GasDayStartedOn: storageItem.GasDayStartedOn,
|
|
|
+ GasInStorage: gasInStorage,
|
|
|
+ Full: full,
|
|
|
+ Trend: trend,
|
|
|
+ Injection: injection,
|
|
|
+ Withdrawal: withdrawal,
|
|
|
+ WorkingGasVolume: workingGasVolume,
|
|
|
+ InjectionCapacity: injectionCapacity,
|
|
|
+ WithdrawalCapacity: withdrawalCapacity,
|
|
|
+ Info: storageItem.Info,
|
|
|
+ CreateTime: time.Now(),
|
|
|
+ ModifyTime: time.Now(),
|
|
|
+ }
|
|
|
+ itemVerifyCode := facItem.GasInStorage + facItem.Full + facItem.Trend + facItem.Injection + facItem.Withdrawal
|
|
|
+
|
|
|
+ if existIndex, ok := existFacIndexMap[facItem.Name+facItem.GasDayStartedOn]; !ok {
|
|
|
+ facItems = append(facItems, &facItem)
|
|
|
+
|
|
|
+ //newID, err := models.AddBaseFromEicIndex(&facItem)
|
|
|
+ //if err != nil {
|
|
|
+ // fmt.Println("insert error:", err)
|
|
|
+ //}
|
|
|
+ //fmt.Println("insert new indexID:", newID)
|
|
|
+ existFacIndexMap[facItem.Name+facItem.GasDayStartedOn] = &facItem
|
|
|
+ } else if existIndex != nil && itemVerifyCode != (existIndex.GasInStorage+existIndex.Full+existIndex.Trend+existIndex.Injection+existIndex.Withdrawal) {
|
|
|
+ //更新
|
|
|
+ err := models.ModifyBaseFromEicIndex(facItem.GasInStorage, facItem.Full, facItem.Trend, facItem.Injection, facItem.Withdrawal, existIndex.BaseFromEicIndexId)
|
|
|
+ if err != nil {
|
|
|
+ fmt.Println("data update err:", err)
|
|
|
+ }
|
|
|
+ }
|
|
|
+ }
|
|
|
+ if _, ok := eicIndexCodeMap[facility.Country+facility.Name]; !ok {
|
|
|
+ codeMappingItem := models.BaseFromTradeMapping{
|
|
|
+ BaseFromTradeMappingId: 0,
|
|
|
+ IndexName: facility.Country+facility.Name,
|
|
|
+ IndexCode: facility.Country+facility.Eic,
|
|
|
+ Exchange: "EIC",
|
|
|
+ }
|
|
|
+ codeMapList = append(codeMapList, &codeMappingItem)
|
|
|
+ eicIndexCodeMap[facility.Country+facility.Name] = facility.Country+facility.Eic
|
|
|
+ }
|
|
|
+ }
|
|
|
+ if len(facItems) != 0 {
|
|
|
+ successNums, err := models.AddEicDataMulti(facItems)
|
|
|
+ fmt.Println("fac successNums:", successNums)
|
|
|
+ if err != nil {
|
|
|
+ fmt.Println("AddFacDataMulti err:", err)
|
|
|
+ }
|
|
|
+ }
|
|
|
+ successNums, err := models.AddEicCodeMulti(codeMapList)
|
|
|
+ fmt.Println("codeMapping successNums:", successNums)
|
|
|
+ if err != nil {
|
|
|
+ fmt.Println("AddEicCodeMulti err:", err)
|
|
|
+ }
|
|
|
+ }
|
|
|
+ //国家级，洲际级统计
|
|
|
+ var codeMapList []*models.BaseFromTradeMapping
|
|
|
+
|
|
|
+ Countries := []string{"eu", "AT", "BE", "BG", "HR", "CZ",
|
|
|
+ "DK", "FR", "DE", "HU", "IE", "IT", "LV", "NL", "PL",
|
|
|
+ "PT", "RO", "SK", "ES", "SE", "GB", "ne", "RS", "UA"}
|
|
|
+ for _, country := range Countries {
|
|
|
+ var countryItems []*models.BaseFromTradeEicIndex
|
|
|
+ //获取所有指标信息
|
|
|
+ allFacIndex, err := models.GetCountryFromEicIndexAll(country)
|
|
|
+ if err != nil {
|
|
|
+ fmt.Println("select err:", err)
|
|
|
+ }
|
|
|
+ existFacIndexMap := make(map[string]*models.BaseFromTradeEicIndex)
|
|
|
+ for _, v := range allFacIndex {
|
|
|
+ existFacIndexMap[v.Name+v.GasDayStartedOn] = v
|
|
|
+ }
|
|
|
+ countryUrl := fmt.Sprintf("https://agsi.gie.eu/api/data/%s", country)
|
|
|
+ fmt.Println("countryUrl:", countryUrl)
|
|
|
+ body, err := http.Get(countryUrl)
|
|
|
+ if err != nil {
|
|
|
+ fmt.Println("GetData Err:" + err.Error())
|
|
|
+ continue
|
|
|
+ }
|
|
|
+ var countryStorages []Storage
|
|
|
+ err = json.Unmarshal(body, &countryStorages)
|
|
|
+
|
|
|
+ for _, countryStorage := range countryStorages {
|
|
|
+ gasInStorage, _ := strconv.ParseFloat(countryStorage.GasInStorage, 64)
|
|
|
+ full, _ := strconv.ParseFloat(countryStorage.Full, 64)
|
|
|
+ trend, _ := strconv.ParseFloat(countryStorage.Trend, 64)
|
|
|
+ injection, _ := strconv.ParseFloat(countryStorage.Injection, 64)
|
|
|
+ withdrawal, _ := strconv.ParseFloat(countryStorage.Withdrawal, 64)
|
|
|
+ workingGasVolume, _ := strconv.ParseFloat(countryStorage.WorkingGasVolume, 64)
|
|
|
+ injectionCapacity, _ := strconv.ParseFloat(countryStorage.InjectionCapacity, 64)
|
|
|
+ withdrawalCapacity, _ := strconv.ParseFloat(countryStorage.WithdrawalCapacity, 64)
|
|
|
+ countryItem := models.BaseFromTradeEicIndex{
|
|
|
+ BaseFromEicIndexId: 0,
|
|
|
+ Country: country,
|
|
|
+ Type: "country",
|
|
|
+ EicCode: country,
|
|
|
+ ShortName: country,
|
|
|
+ Name: country,
|
|
|
+ Status: countryStorage.Status,
|
|
|
+ GasDayStartedOn: countryStorage.GasDayStartedOn,
|
|
|
+ GasInStorage: gasInStorage,
|
|
|
+ Full: full,
|
|
|
+ Trend: trend,
|
|
|
+ Injection: injection,
|
|
|
+ Withdrawal: withdrawal,
|
|
|
+ WorkingGasVolume: workingGasVolume,
|
|
|
+ InjectionCapacity: injectionCapacity,
|
|
|
+ WithdrawalCapacity: withdrawalCapacity,
|
|
|
+ Info: countryStorage.Info,
|
|
|
+ CreateTime: time.Now(),
|
|
|
+ ModifyTime: time.Now(),
|
|
|
+ }
|
|
|
+ if country == "eu" || country == "ne" {
|
|
|
+ countryItem.Type = "continent"
|
|
|
+ }
|
|
|
+ itemVerifyCode := countryItem.GasInStorage + countryItem.Full + countryItem.Trend + countryItem.Injection + countryItem.Withdrawal
|
|
|
+
|
|
|
+ if existIndex, ok := existFacIndexMap[countryItem.Name+countryItem.GasDayStartedOn]; !ok {
|
|
|
+ countryItems = append(countryItems, &countryItem)
|
|
|
+
|
|
|
+ //newID, err := models.AddBaseFromEicIndex(&countryItem)
|
|
|
+ //if err != nil {
|
|
|
+ // fmt.Println("insert error:", err)
|
|
|
+ //}
|
|
|
+ //fmt.Println("insert new indexID:", newID)
|
|
|
+ existFacIndexMap[countryItem.Name+countryItem.GasDayStartedOn] = &countryItem
|
|
|
+ } else if existIndex != nil && itemVerifyCode != (existIndex.GasInStorage+existIndex.Full+existIndex.Trend+existIndex.Injection+existIndex.Withdrawal) {
|
|
|
+ //更新
|
|
|
+ err := models.ModifyBaseFromEicIndex(countryItem.GasInStorage, countryItem.Full, countryItem.Trend, countryItem.Injection, countryItem.Withdrawal, existIndex.BaseFromEicIndexId)
|
|
|
+ if err != nil {
|
|
|
+ fmt.Println("data update err:", err)
|
|
|
+ }
|
|
|
+ }
|
|
|
+ }
|
|
|
+ if len(countryItems) != 0 {
|
|
|
+ successNums, err := models.AddEicDataMulti(countryItems)
|
|
|
+ fmt.Println("Country successNums:", successNums)
|
|
|
+ if err != nil {
|
|
|
+ fmt.Println("AddCountryDataMulti err:", err)
|
|
|
+ }
|
|
|
+
|
|
|
+ }
|
|
|
+
|
|
|
+ if _, ok := eicIndexCodeMap[country]; !ok {
|
|
|
+ codeMappingItem := models.BaseFromTradeMapping{
|
|
|
+ BaseFromTradeMappingId: 0,
|
|
|
+ IndexName: country,
|
|
|
+ IndexCode: country,
|
|
|
+ Exchange: "EIC",
|
|
|
+ }
|
|
|
+ codeMapList = append(codeMapList, &codeMappingItem)
|
|
|
+ eicIndexCodeMap[country] = country
|
|
|
+ }
|
|
|
+ }
|
|
|
+ successNums, err := models.AddEicCodeMulti(codeMapList)
|
|
|
+ fmt.Println("codeMapping successNums:", successNums)
|
|
|
+ if err != nil {
|
|
|
+ fmt.Println("AddEicCodeMulti err:", err)
|
|
|
+ }
|
|
|
+}
|
xingzai