package services import ( "encoding/json" "eta/eta_crawler/models" "eta/eta_crawler/utils" "fmt" "io/ioutil" "net/http" "strconv" "time" ) type EicData struct { LastPage int `json:"last_page"` Total int `json:"total"` DataSet string `json:"dataset"` GasDay string `json:"gas_day"` Data []Children `json:"data"` Error string `json:"error"` Message string `json:"message"` } type Children struct { Name string `json:"name"` Code string `json:"code"` URL string `json:"url"` GasDayStart string `json:"gasDayStart"` GasInStorage string `json:"gasInStorage"` Consumption string `json:"consumption"` ConsumptionFull string `json:"consumptionFull"` Injection string `json:"injection"` Withdrawal string `json:"withdrawal"` WorkingGasVolume string `json:"workingGasVolume"` InjectionCapacity string `json:"injectionCapacity"` WithdrawalCapacity string `json:"withdrawalCapacity"` Status string `json:"status"` Trend string `json:"trend"` Full string `json:"full"` Info string `json:"-"` Children []Children `json:"children"` } type Storage1 struct { Status string `json:"status"` GasDayStartedOn string `json:"gasDayStartedOn"` GasInStorage string `json:"gasInStorage"` Full string `json:"full"` Trend string `json:"trend"` Injection string `json:"injection"` Withdrawal string `json:"withdrawal"` WorkingGasVolume string `json:"workingGasVolume"` InjectionCapacity string `json:"injectionCapacity"` WithdrawalCapacity string `json:"withdrawalCapacity"` Info string `json:"info"` } var eicIndexCodeMapV2 = make(map[string]string) func SyncStorageFromEicV2() { allCode, err := models.GetIndexCodeFromMapping("Eic") if err != nil { fmt.Println("select Code err:", err) } for _, item := range allCode { eicIndexCodeMapV2[item.IndexName] = item.IndexCode } for i := 10; i >= 0; i-- { baseUrl := "https://agsi.gie.eu/api?date=%s" //t, _ := time.Parse(utils.FormatDate, "2022-08-15") //st := t.AddDate(0, 0, i).Format(utils.FormatDate) st := time.Now().AddDate(0, 0, -i).Format(utils.FormatDate) tt, _ := time.Parse(utils.FormatDate, st) now := time.Now() if tt.After(now) { break } baseUrl = fmt.Sprintf(baseUrl, st) client := http.Client{} req, err := http.NewRequest(http.MethodGet, baseUrl, nil) if err != nil { fmt.Println("err") } // 添加请求头 req.Header.Add("Content-type", "application/json;charset=utf-8") req.Header.Add("x-key", "3669f7349c0598d50948429d6ea60070") // 发送请求 resp, err := client.Do(req) if err != nil { fmt.Println("err") } defer resp.Body.Close() body, err := ioutil.ReadAll(resp.Body) if err != nil { fmt.Println("err") } //body, err := http.Get(baseUrl) // //if err != nil { // fmt.Println("GetData Err:" + err.Error()) // return //} fmt.Println("URL:", baseUrl) var eicData EicData err = json.Unmarshal(body, &eicData) if err != nil { fmt.Println("Unmarshal Err:" + err.Error()) return } //洲际级 var insertItems []*models.BaseFromTradeEicIndexV2 for _, continent := range eicData.Data { var codeMapList []*models.BaseFromTradeMapping //为保持唯一性,code使用md5加密url code := utils.MD5(continent.URL) gasInStorage, _ := strconv.ParseFloat(continent.GasInStorage, 64) full, _ := strconv.ParseFloat(continent.Full, 64) trend, _ := strconv.ParseFloat(continent.Trend, 64) injection, _ := strconv.ParseFloat(continent.Injection, 64) withdrawal, _ := strconv.ParseFloat(continent.Withdrawal, 64) workingGasVolume, _ := strconv.ParseFloat(continent.WorkingGasVolume, 64) injectionCapacity, _ := strconv.ParseFloat(continent.InjectionCapacity, 64) withdrawalCapacity, _ := strconv.ParseFloat(continent.WithdrawalCapacity, 64) continentItem := models.BaseFromTradeEicIndexV2{ BaseFromEicIndexId: 0, Type: "continent", EicCode: code, Name: continent.Name, Status: continent.Status, GasDayStart: continent.GasDayStart, GasInStorage: gasInStorage, GasInStorageCode: code + "GS", Consumption: continent.Consumption, ConsumptionCode: code + "C", ConsumptionFull: continent.ConsumptionFull, ConsumptionFullCode: code + "CF", Full: full, FullCode: code + "F", Trend: trend, TrendCode: code + "T", Injection: injection, InjectionCode: code + "In", Withdrawal: withdrawal, WithdrawalCode: code + "Out", WorkingGasVolume: workingGasVolume, WorkingGasVolumeCode: code + "WGV", InjectionCapacity: injectionCapacity, InjectionCapacityCode: code + "IC", WithdrawalCapacity: withdrawalCapacity, WithdrawalCapacityCode: code + "WC", Info: continent.Info, CreateTime: time.Now(), ModifyTime: time.Now(), } //校验 existIndexMap := make(map[string]*models.BaseFromTradeEicIndexV2) allIndex, err := models.GetContinentFromEicIndexAllV2(continent.GasDayStart) if err != nil { fmt.Println("select err:", err) } for _, v := range allIndex { existIndexMap[v.GasDayStart+v.EicCode] = v } itemVerifyCode := continentItem.GasInStorage + continentItem.Full + continentItem.Trend + continentItem.Injection + continentItem.Withdrawal if existIndex, ok := existIndexMap[continentItem.GasDayStart+continentItem.EicCode]; !ok { //新增 if continentItem.Status != "N" { insertItems = append(insertItems, &continentItem) existIndexMap[continentItem.GasDayStart+continentItem.EicCode] = &continentItem } } else if existIndex != nil && itemVerifyCode != (existIndex.GasInStorage+existIndex.Full+existIndex.Trend+existIndex.Injection+existIndex.Withdrawal) { //更新 if continentItem.Status != "N" { err := models.ModifyBaseFromEicIndexV2(continentItem.GasInStorage, continentItem.Full, continentItem.Trend, continentItem.Injection, continentItem.Withdrawal, existIndex.BaseFromEicIndexId) if err != nil { fmt.Println("data update err:", err) } } } if _, ok := eicIndexCodeMapV2[continentItem.Name]; !ok { codeMappingItem := models.BaseFromTradeMapping{ BaseFromTradeMappingId: 0, IndexName: continentItem.Name, IndexCode: continentItem.EicCode, Exchange: "EIC", } codeMapList = append(codeMapList, &codeMappingItem) eicIndexCodeMapV2[continentItem.Name] = continentItem.EicCode } //国家级 for _, country := range continent.Children { //为保持唯一性,code使用md5加密url code := utils.MD5(country.URL) gasInStorage, _ := strconv.ParseFloat(country.GasInStorage, 64) full, _ := strconv.ParseFloat(country.Full, 64) trend, _ := strconv.ParseFloat(country.Trend, 64) injection, _ := strconv.ParseFloat(country.Injection, 64) withdrawal, _ := strconv.ParseFloat(country.Withdrawal, 64) workingGasVolume, _ := strconv.ParseFloat(country.WorkingGasVolume, 64) injectionCapacity, _ := strconv.ParseFloat(country.InjectionCapacity, 64) withdrawalCapacity, _ := strconv.ParseFloat(country.WithdrawalCapacity, 64) countryItem := models.BaseFromTradeEicIndexV2{ BaseFromEicIndexId: 0, Type: "country", EicCode: code, Name: country.Name, Status: country.Status, GasDayStart: country.GasDayStart, GasInStorage: gasInStorage, GasInStorageCode: code + "GS", Consumption: country.Consumption, ConsumptionCode: code + "C", ConsumptionFull: country.ConsumptionFull, ConsumptionFullCode: code + "CF", Full: full, FullCode: code + "F", Trend: trend, TrendCode: code + "T", Injection: injection, InjectionCode: code + "In", Withdrawal: withdrawal, WithdrawalCode: code + "Out", WorkingGasVolume: workingGasVolume, WorkingGasVolumeCode: code + "WGV", InjectionCapacity: injectionCapacity, InjectionCapacityCode: code + "IC", WithdrawalCapacity: withdrawalCapacity, WithdrawalCapacityCode: code + "WC", Info: country.Info, Parent: continent.Name, CreateTime: time.Now(), ModifyTime: time.Now(), } //校验 existIndexMap := make(map[string]*models.BaseFromTradeEicIndexV2) allIndex, err := models.GetCountryFromEicIndexAllV2(country.GasDayStart) if err != nil { fmt.Println("select err:", err) } for _, v := range allIndex { existIndexMap[v.GasDayStart+v.EicCode] = v } itemVerifyCode := countryItem.GasInStorage + countryItem.Full + countryItem.Trend + countryItem.Injection + countryItem.Withdrawal if existIndex, ok := existIndexMap[countryItem.GasDayStart+countryItem.EicCode]; !ok { //新增 if countryItem.Status != "N" { insertItems = append(insertItems, &countryItem) existIndexMap[countryItem.GasDayStart+countryItem.EicCode] = &countryItem } } else if existIndex != nil && itemVerifyCode != (existIndex.GasInStorage+existIndex.Full+existIndex.Trend+existIndex.Injection+existIndex.Withdrawal) { //更新 if countryItem.Status != "N" { err := models.ModifyBaseFromEicIndexV2(countryItem.GasInStorage, countryItem.Full, countryItem.Trend, countryItem.Injection, countryItem.Withdrawal, existIndex.BaseFromEicIndexId) if err != nil { fmt.Println("data update err:", err) } } } if _, ok := eicIndexCodeMapV2[countryItem.Name]; !ok { codeMappingItem := models.BaseFromTradeMapping{ BaseFromTradeMappingId: 0, IndexName: countryItem.Name, IndexCode: countryItem.EicCode, Exchange: "EIC", } codeMapList = append(codeMapList, &codeMappingItem) eicIndexCodeMapV2[countryItem.Name] = countryItem.EicCode } //厂商级 for _, sso := range country.Children { //为保持唯一性,code使用md5加密url code := utils.MD5(sso.URL) gasInStorage, _ := strconv.ParseFloat(sso.GasInStorage, 64) full, _ := strconv.ParseFloat(sso.Full, 64) trend, _ := strconv.ParseFloat(sso.Trend, 64) injection, _ := strconv.ParseFloat(sso.Injection, 64) withdrawal, _ := strconv.ParseFloat(sso.Withdrawal, 64) workingGasVolume, _ := strconv.ParseFloat(sso.WorkingGasVolume, 64) injectionCapacity, _ := strconv.ParseFloat(sso.InjectionCapacity, 64) withdrawalCapacity, _ := strconv.ParseFloat(sso.WithdrawalCapacity, 64) ssoItem := models.BaseFromTradeEicIndexV2{ BaseFromEicIndexId: 0, Type: "sso", EicCode: code, Name: sso.Name, Status: sso.Status, GasDayStart: sso.GasDayStart, GasInStorage: gasInStorage, GasInStorageCode: code + "GS", Consumption: sso.Consumption, ConsumptionCode: code + "C", ConsumptionFull: sso.ConsumptionFull, ConsumptionFullCode: code + "CF", Full: full, FullCode: code + "F", Trend: trend, TrendCode: code + "T", Injection: injection, InjectionCode: code + "In", Withdrawal: withdrawal, WithdrawalCode: code + "Out", WorkingGasVolume: workingGasVolume, WorkingGasVolumeCode: code + "WGV", InjectionCapacity: injectionCapacity, InjectionCapacityCode: code + "IC", WithdrawalCapacity: withdrawalCapacity, WithdrawalCapacityCode: code + "WC", Info: sso.Info, Parent: country.Name, CreateTime: time.Now(), ModifyTime: time.Now(), } //校验 existIndexMap := make(map[string]*models.BaseFromTradeEicIndexV2) allIndex, err := models.GetSSOFromEicIndexAllV2(sso.GasDayStart) if err != nil { fmt.Println("select err:", err) } for _, v := range allIndex { existIndexMap[v.GasDayStart+v.EicCode] = v } itemVerifyCode := ssoItem.GasInStorage + ssoItem.Full + ssoItem.Trend + ssoItem.Injection + ssoItem.Withdrawal if existIndex, ok := existIndexMap[ssoItem.GasDayStart+ssoItem.EicCode]; !ok { //新增 if ssoItem.Status != "N" { insertItems = append(insertItems, &ssoItem) existIndexMap[ssoItem.GasDayStart+ssoItem.EicCode] = &ssoItem } } else if existIndex != nil && itemVerifyCode != (existIndex.GasInStorage+existIndex.Full+existIndex.Trend+existIndex.Injection+existIndex.Withdrawal) { //更新 if ssoItem.Status != "N" { err := models.ModifyBaseFromEicIndexV2(ssoItem.GasInStorage, ssoItem.Full, ssoItem.Trend, ssoItem.Injection, ssoItem.Withdrawal, existIndex.BaseFromEicIndexId) if err != nil { fmt.Println("data update err:", err) } } } if _, ok := eicIndexCodeMapV2[ssoItem.Name]; !ok { codeMappingItem := models.BaseFromTradeMapping{ BaseFromTradeMappingId: 0, IndexName: ssoItem.Name, IndexCode: ssoItem.EicCode, Exchange: "EIC", } codeMapList = append(codeMapList, &codeMappingItem) eicIndexCodeMapV2[ssoItem.Name] = ssoItem.EicCode } //设施级 for _, fac := range sso.Children { //为保持唯一性,code使用md5加密url code := utils.MD5(fac.URL) gasInStorage, _ := strconv.ParseFloat(fac.GasInStorage, 64) full, _ := strconv.ParseFloat(fac.Full, 64) trend, _ := strconv.ParseFloat(fac.Trend, 64) injection, _ := strconv.ParseFloat(fac.Injection, 64) withdrawal, _ := strconv.ParseFloat(fac.Withdrawal, 64) workingGasVolume, _ := strconv.ParseFloat(fac.WorkingGasVolume, 64) injectionCapacity, _ := strconv.ParseFloat(fac.InjectionCapacity, 64) withdrawalCapacity, _ := strconv.ParseFloat(fac.WithdrawalCapacity, 64) facItem := models.BaseFromTradeEicIndexV2{ BaseFromEicIndexId: 0, Type: "fac", EicCode: code, Name: fac.Name, Status: fac.Status, GasDayStart: fac.GasDayStart, GasInStorage: gasInStorage, GasInStorageCode: code + "GS", Consumption: fac.Consumption, ConsumptionCode: code + "C", ConsumptionFull: fac.ConsumptionFull, ConsumptionFullCode: code + "CF", Full: full, FullCode: code + "F", Trend: trend, TrendCode: code + "T", Injection: injection, InjectionCode: code + "In", Withdrawal: withdrawal, WithdrawalCode: code + "Out", WorkingGasVolume: workingGasVolume, WorkingGasVolumeCode: code + "WGV", InjectionCapacity: injectionCapacity, InjectionCapacityCode: code + "IC", WithdrawalCapacity: withdrawalCapacity, WithdrawalCapacityCode: code + "WC", Info: fac.Info, Parent: sso.Name, CreateTime: time.Now(), ModifyTime: time.Now(), } //校验 existIndexMap := make(map[string]*models.BaseFromTradeEicIndexV2) allIndex, err := models.GetFacFromEicIndexAllV2(fac.GasDayStart) if err != nil { fmt.Println("select err:", err) } for _, v := range allIndex { existIndexMap[v.GasDayStart+v.EicCode] = v } itemVerifyCode := facItem.GasInStorage + facItem.Full + facItem.Trend + facItem.Injection + facItem.Withdrawal if existIndex, ok := existIndexMap[facItem.GasDayStart+facItem.EicCode]; !ok { //新增 if facItem.Status != "N" { insertItems = append(insertItems, &facItem) existIndexMap[facItem.GasDayStart+facItem.EicCode] = &facItem } } else if existIndex != nil && itemVerifyCode != (existIndex.GasInStorage+existIndex.Full+existIndex.Trend+existIndex.Injection+existIndex.Withdrawal) { //更新 if facItem.Status != "N" { err := models.ModifyBaseFromEicIndexV2(facItem.GasInStorage, facItem.Full, facItem.Trend, facItem.Injection, facItem.Withdrawal, existIndex.BaseFromEicIndexId) if err != nil { fmt.Println("data update err:", err) } } } if _, ok := eicIndexCodeMapV2[facItem.Name]; !ok { codeMappingItem := models.BaseFromTradeMapping{ BaseFromTradeMappingId: 0, IndexName: facItem.Name, IndexCode: facItem.EicCode, Exchange: "EIC", } codeMapList = append(codeMapList, &codeMappingItem) eicIndexCodeMapV2[facItem.Name] = facItem.EicCode } } } } //codeMap新增 if len(codeMapList) != 0 { successNums, err := models.AddEicCodeMulti(codeMapList) fmt.Println("codeMapping successNums:", successNums) if err != nil { fmt.Println("AddEicCodeMulti err:", err) } } } //洲际级新增 if len(insertItems) != 0 { successNums, err := models.AddEicDataMultiV2(insertItems) fmt.Println("successNums:", successNums) if err != nil { fmt.Println("AddContinentDataMulti err:", err) } } } }