eta_crawler/services/sso_eic_v2.go

package services

import (
	"encoding/json"
	"eta/eta_crawler/models"
	"eta/eta_crawler/utils"
	"fmt"
	"io/ioutil"
	"net/http"
	"strconv"
	"time"
)

type EicData struct {
	LastPage int        `json:"last_page"`
	Total    int        `json:"total"`
	DataSet  string     `json:"dataset"`
	GasDay   string     `json:"gas_day"`
	Data     []Children `json:"data"`
	Error    string     `json:"error"`
	Message  string     `json:"message"`
}

type Children struct {
	Name               string     `json:"name"`
	Code               string     `json:"code"`
	URL                string     `json:"url"`
	GasDayStart        string     `json:"gasDayStart"`
	GasInStorage       string     `json:"gasInStorage"`
	Consumption        string     `json:"consumption"`
	ConsumptionFull    string     `json:"consumptionFull"`
	Injection          string     `json:"injection"`
	Withdrawal         string     `json:"withdrawal"`
	WorkingGasVolume   string     `json:"workingGasVolume"`
	InjectionCapacity  string     `json:"injectionCapacity"`
	WithdrawalCapacity string     `json:"withdrawalCapacity"`
	Status             string     `json:"status"`
	Trend              string     `json:"trend"`
	Full               string     `json:"full"`
	Info               string     `json:"-"`
	Children           []Children `json:"children"`
}

type Storage1 struct {
	Status             string `json:"status"`
	GasDayStartedOn    string `json:"gasDayStartedOn"`
	GasInStorage       string `json:"gasInStorage"`
	Full               string `json:"full"`
	Trend              string `json:"trend"`
	Injection          string `json:"injection"`
	Withdrawal         string `json:"withdrawal"`
	WorkingGasVolume   string `json:"workingGasVolume"`
	InjectionCapacity  string `json:"injectionCapacity"`
	WithdrawalCapacity string `json:"withdrawalCapacity"`
	Info               string `json:"info"`
}

var eicIndexCodeMapV2 = make(map[string]string)

func SyncStorageFromEicV2() {
	allCode, err := models.GetIndexCodeFromMapping("Eic")
	if err != nil {
		fmt.Println("select Code err:", err)
	}
	for _, item := range allCode {
		eicIndexCodeMapV2[item.IndexName] = item.IndexCode
	}
	for i := 10; i >= 0; i-- {
		baseUrl := "https://agsi.gie.eu/api?date=%s"
		//t, _ := time.Parse(utils.FormatDate, "2022-08-15")
		//st := t.AddDate(0, 0, i).Format(utils.FormatDate)
		st := time.Now().AddDate(0, 0, -i).Format(utils.FormatDate)
		tt, _ := time.Parse(utils.FormatDate, st)
		now := time.Now()
		if tt.After(now) {
			break
		}
		baseUrl = fmt.Sprintf(baseUrl, st)

		client := http.Client{}
		req, err := http.NewRequest(http.MethodGet, baseUrl, nil)
		if err != nil {
			fmt.Println("err")
		}
		// 添加请求头
		req.Header.Add("Content-type", "application/json;charset=utf-8")
		req.Header.Add("x-key", "3669f7349c0598d50948429d6ea60070")
		// 发送请求
		resp, err := client.Do(req)
		if err != nil {
			fmt.Println("err")
		}
		defer resp.Body.Close()
		body, err := ioutil.ReadAll(resp.Body)
		if err != nil {
			fmt.Println("err")
		}

		//body, err := http.Get(baseUrl)
		//
		//if err != nil {
		//	fmt.Println("GetData Err:" + err.Error())
		//	return
		//}
		fmt.Println("URL:", baseUrl)
		var eicData EicData
		err = json.Unmarshal(body, &eicData)
		if err != nil {
			fmt.Println("Unmarshal Err:" + err.Error())
			return
		}
		//洲际级
		var insertItems []*models.BaseFromTradeEicIndexV2
		for _, continent := range eicData.Data {
			var codeMapList []*models.BaseFromTradeMapping

			//为保持唯一性，code使用md5加密url
			code := utils.MD5(continent.URL)
			gasInStorage, _ := strconv.ParseFloat(continent.GasInStorage, 64)
			full, _ := strconv.ParseFloat(continent.Full, 64)
			trend, _ := strconv.ParseFloat(continent.Trend, 64)
			injection, _ := strconv.ParseFloat(continent.Injection, 64)
			withdrawal, _ := strconv.ParseFloat(continent.Withdrawal, 64)
			workingGasVolume, _ := strconv.ParseFloat(continent.WorkingGasVolume, 64)
			injectionCapacity, _ := strconv.ParseFloat(continent.InjectionCapacity, 64)
			withdrawalCapacity, _ := strconv.ParseFloat(continent.WithdrawalCapacity, 64)
			continentItem := models.BaseFromTradeEicIndexV2{
				BaseFromEicIndexId:     0,
				Type:                   "continent",
				EicCode:                code,
				Name:                   continent.Name,
				Status:                 continent.Status,
				GasDayStart:            continent.GasDayStart,
				GasInStorage:           gasInStorage,
				GasInStorageCode:       code + "GS",
				Consumption:            continent.Consumption,
				ConsumptionCode:        code + "C",
				ConsumptionFull:        continent.ConsumptionFull,
				ConsumptionFullCode:    code + "CF",
				Full:                   full,
				FullCode:               code + "F",
				Trend:                  trend,
				TrendCode:              code + "T",
				Injection:              injection,
				InjectionCode:          code + "In",
				Withdrawal:             withdrawal,
				WithdrawalCode:         code + "Out",
				WorkingGasVolume:       workingGasVolume,
				WorkingGasVolumeCode:   code + "WGV",
				InjectionCapacity:      injectionCapacity,
				InjectionCapacityCode:  code + "IC",
				WithdrawalCapacity:     withdrawalCapacity,
				WithdrawalCapacityCode: code + "WC",
				Info:                   continent.Info,
				CreateTime:             time.Now(),
				ModifyTime:             time.Now(),
			}
			//校验
			existIndexMap := make(map[string]*models.BaseFromTradeEicIndexV2)
			allIndex, err := models.GetContinentFromEicIndexAllV2(continent.GasDayStart)
			if err != nil {
				fmt.Println("select err:", err)
			}
			for _, v := range allIndex {
				existIndexMap[v.GasDayStart+v.EicCode] = v
			}

			itemVerifyCode := continentItem.GasInStorage + continentItem.Full + continentItem.Trend + continentItem.Injection + continentItem.Withdrawal

			if existIndex, ok := existIndexMap[continentItem.GasDayStart+continentItem.EicCode]; !ok {
				//新增
				if continentItem.Status != "N" {
					insertItems = append(insertItems, &continentItem)
					existIndexMap[continentItem.GasDayStart+continentItem.EicCode] = &continentItem
				}
			} else if existIndex != nil && itemVerifyCode != (existIndex.GasInStorage+existIndex.Full+existIndex.Trend+existIndex.Injection+existIndex.Withdrawal) {
				//更新
				if continentItem.Status != "N" {
					err := models.ModifyBaseFromEicIndexV2(continentItem.GasInStorage, continentItem.Full, continentItem.Trend, continentItem.Injection, continentItem.Withdrawal, existIndex.BaseFromEicIndexId)
					if err != nil {
						fmt.Println("data update err:", err)
					}
				}
			}
			if _, ok := eicIndexCodeMapV2[continentItem.Name]; !ok {
				codeMappingItem := models.BaseFromTradeMapping{
					BaseFromTradeMappingId: 0,
					IndexName:              continentItem.Name,
					IndexCode:              continentItem.EicCode,
					Exchange:               "EIC",
				}
				codeMapList = append(codeMapList, &codeMappingItem)
				eicIndexCodeMapV2[continentItem.Name] = continentItem.EicCode
			}

			//国家级
			for _, country := range continent.Children {
				//为保持唯一性，code使用md5加密url
				code := utils.MD5(country.URL)
				gasInStorage, _ := strconv.ParseFloat(country.GasInStorage, 64)
				full, _ := strconv.ParseFloat(country.Full, 64)
				trend, _ := strconv.ParseFloat(country.Trend, 64)
				injection, _ := strconv.ParseFloat(country.Injection, 64)
				withdrawal, _ := strconv.ParseFloat(country.Withdrawal, 64)
				workingGasVolume, _ := strconv.ParseFloat(country.WorkingGasVolume, 64)
				injectionCapacity, _ := strconv.ParseFloat(country.InjectionCapacity, 64)
				withdrawalCapacity, _ := strconv.ParseFloat(country.WithdrawalCapacity, 64)
				countryItem := models.BaseFromTradeEicIndexV2{
					BaseFromEicIndexId:     0,
					Type:                   "country",
					EicCode:                code,
					Name:                   country.Name,
					Status:                 country.Status,
					GasDayStart:            country.GasDayStart,
					GasInStorage:           gasInStorage,
					GasInStorageCode:       code + "GS",
					Consumption:            country.Consumption,
					ConsumptionCode:        code + "C",
					ConsumptionFull:        country.ConsumptionFull,
					ConsumptionFullCode:    code + "CF",
					Full:                   full,
					FullCode:               code + "F",
					Trend:                  trend,
					TrendCode:              code + "T",
					Injection:              injection,
					InjectionCode:          code + "In",
					Withdrawal:             withdrawal,
					WithdrawalCode:         code + "Out",
					WorkingGasVolume:       workingGasVolume,
					WorkingGasVolumeCode:   code + "WGV",
					InjectionCapacity:      injectionCapacity,
					InjectionCapacityCode:  code + "IC",
					WithdrawalCapacity:     withdrawalCapacity,
					WithdrawalCapacityCode: code + "WC",
					Info:                   country.Info,
					Parent:                 continent.Name,
					CreateTime:             time.Now(),
					ModifyTime:             time.Now(),
				}
				//校验
				existIndexMap := make(map[string]*models.BaseFromTradeEicIndexV2)
				allIndex, err := models.GetCountryFromEicIndexAllV2(country.GasDayStart)
				if err != nil {
					fmt.Println("select err:", err)
				}
				for _, v := range allIndex {
					existIndexMap[v.GasDayStart+v.EicCode] = v
				}

				itemVerifyCode := countryItem.GasInStorage + countryItem.Full + countryItem.Trend + countryItem.Injection + countryItem.Withdrawal

				if existIndex, ok := existIndexMap[countryItem.GasDayStart+countryItem.EicCode]; !ok {
					//新增
					if countryItem.Status != "N" {
						insertItems = append(insertItems, &countryItem)
						existIndexMap[countryItem.GasDayStart+countryItem.EicCode] = &countryItem
					}
				} else if existIndex != nil && itemVerifyCode != (existIndex.GasInStorage+existIndex.Full+existIndex.Trend+existIndex.Injection+existIndex.Withdrawal) {
					//更新
					if countryItem.Status != "N" {
						err := models.ModifyBaseFromEicIndexV2(countryItem.GasInStorage, countryItem.Full, countryItem.Trend, countryItem.Injection, countryItem.Withdrawal, existIndex.BaseFromEicIndexId)
						if err != nil {
							fmt.Println("data update err:", err)
						}
					}
				}

				if _, ok := eicIndexCodeMapV2[countryItem.Name]; !ok {
					codeMappingItem := models.BaseFromTradeMapping{
						BaseFromTradeMappingId: 0,
						IndexName:              countryItem.Name,
						IndexCode:              countryItem.EicCode,
						Exchange:               "EIC",
					}
					codeMapList = append(codeMapList, &codeMappingItem)
					eicIndexCodeMapV2[countryItem.Name] = countryItem.EicCode
				}

				//厂商级
				for _, sso := range country.Children {
					//为保持唯一性，code使用md5加密url
					code := utils.MD5(sso.URL)
					gasInStorage, _ := strconv.ParseFloat(sso.GasInStorage, 64)
					full, _ := strconv.ParseFloat(sso.Full, 64)
					trend, _ := strconv.ParseFloat(sso.Trend, 64)
					injection, _ := strconv.ParseFloat(sso.Injection, 64)
					withdrawal, _ := strconv.ParseFloat(sso.Withdrawal, 64)
					workingGasVolume, _ := strconv.ParseFloat(sso.WorkingGasVolume, 64)
					injectionCapacity, _ := strconv.ParseFloat(sso.InjectionCapacity, 64)
					withdrawalCapacity, _ := strconv.ParseFloat(sso.WithdrawalCapacity, 64)
					ssoItem := models.BaseFromTradeEicIndexV2{
						BaseFromEicIndexId:     0,
						Type:                   "sso",
						EicCode:                code,
						Name:                   sso.Name,
						Status:                 sso.Status,
						GasDayStart:            sso.GasDayStart,
						GasInStorage:           gasInStorage,
						GasInStorageCode:       code + "GS",
						Consumption:            sso.Consumption,
						ConsumptionCode:        code + "C",
						ConsumptionFull:        sso.ConsumptionFull,
						ConsumptionFullCode:    code + "CF",
						Full:                   full,
						FullCode:               code + "F",
						Trend:                  trend,
						TrendCode:              code + "T",
						Injection:              injection,
						InjectionCode:          code + "In",
						Withdrawal:             withdrawal,
						WithdrawalCode:         code + "Out",
						WorkingGasVolume:       workingGasVolume,
						WorkingGasVolumeCode:   code + "WGV",
						InjectionCapacity:      injectionCapacity,
						InjectionCapacityCode:  code + "IC",
						WithdrawalCapacity:     withdrawalCapacity,
						WithdrawalCapacityCode: code + "WC",
						Info:                   sso.Info,
						Parent:                 country.Name,
						CreateTime:             time.Now(),
						ModifyTime:             time.Now(),
					}
					//校验
					existIndexMap := make(map[string]*models.BaseFromTradeEicIndexV2)
					allIndex, err := models.GetSSOFromEicIndexAllV2(sso.GasDayStart)
					if err != nil {
						fmt.Println("select err:", err)
					}
					for _, v := range allIndex {
						existIndexMap[v.GasDayStart+v.EicCode] = v
					}

					itemVerifyCode := ssoItem.GasInStorage + ssoItem.Full + ssoItem.Trend + ssoItem.Injection + ssoItem.Withdrawal

					if existIndex, ok := existIndexMap[ssoItem.GasDayStart+ssoItem.EicCode]; !ok {
						//新增
						if ssoItem.Status != "N" {
							insertItems = append(insertItems, &ssoItem)
							existIndexMap[ssoItem.GasDayStart+ssoItem.EicCode] = &ssoItem
						}
					} else if existIndex != nil && itemVerifyCode != (existIndex.GasInStorage+existIndex.Full+existIndex.Trend+existIndex.Injection+existIndex.Withdrawal) {
						//更新
						if ssoItem.Status != "N" {
							err := models.ModifyBaseFromEicIndexV2(ssoItem.GasInStorage, ssoItem.Full, ssoItem.Trend, ssoItem.Injection, ssoItem.Withdrawal, existIndex.BaseFromEicIndexId)
							if err != nil {
								fmt.Println("data update err:", err)
							}
						}
					}

					if _, ok := eicIndexCodeMapV2[ssoItem.Name]; !ok {
						codeMappingItem := models.BaseFromTradeMapping{
							BaseFromTradeMappingId: 0,
							IndexName:              ssoItem.Name,
							IndexCode:              ssoItem.EicCode,
							Exchange:               "EIC",
						}
						codeMapList = append(codeMapList, &codeMappingItem)
						eicIndexCodeMapV2[ssoItem.Name] = ssoItem.EicCode
					}

					//设施级
					for _, fac := range sso.Children {
						//为保持唯一性，code使用md5加密url
						code := utils.MD5(fac.URL)
						gasInStorage, _ := strconv.ParseFloat(fac.GasInStorage, 64)
						full, _ := strconv.ParseFloat(fac.Full, 64)
						trend, _ := strconv.ParseFloat(fac.Trend, 64)
						injection, _ := strconv.ParseFloat(fac.Injection, 64)
						withdrawal, _ := strconv.ParseFloat(fac.Withdrawal, 64)
						workingGasVolume, _ := strconv.ParseFloat(fac.WorkingGasVolume, 64)
						injectionCapacity, _ := strconv.ParseFloat(fac.InjectionCapacity, 64)
						withdrawalCapacity, _ := strconv.ParseFloat(fac.WithdrawalCapacity, 64)
						facItem := models.BaseFromTradeEicIndexV2{
							BaseFromEicIndexId:     0,
							Type:                   "fac",
							EicCode:                code,
							Name:                   fac.Name,
							Status:                 fac.Status,
							GasDayStart:            fac.GasDayStart,
							GasInStorage:           gasInStorage,
							GasInStorageCode:       code + "GS",
							Consumption:            fac.Consumption,
							ConsumptionCode:        code + "C",
							ConsumptionFull:        fac.ConsumptionFull,
							ConsumptionFullCode:    code + "CF",
							Full:                   full,
							FullCode:               code + "F",
							Trend:                  trend,
							TrendCode:              code + "T",
							Injection:              injection,
							InjectionCode:          code + "In",
							Withdrawal:             withdrawal,
							WithdrawalCode:         code + "Out",
							WorkingGasVolume:       workingGasVolume,
							WorkingGasVolumeCode:   code + "WGV",
							InjectionCapacity:      injectionCapacity,
							InjectionCapacityCode:  code + "IC",
							WithdrawalCapacity:     withdrawalCapacity,
							WithdrawalCapacityCode: code + "WC",
							Info:                   fac.Info,
							Parent:                 sso.Name,
							CreateTime:             time.Now(),
							ModifyTime:             time.Now(),
						}
						//校验
						existIndexMap := make(map[string]*models.BaseFromTradeEicIndexV2)
						allIndex, err := models.GetFacFromEicIndexAllV2(fac.GasDayStart)
						if err != nil {
							fmt.Println("select err:", err)
						}
						for _, v := range allIndex {
							existIndexMap[v.GasDayStart+v.EicCode] = v
						}

						itemVerifyCode := facItem.GasInStorage + facItem.Full + facItem.Trend + facItem.Injection + facItem.Withdrawal

						if existIndex, ok := existIndexMap[facItem.GasDayStart+facItem.EicCode]; !ok {
							//新增
							if facItem.Status != "N" {
								insertItems = append(insertItems, &facItem)
								existIndexMap[facItem.GasDayStart+facItem.EicCode] = &facItem
							}
						} else if existIndex != nil && itemVerifyCode != (existIndex.GasInStorage+existIndex.Full+existIndex.Trend+existIndex.Injection+existIndex.Withdrawal) {
							//更新
							if facItem.Status != "N" {
								err := models.ModifyBaseFromEicIndexV2(facItem.GasInStorage, facItem.Full, facItem.Trend, facItem.Injection, facItem.Withdrawal, existIndex.BaseFromEicIndexId)
								if err != nil {
									fmt.Println("data update err:", err)
								}
							}
						}

						if _, ok := eicIndexCodeMapV2[facItem.Name]; !ok {
							codeMappingItem := models.BaseFromTradeMapping{
								BaseFromTradeMappingId: 0,
								IndexName:              facItem.Name,
								IndexCode:              facItem.EicCode,
								Exchange:               "EIC",
							}
							codeMapList = append(codeMapList, &codeMappingItem)
							eicIndexCodeMapV2[facItem.Name] = facItem.EicCode
						}
					}
				}
			}
			//codeMap新增
			if len(codeMapList) != 0 {
				successNums, err := models.AddEicCodeMulti(codeMapList)
				fmt.Println("codeMapping successNums:", successNums)
				if err != nil {
					fmt.Println("AddEicCodeMulti err:", err)
				}
			}
		}
		//洲际级新增
		if len(insertItems) != 0 {
			successNums, err := models.AddEicDataMultiV2(insertItems)
			fmt.Println("successNums:", successNums)
			if err != nil {
				fmt.Println("AddContinentDataMulti err:", err)
			}
		}
	}
}