eic

models/base_from_trade_eic_v2.go

@@ -0,0 +1,94 @@
+package models
+
+import (
+	"github.com/rdlucklib/rdluck_tools/orm"
+	"time"
+)
+
+type BaseFromTradeEicIndexV2 struct {
+	BaseFromEicIndexId     int `orm:"column(base_from_eic_index_id);pk"`
+	Type                   string
+	EicCode                string
+	Name                   string
+	Status                 string
+	GasDayStart            string
+	GasInStorage           float64
+	GasInStorageCode       string
+	Consumption            string
+	ConsumptionCode        string
+	ConsumptionFull        string
+	ConsumptionFullCode    string
+	Full                   float64
+	FullCode               string
+	Trend                  float64
+	TrendCode              string
+	Injection              float64
+	InjectionCode          string
+	Withdrawal             float64
+	WithdrawalCode         string
+	WorkingGasVolume       float64
+	WorkingGasVolumeCode   string
+	InjectionCapacity      float64
+	InjectionCapacityCode  string
+	WithdrawalCapacity     float64
+	WithdrawalCapacityCode string
+	Info                   string
+	Parent                 string
+	CreateTime             time.Time
+	ModifyTime             time.Time
+}
+
+func AddBaseFromEicIndexV2(item *BaseFromTradeEicIndexV2) (lastId int64, err error) {
+	o := orm.NewOrm()
+	o.Using("data")
+	lastId, err = o.Insert(item)
+	return
+}
+
+func GetSSOFromEicIndexAllV2(date string) (list []*BaseFromTradeEicIndexV2, err error) {
+	o := orm.NewOrm()
+	o.Using("data")
+	sql := `SELECT * FROM base_from_trade_eic_index_v2 where gas_day_start=? and type='sso'`
+	_, err = o.Raw(sql, date).QueryRows(&list)
+	return
+}
+
+func GetFacFromEicIndexAllV2(date string) (list []*BaseFromTradeEicIndexV2, err error) {
+	o := orm.NewOrm()
+	o.Using("data")
+	sql := `SELECT * FROM base_from_trade_eic_index_v2 where gas_day_start=? and type='fac'`
+	_, err = o.Raw(sql, date).QueryRows(&list)
+	return
+}
+
+func ModifyBaseFromEicIndexV2(gasInStorage, full, trend, injection, withdrawal float64, dataId int) (err error) {
+	o := orm.NewOrm()
+	o.Using("data")
+	sql := `UPDATE base_from_trade_eic_index_v2 SET gas_in_storage=?,full=?,trend=?,injection=?,withdrawal=?,modify_time=NOW() WHERE base_from_eic_index_id=? `
+
+	_, err = o.Raw(sql, gasInStorage, full, trend, injection, withdrawal, dataId).Exec()
+	return
+}
+
+func GetCountryFromEicIndexAllV2(date string) (list []*BaseFromTradeEicIndexV2, err error) {
+	o := orm.NewOrm()
+	o.Using("data")
+	sql := `SELECT * FROM base_from_trade_eic_index_v2 where gas_day_start=? and type='country' `
+	_, err = o.Raw(sql, date).QueryRows(&list)
+	return
+}
+
+func GetContinentFromEicIndexAllV2(date string) (list []*BaseFromTradeEicIndexV2, err error) {
+	o := orm.NewOrm()
+	o.Using("data")
+	sql := `SELECT * FROM base_from_trade_eic_index_v2 where gas_day_start=? and type='continent' `
+	_, err = o.Raw(sql, date).QueryRows(&list)
+	return
+}
+
+func AddEicDataMultiV2(items []*BaseFromTradeEicIndexV2) (successNums int64, err error) {
+	o := orm.NewOrm()
+	o.Using("data")
+	successNums, err = o.InsertMulti(1, items)
+	return
+}

models/db.go

@@ -32,6 +32,7 @@ func init() {
 		new(BaseFromTradeIneIndex),
 		new(BaseFromTradeCffexIndex),
 		new(BaseFromTradeEicIndex),
+		new(BaseFromTradeEicIndexV2),
 		new(BaseFromTradeMapping),
 		//
 		new(BaseFromTradeDalianIndex),

services/sso_eic_v2.go

@@ -0,0 +1,414 @@
+package services
+
+import (
+	"encoding/json"
+	"fmt"
+	"github.com/rdlucklib/rdluck_tools/http"
+	"hongze/hongze_data_crawler/models"
+	"hongze/hongze_data_crawler/utils"
+	"strconv"
+	"time"
+)
+
+type EicData struct {
+	LastPage string     `json:"last_page"`
+	Total    string     `json:"total"`
+	DataSet  string     `json:"dataset"`
+	GasDay   string     `json:"gas_day"`
+	Data     []Children `json:"data"`
+}
+
+type Children struct {
+	Name               string     `json:"name"`
+	Code               string     `json:"code"`
+	URL                string     `json:"url"`
+	GasDayStart        string     `json:"gasDayStart"`
+	GasInStorage       string     `json:"gasInStorage"`
+	Consumption        string     `json:"consumption"`
+	ConsumptionFull    string     `json:"consumptionFull"`
+	Injection          string     `json:"injection"`
+	Withdrawal         string     `json:"withdrawal"`
+	WorkingGasVolume   string     `json:"workingGasVolume"`
+	InjectionCapacity  string     `json:"injectionCapacity"`
+	WithdrawalCapacity string     `json:"withdrawalCapacity"`
+	Status             string     `json:"status"`
+	Trend              string     `json:"trend"`
+	Full               string     `json:"full"`
+	Info               string     `json:"info"`
+	Children           []Children `json:"children"`
+}
+
+type Storage1 struct {
+	Status             string `json:"status"`
+	GasDayStartedOn    string `json:"gasDayStartedOn"`
+	GasInStorage       string `json:"gasInStorage"`
+	Full               string `json:"full"`
+	Trend              string `json:"trend"`
+	Injection          string `json:"injection"`
+	Withdrawal         string `json:"withdrawal"`
+	WorkingGasVolume   string `json:"workingGasVolume"`
+	InjectionCapacity  string `json:"injectionCapacity"`
+	WithdrawalCapacity string `json:"withdrawalCapacity"`
+	Info               string `json:"info"`
+}
+
+var eicIndexCodeMapV2 = make(map[string]string)
+
+func SyncStorageFromEicV2() {
+	allCode, err := models.GetIndexCodeFromMapping("Eic")
+	if err != nil {
+		fmt.Println("select Code err:", err)
+	}
+	for _, item := range allCode {
+		eicIndexCodeMapV2[item.IndexName] = item.IndexCode
+	}
+	for i :=10; i >= 0; i-- {
+		baseUrl := "https://agsi.gie.eu/api?date=%s"
+		baseUrl = fmt.Sprintf(baseUrl, time.Now().AddDate(0, 0, -i).Format(utils.FormatDate))
+		body, err := http.Get(baseUrl)
+
+		if err != nil {
+			fmt.Println("GetData Err:" + err.Error())
+			return
+		}
+		fmt.Println("URL:", baseUrl)
+		var eicData EicData
+		err = json.Unmarshal(body, &eicData)
+		//洲际级
+		var insertItems []*models.BaseFromTradeEicIndexV2
+		for _, continent := range eicData.Data {
+			var codeMapList []*models.BaseFromTradeMapping
+
+			//为保持唯一性，code使用md5加密url
+			code := utils.MD5(continent.URL)
+			gasInStorage, _ := strconv.ParseFloat(continent.GasInStorage, 64)
+			full, _ := strconv.ParseFloat(continent.Full, 64)
+			trend, _ := strconv.ParseFloat(continent.Trend, 64)
+			injection, _ := strconv.ParseFloat(continent.Injection, 64)
+			withdrawal, _ := strconv.ParseFloat(continent.Withdrawal, 64)
+			workingGasVolume, _ := strconv.ParseFloat(continent.WorkingGasVolume, 64)
+			injectionCapacity, _ := strconv.ParseFloat(continent.InjectionCapacity, 64)
+			withdrawalCapacity, _ := strconv.ParseFloat(continent.WithdrawalCapacity, 64)
+			continentItem := models.BaseFromTradeEicIndexV2{
+				BaseFromEicIndexId:     0,
+				Type:                   "continent",
+				EicCode:                code,
+				Name:                   continent.Name,
+				Status:                 continent.Status,
+				GasDayStart:            continent.GasDayStart,
+				GasInStorage:           gasInStorage,
+				GasInStorageCode:       code + "GS",
+				Consumption:            continent.Consumption,
+				ConsumptionCode:        code + "C",
+				ConsumptionFull:        continent.ConsumptionFull,
+				ConsumptionFullCode:    code + "CF",
+				Full:                   full,
+				FullCode:               code + "F",
+				Trend:                  trend,
+				TrendCode:              code + "T",
+				Injection:              injection,
+				InjectionCode:          code + "In",
+				Withdrawal:             withdrawal,
+				WithdrawalCode:         code + "Out",
+				WorkingGasVolume:       workingGasVolume,
+				WorkingGasVolumeCode:   code + "WGV",
+				InjectionCapacity:      injectionCapacity,
+				InjectionCapacityCode:  code + "IC",
+				WithdrawalCapacity:     withdrawalCapacity,
+				WithdrawalCapacityCode: code + "WC",
+				Info:                   continent.Info,
+				CreateTime:             time.Now(),
+				ModifyTime:             time.Now(),
+			}
+			//校验
+			existIndexMap := make(map[string]*models.BaseFromTradeEicIndexV2)
+			allIndex, err := models.GetContinentFromEicIndexAllV2(continent.GasDayStart)
+			if err != nil {
+				fmt.Println("select err:", err)
+			}
+			for _, v := range allIndex {
+				existIndexMap[v.GasDayStart+v.EicCode] = v
+			}
+
+			itemVerifyCode := continentItem.GasInStorage + continentItem.Full + continentItem.Trend + continentItem.Injection + continentItem.Withdrawal
+
+			if existIndex, ok := existIndexMap[continentItem.GasDayStart+continentItem.EicCode]; !ok {
+				//新增
+				insertItems = append(insertItems, &continentItem)
+				existIndexMap[continentItem.GasDayStart+continentItem.EicCode] = &continentItem
+			} else if existIndex != nil && itemVerifyCode != (existIndex.GasInStorage+existIndex.Full+existIndex.Trend+existIndex.Injection+existIndex.Withdrawal) {
+				//更新
+				err := models.ModifyBaseFromEicIndexV2(continentItem.GasInStorage, continentItem.Full, continentItem.Trend, continentItem.Injection, continentItem.Withdrawal, existIndex.BaseFromEicIndexId)
+				if err != nil {
+					fmt.Println("data update err:", err)
+				}
+			}
+			if _, ok := eicIndexCodeMapV2[continentItem.Name]; !ok {
+				codeMappingItem := models.BaseFromTradeMapping{
+					BaseFromTradeMappingId: 0,
+					IndexName:              continentItem.Name,
+					IndexCode:              continentItem.EicCode,
+					Exchange:               "EIC",
+				}
+				codeMapList = append(codeMapList, &codeMappingItem)
+				eicIndexCodeMapV2[continentItem.Name] = continentItem.EicCode
+			}
+
+			//国家级
+			for _, country := range continent.Children {
+				//为保持唯一性，code使用md5加密url
+				code := utils.MD5(country.URL)
+				gasInStorage, _ := strconv.ParseFloat(country.GasInStorage, 64)
+				full, _ := strconv.ParseFloat(country.Full, 64)
+				trend, _ := strconv.ParseFloat(country.Trend, 64)
+				injection, _ := strconv.ParseFloat(country.Injection, 64)
+				withdrawal, _ := strconv.ParseFloat(country.Withdrawal, 64)
+				workingGasVolume, _ := strconv.ParseFloat(country.WorkingGasVolume, 64)
+				injectionCapacity, _ := strconv.ParseFloat(country.InjectionCapacity, 64)
+				withdrawalCapacity, _ := strconv.ParseFloat(country.WithdrawalCapacity, 64)
+				countryItem := models.BaseFromTradeEicIndexV2{
+					BaseFromEicIndexId:     0,
+					Type:                   "country",
+					EicCode:                code,
+					Name:                   country.Name,
+					Status:                 country.Status,
+					GasDayStart:            country.GasDayStart,
+					GasInStorage:           gasInStorage,
+					GasInStorageCode:       code + "GS",
+					Consumption:            country.Consumption,
+					ConsumptionCode:        code + "C",
+					ConsumptionFull:        country.ConsumptionFull,
+					ConsumptionFullCode:    code + "CF",
+					Full:                   full,
+					FullCode:               code + "F",
+					Trend:                  trend,
+					TrendCode:              code + "T",
+					Injection:              injection,
+					InjectionCode:          code + "In",
+					Withdrawal:             withdrawal,
+					WithdrawalCode:         code + "Out",
+					WorkingGasVolume:       workingGasVolume,
+					WorkingGasVolumeCode:   code + "WGV",
+					InjectionCapacity:      injectionCapacity,
+					InjectionCapacityCode:  code + "IC",
+					WithdrawalCapacity:     withdrawalCapacity,
+					WithdrawalCapacityCode: code + "WC",
+					Info:                   country.Info,
+					Parent:                 continent.Name,
+					CreateTime:             time.Now(),
+					ModifyTime:             time.Now(),
+				}
+				//校验
+				existIndexMap := make(map[string]*models.BaseFromTradeEicIndexV2)
+				allIndex, err := models.GetCountryFromEicIndexAllV2(country.GasDayStart)
+				if err != nil {
+					fmt.Println("select err:", err)
+				}
+				for _, v := range allIndex {
+					existIndexMap[v.GasDayStart+v.EicCode] = v
+				}
+
+				itemVerifyCode := countryItem.GasInStorage + countryItem.Full + countryItem.Trend + countryItem.Injection + countryItem.Withdrawal
+
+				if existIndex, ok := existIndexMap[countryItem.GasDayStart+countryItem.EicCode]; !ok {
+					//新增
+					insertItems = append(insertItems, &countryItem)
+					existIndexMap[countryItem.GasDayStart+countryItem.EicCode] = &countryItem
+				} else if existIndex != nil && itemVerifyCode != (existIndex.GasInStorage+existIndex.Full+existIndex.Trend+existIndex.Injection+existIndex.Withdrawal) {
+					//更新
+					err := models.ModifyBaseFromEicIndexV2(countryItem.GasInStorage, countryItem.Full, countryItem.Trend, countryItem.Injection, countryItem.Withdrawal, existIndex.BaseFromEicIndexId)
+					if err != nil {
+						fmt.Println("data update err:", err)
+					}
+				}
+
+				if _, ok := eicIndexCodeMapV2[countryItem.Name]; !ok {
+					codeMappingItem := models.BaseFromTradeMapping{
+						BaseFromTradeMappingId: 0,
+						IndexName:              countryItem.Name,
+						IndexCode:              countryItem.EicCode,
+						Exchange:               "EIC",
+					}
+					codeMapList = append(codeMapList, &codeMappingItem)
+					eicIndexCodeMapV2[countryItem.Name] = countryItem.EicCode
+				}
+
+				//厂商级
+				for _, sso := range country.Children {
+					//为保持唯一性，code使用md5加密url
+					code := utils.MD5(sso.URL)
+					gasInStorage, _ := strconv.ParseFloat(sso.GasInStorage, 64)
+					full, _ := strconv.ParseFloat(sso.Full, 64)
+					trend, _ := strconv.ParseFloat(sso.Trend, 64)
+					injection, _ := strconv.ParseFloat(sso.Injection, 64)
+					withdrawal, _ := strconv.ParseFloat(sso.Withdrawal, 64)
+					workingGasVolume, _ := strconv.ParseFloat(sso.WorkingGasVolume, 64)
+					injectionCapacity, _ := strconv.ParseFloat(sso.InjectionCapacity, 64)
+					withdrawalCapacity, _ := strconv.ParseFloat(sso.WithdrawalCapacity, 64)
+					ssoItem := models.BaseFromTradeEicIndexV2{
+						BaseFromEicIndexId:     0,
+						Type:                   "sso",
+						EicCode:                code,
+						Name:                   sso.Name,
+						Status:                 sso.Status,
+						GasDayStart:            sso.GasDayStart,
+						GasInStorage:           gasInStorage,
+						GasInStorageCode:       code + "GS",
+						Consumption:            sso.Consumption,
+						ConsumptionCode:        code + "C",
+						ConsumptionFull:        sso.ConsumptionFull,
+						ConsumptionFullCode:    code + "CF",
+						Full:                   full,
+						FullCode:               code + "F",
+						Trend:                  trend,
+						TrendCode:              code + "T",
+						Injection:              injection,
+						InjectionCode:          code + "In",
+						Withdrawal:             withdrawal,
+						WithdrawalCode:         code + "Out",
+						WorkingGasVolume:       workingGasVolume,
+						WorkingGasVolumeCode:   code + "WGV",
+						InjectionCapacity:      injectionCapacity,
+						InjectionCapacityCode:  code + "IC",
+						WithdrawalCapacity:     withdrawalCapacity,
+						WithdrawalCapacityCode: code + "WC",
+						Info:                   sso.Info,
+						Parent:                 country.Name,
+						CreateTime:             time.Now(),
+						ModifyTime:             time.Now(),
+					}
+					//校验
+					existIndexMap := make(map[string]*models.BaseFromTradeEicIndexV2)
+					allIndex, err := models.GetSSOFromEicIndexAllV2(sso.GasDayStart)
+					if err != nil {
+						fmt.Println("select err:", err)
+					}
+					for _, v := range allIndex {
+						existIndexMap[v.GasDayStart+v.EicCode] = v
+					}
+
+					itemVerifyCode := ssoItem.GasInStorage + ssoItem.Full + ssoItem.Trend + ssoItem.Injection + ssoItem.Withdrawal
+
+					if existIndex, ok := existIndexMap[ssoItem.GasDayStart+ssoItem.EicCode]; !ok {
+						//新增
+						insertItems = append(insertItems, &ssoItem)
+						existIndexMap[ssoItem.GasDayStart+ssoItem.EicCode] = &ssoItem
+					} else if existIndex != nil && itemVerifyCode != (existIndex.GasInStorage+existIndex.Full+existIndex.Trend+existIndex.Injection+existIndex.Withdrawal) {
+						//更新
+						err := models.ModifyBaseFromEicIndexV2(ssoItem.GasInStorage, ssoItem.Full, ssoItem.Trend, ssoItem.Injection, ssoItem.Withdrawal, existIndex.BaseFromEicIndexId)
+						if err != nil {
+							fmt.Println("data update err:", err)
+						}
+					}
+
+					if _, ok := eicIndexCodeMapV2[ssoItem.Name]; !ok {
+						codeMappingItem := models.BaseFromTradeMapping{
+							BaseFromTradeMappingId: 0,
+							IndexName:              ssoItem.Name,
+							IndexCode:              ssoItem.EicCode,
+							Exchange:               "EIC",
+						}
+						codeMapList = append(codeMapList, &codeMappingItem)
+						eicIndexCodeMapV2[ssoItem.Name] = ssoItem.EicCode
+					}
+
+					//设施级
+					for _, fac := range sso.Children {
+						//为保持唯一性，code使用md5加密url
+						code := utils.MD5(fac.URL)
+						gasInStorage, _ := strconv.ParseFloat(fac.GasInStorage, 64)
+						full, _ := strconv.ParseFloat(fac.Full, 64)
+						trend, _ := strconv.ParseFloat(fac.Trend, 64)
+						injection, _ := strconv.ParseFloat(fac.Injection, 64)
+						withdrawal, _ := strconv.ParseFloat(fac.Withdrawal, 64)
+						workingGasVolume, _ := strconv.ParseFloat(fac.WorkingGasVolume, 64)
+						injectionCapacity, _ := strconv.ParseFloat(fac.InjectionCapacity, 64)
+						withdrawalCapacity, _ := strconv.ParseFloat(fac.WithdrawalCapacity, 64)
+						facItem := models.BaseFromTradeEicIndexV2{
+							BaseFromEicIndexId:     0,
+							Type:                   "fac",
+							EicCode:                code,
+							Name:                   fac.Name,
+							Status:                 fac.Status,
+							GasDayStart:            fac.GasDayStart,
+							GasInStorage:           gasInStorage,
+							GasInStorageCode:       code + "GS",
+							Consumption:            fac.Consumption,
+							ConsumptionCode:        code + "C",
+							ConsumptionFull:        fac.ConsumptionFull,
+							ConsumptionFullCode:    code + "CF",
+							Full:                   full,
+							FullCode:               code + "F",
+							Trend:                  trend,
+							TrendCode:              code + "T",
+							Injection:              injection,
+							InjectionCode:          code + "In",
+							Withdrawal:             withdrawal,
+							WithdrawalCode:         code + "Out",
+							WorkingGasVolume:       workingGasVolume,
+							WorkingGasVolumeCode:   code + "WGV",
+							InjectionCapacity:      injectionCapacity,
+							InjectionCapacityCode:  code + "IC",
+							WithdrawalCapacity:     withdrawalCapacity,
+							WithdrawalCapacityCode: code + "WC",
+							Info:                   fac.Info,
+							Parent:                 sso.Name,
+							CreateTime:             time.Now(),
+							ModifyTime:             time.Now(),
+						}
+						//校验
+						existIndexMap := make(map[string]*models.BaseFromTradeEicIndexV2)
+						allIndex, err := models.GetFacFromEicIndexAllV2(fac.GasDayStart)
+						if err != nil {
+							fmt.Println("select err:", err)
+						}
+						for _, v := range allIndex {
+							existIndexMap[v.GasDayStart+v.EicCode] = v
+						}
+
+						itemVerifyCode := facItem.GasInStorage + facItem.Full + facItem.Trend + facItem.Injection + facItem.Withdrawal
+
+						if existIndex, ok := existIndexMap[facItem.GasDayStart+facItem.EicCode]; !ok {
+							//新增
+							insertItems = append(insertItems, &facItem)
+							existIndexMap[facItem.GasDayStart+facItem.EicCode] = &facItem
+						} else if existIndex != nil && itemVerifyCode != (existIndex.GasInStorage+existIndex.Full+existIndex.Trend+existIndex.Injection+existIndex.Withdrawal) {
+							//更新
+							err := models.ModifyBaseFromEicIndexV2(facItem.GasInStorage, facItem.Full, facItem.Trend, facItem.Injection, facItem.Withdrawal, existIndex.BaseFromEicIndexId)
+							if err != nil {
+								fmt.Println("data update err:", err)
+							}
+						}
+
+						if _, ok := eicIndexCodeMapV2[facItem.Name]; !ok {
+							codeMappingItem := models.BaseFromTradeMapping{
+								BaseFromTradeMappingId: 0,
+								IndexName:              facItem.Name,
+								IndexCode:              facItem.EicCode,
+								Exchange:               "EIC",
+							}
+							codeMapList = append(codeMapList, &codeMappingItem)
+							eicIndexCodeMapV2[facItem.Name] = facItem.EicCode
+						}
+					}
+				}
+			}
+			//codeMap新增
+			if len(codeMapList) != 0 {
+				successNums, err := models.AddEicCodeMulti(codeMapList)
+				fmt.Println("codeMapping successNums:", successNums)
+				if err != nil {
+					fmt.Println("AddEicCodeMulti err:", err)
+				}
+			}
+		}
+		//洲际级新增
+		if len(insertItems) != 0 {
+			successNums, err := models.AddEicDataMultiV2(insertItems)
+			fmt.Println("successNums:", successNums)
+			if err != nil {
+				fmt.Println("AddContinentDataMulti err:", err)
+			}
+		}
+	}
+}