汾渭数据源-初始化

models/base_from_fenwei.go

@@ -58,3 +58,18 @@ type EdbLibFenweiIndexListResponse struct {
 	ErrCode string
 	Data    []BaseFromFenweiIndex
 }
+
+// 接口爬取汾渭网页数据
+type RequestParams struct {
+	Category     interface{}         `json:"category"`
+	CheckedDims  map[string][]string `json:"checkedDims"`
+	DateRange    string              `json:"dateRange"`
+	ProductCode  string              `json:"productCode"`
+	QuotaName    string              `json:"quotaName"`
+	SplitTypeKey string              `json:"splitTypeKey"`
+	IsTotal      int                 `json:"isTotal"`
+	DataType     interface{}         `json:"dataType"`
+	Type         int                 `json:"type"`
+	IsSeason     int                 `json:"isSeason"`
+	splitTypeKey string              `json:"splitTypeKey"`
+}

services/fenwei/base_from_fenwei_service.go

@@ -0,0 +1,22 @@
+// Package fenwei
+// @Author gmy 2024/8/20 15:06:00
+package fenwei
+
+import (
+	"eta/eta_data_analysis/utils"
+	"fmt"
+)
+
+// FenWeiNetDataDeal 汾渭网络数据处理
+func FenWeiNetDataDeal(err error) {
+	defer func() {
+		if err != nil {
+			fmt.Println("FenweiReadWatchIndexFile Err:" + err.Error())
+			utils.FileLog.Info(fmt.Sprintf("FenweiReadWatchIndexFile Err: %s", err.Error()))
+		}
+	}()
+	if utils.FenweiFileDir == "" {
+		utils.FileLog.Info("文件目录未配置")
+		return
+	}
+}

services/fenwei/data_processor.go

@@ -0,0 +1,129 @@
+// Package fenwei
+// @Author gmy 2024/8/20 14:47:00
+package fenwei
+
+import (
+	"bytes"
+	"context"
+	"eta/eta_data_analysis/utils"
+	"fmt"
+	"github.com/chromedp/cdproto/network"
+	"github.com/chromedp/chromedp"
+	"io"
+	"log"
+	"net/http"
+	"time"
+)
+
+type DataProcessor interface {
+	FetchAndProcess() (string, error)
+	GenerateRequestParams(string) map[string]string
+}
+
+type BaseProcessor struct {
+	URL string
+}
+
+func (p *BaseProcessor) FetchAndProcess() (string, error) {
+
+	// 获取当前时间 yyyy-MM-dd
+	now := time.Now()
+	currentTime := now.Format(utils.FormatDateUnSpace)
+
+	params := p.GenerateRequestParams(currentTime)
+
+	req, err := http.NewRequest("POST", p.URL, bytes.NewBufferString(params["params"]))
+	if err != nil {
+		return "", err
+	}
+
+	// 设置请求头
+	req.Header.Set("Content-Type", "application/json")
+	req.Header.Set("accept-language", "zh-CN,zh;q=0.9")
+
+	client := &http.Client{}
+	resp, err := client.Do(req)
+	if err != nil {
+		return "", err
+	}
+	defer resp.Body.Close()
+
+	body, err := io.ReadAll(resp.Body)
+	if err != nil {
+		return "", err
+	}
+
+	return p.ProcessResponse(string(body)), nil
+}
+
+// GenerateRequestParams 让子类来实现这个方法
+func (p *BaseProcessor) GenerateRequestParams(string) map[string]string {
+	return map[string]string{}
+}
+
+func (p *BaseProcessor) ProcessResponse(data string) string {
+
+	return "Processed Data: " + data
+}
+
+// GetCookieByChrome 获取cookie
+func getCookieByChrome() (cookieStr string, err error) {
+	// 读取Cookie
+	if utils.FenweiNetUseName == "" {
+		err = fmt.Errorf("汾渭账号未设置")
+		return
+	}
+	if utils.FenweiNetPassword == "" {
+		err = fmt.Errorf("汾渭密码未设置")
+		return
+	}
+	opts := append(
+		chromedp.DefaultExecAllocatorOptions[:],
+		chromedp.Flag("headless", false),
+	)
+	allocCtx, cancel1 := chromedp.NewExecAllocator(context.Background(), opts...)
+	defer cancel1()
+
+	// 创建chrome实例
+	ctx, cancel2 := chromedp.NewContext(
+		allocCtx,
+		chromedp.WithLogf(log.Printf),
+	)
+	defer cancel2()
+	err = chromedp.Run(ctx,
+		chromedp.Navigate(`https://www.sxcoal.com/`),
+		chromedp.Click(`.pc_content__jO_mq`, chromedp.ByQuery),
+		chromedp.Sleep(2*time.Second),
+
+		chromedp.SetValue(`div.Sign_username__7eYwE input[type="text"]`, utils.FenweiNetUseName, chromedp.ByQuery),
+		chromedp.SetValue(`div.Sign_password__dwxMn input[type="password"]`, utils.FenweiNetPassword, chromedp.ByQuery),
+		chromedp.Sleep(2*time.Second),
+
+		// 登录
+		chromedp.Click(`.Button_text__D25sy`, chromedp.ByQuery),
+
+		chromedp.ActionFunc(func(ctx context.Context) error {
+			cookies, err := network.GetCookies().Do(ctx)
+			if err != nil {
+				return err
+			}
+
+			for _, v := range cookies {
+				cookieStr = cookieStr + v.Name + "=" + v.Value + ";"
+			}
+			utils.FileLog.Info("header cookie:" + cookieStr)
+
+			return nil
+		}),
+	)
+
+	return
+}
+
+func encodeParams(params map[string]string) string {
+	var result string
+	for key, value := range params {
+		result += fmt.Sprintf("%s=%s&", key, value)
+	}
+	return result[:len(result)-1]
+}

services/fenwei/processor_business_logic.go

@@ -0,0 +1,638 @@
+// Package fenwei
+// @Author gmy 2024/8/20 14:47:00
+package fenwei
+
+import (
+	"encoding/json"
+	"eta/eta_data_analysis/models"
+)
+
+// ThermalCoalSupplyProcessor 动力煤供应量
+type ThermalCoalSupplyProcessor struct {
+	BaseProcessor
+}
+
+func (p *ThermalCoalSupplyProcessor) GenerateRequestParams(currentTime string) map[string]string {
+	params := models.RequestParams{
+		Category: nil,
+		CheckedDims: map[string][]string{
+			"province": {"20", "16", "21", "6", "19", "23", "8", "1", "7", "17", "9", "15", "11", "22", "24", "4", "25", "12", "14", "13", "18", "3", "10", "5", "2", "37"},
+		},
+		DateRange:    "20190820-" + currentTime,
+		ProductCode:  "FW4002D",
+		QuotaName:    "monthly_value,monthly_accumulation",
+		SplitTypeKey: "province",
+		IsTotal:      0,
+		DataType:     nil,
+		IsSeason:     1,
+	}
+
+	// 将结构体转换为 JSON 字符串
+	paramsJSON, _ := json.Marshal(params)
+
+	// 返回为 map[string]string 类型
+	return map[string]string{
+		"params": string(paramsJSON),
+	}
+}
+
+// CokingCleanCoalSupplyProcessor 炼焦精煤供应量
+type CokingCleanCoalSupplyProcessor struct {
+	BaseProcessor
+}
+
+func (p *CokingCleanCoalSupplyProcessor) GenerateRequestParams(currentTime string) map[string]string {
+	params := models.RequestParams{
+		Category: nil,
+		CheckedDims: map[string][]string{
+			"province":  {"20", "16", "21", "6", "19", "23", "8", "1", "7", "17", "9", "15", "11", "22", "24", "4", "25", "12", "14", "13", "18", "3", "10", "5", "2", "37"},
+			"coal_type": {"13", "10", "9", "8", "7", "5", "3"},
+		},
+		DateRange:    "20190820-" + currentTime,
+		ProductCode:  "FW4001D",
+		QuotaName:    "monthly_value,monthly_accumulation",
+		SplitTypeKey: "coal_type",
+		IsTotal:      0,
+		DataType:     nil,
+		IsSeason:     1,
+	}
+
+	// 将结构体转换为 JSON 字符串
+	paramsJSON, _ := json.Marshal(params)
+
+	// 返回为 map[string]string 类型
+	return map[string]string{
+		"params": string(paramsJSON),
+	}
+}
+
+// RawCoalProvinceProductionProcessor 原煤分省分煤种产量
+type RawCoalProvinceProductionProcessor struct {
+	BaseProcessor
+}
+
+func (p *RawCoalProvinceProductionProcessor) GenerateRequestParams(currentTime string) map[string]string {
+	params := models.RequestParams{
+		Category: nil,
+		CheckedDims: map[string][]string{
+			"province":  {"20", "16", "21", "6", "19", "23", "8", "1", "7", "17", "9", "15", "11", "22", "24", "4", "25", "12", "14", "13", "18", "3", "10", "5", "2", "37"},
+			"coal_type": {"14", "6", "40", "21"},
+		},
+		DateRange:    "20190820-" + currentTime,
+		ProductCode:  "FW3050D",
+		QuotaName:    "monthly_value,monthly_accumulation",
+		SplitTypeKey: "coal_type",
+		IsTotal:      0,
+		DataType:     nil,
+		IsSeason:     1,
+	}
+
+	// 将结构体转换为 JSON 字符串
+	paramsJSON, _ := json.Marshal(params)
+
+	// 返回为 map[string]string 类型
+	return map[string]string{
+		"params": string(paramsJSON),
+	}
+}
+
+// StateOwnedKeyCoalMineRawCoalProductionProcessor 国有重点煤矿原煤产量
+type StateOwnedKeyCoalMineRawCoalProductionProcessor struct {
+	BaseProcessor
+}
+
+func (p *StateOwnedKeyCoalMineRawCoalProductionProcessor) GenerateRequestParams(currentTime string) map[string]string {
+	params := models.RequestParams{
+		Category: nil,
+		CheckedDims: map[string][]string{
+			"province": {"20", "16", "21", "6", "19", "23", "8", "1", "7", "17", "9", "15", "11", "22", "24", "4", "25", "12", "14", "13", "2", "27", "28"},
+		},
+		DateRange:    "20190820-" + currentTime,
+		ProductCode:  "FW3007D",
+		QuotaName:    "monthly_value,monthly_accumulation",
+		SplitTypeKey: "province",
+		IsTotal:      0,
+		DataType:     nil,
+		IsSeason:     1,
+	}
+
+	// 将结构体转换为 JSON 字符串
+	paramsJSON, _ := json.Marshal(params)
+
+	// 返回为 map[string]string 类型
+	return map[string]string{
+		"params": string(paramsJSON),
+	}
+}
+
+// CokingBituminousCoalProductionProcessor 炼焦烟煤分煤种产量
+type CokingBituminousCoalProductionProcessor struct {
+	BaseProcessor
+}
+
+func (p *CokingBituminousCoalProductionProcessor) GenerateRequestParams(currentTime string) map[string]string {
+	params := models.RequestParams{
+		Category: nil,
+		CheckedDims: map[string][]string{
+			"province":  {"20", "16", "21", "6", "19", "23", "8", "1", "7", "17", "9", "15", "11", "22", "24", "4", "25", "12", "14", "13", "18", "3", "10", "5", "2", "37"},
+			"coal_type": {"13", "10", "9", "8", "7", "5", "3"},
+		},
+		DateRange:    "20190820-" + currentTime,
+		ProductCode:  "FW3001D",
+		QuotaName:    "monthly_value,monthly_accumulation",
+		SplitTypeKey: "coal_type",
+		IsTotal:      0,
+		DataType:     nil,
+		IsSeason:     1,
+	}
+
+	// 将结构体转换为 JSON 字符串
+	paramsJSON, _ := json.Marshal(params)
+
+	// 返回为 map[string]string 类型
+	return map[string]string{
+		"params": string(paramsJSON),
+	}
+}
+
+// ThermalCoalInventorySocietyProcessor 动力煤库存-全社会
+type ThermalCoalInventorySocietyProcessor struct {
+	BaseProcessor
+}
+
+func (p *ThermalCoalInventorySocietyProcessor) GenerateRequestParams(currentTime string) map[string]string {
+	params := models.RequestParams{
+		Category: "79",
+		CheckedDims: map[string][]string{
+			"product_item_code": {"FW8004D-1"},
+		},
+		DateRange:    "20190820-" + currentTime,
+		ProductCode:  "FW8004D",
+		QuotaName:    "stock,avaliable_days,daily_consumption,monthly_value,inventory_index,week_value",
+		SplitTypeKey: "",
+		DataType:     nil,
+		IsSeason:     1,
+	}
+
+	// 将结构体转换为 JSON 字符串
+	paramsJSON, _ := json.Marshal(params)
+
+	// 返回为 map[string]string 类型
+	return map[string]string{
+		"params": string(paramsJSON),
+	}
+}
+
+// ThermalCoalInventoryProductionProcessor 动力煤库存-生产企业
+type ThermalCoalInventoryProductionProcessor struct {
+	BaseProcessor
+}
+
+func (p *ThermalCoalInventoryProductionProcessor) GenerateRequestParams(currentTime string) map[string]string {
+	params := models.RequestParams{
+		Category: "116",
+		CheckedDims: map[string][]string{
+			"product_item_code": {"FW8004D-2"},
+		},
+		DateRange:    "20190820-" + currentTime,
+		ProductCode:  "FW8004D",
+		QuotaName:    "stock,avaliable_days,daily_consumption,monthly_value,inventory_index,week_value",
+		SplitTypeKey: "",
+		DataType:     nil,
+		IsSeason:     1,
+	}
+
+	// 将结构体转换为 JSON 字符串
+	paramsJSON, _ := json.Marshal(params)
+
+	// 返回为 map[string]string 类型
+	return map[string]string{
+		"params": string(paramsJSON),
+	}
+}
+
+// ThermalCoalInventorySixPowerPlantProcessor 动力煤库存-六大电厂
+type ThermalCoalInventorySixPowerPlantProcessor struct {
+	BaseProcessor
+}
+
+func (p *ThermalCoalInventorySixPowerPlantProcessor) GenerateRequestParams(currentTime string) map[string]string {
+	params := models.RequestParams{
+		Category: "118",
+		CheckedDims: map[string][]string{
+			"product_item_code": {"FW8004D-125500", "FW8004D-112534", "FW8004D-112548", "FW8004D-112549", "FW8004D-1129", "FW8004D-125", "FW8004D-125382", "FW8004D-125383"},
+		},
+		DateRange:    "20190820-" + currentTime,
+		ProductCode:  "FW8004D",
+		QuotaName:    "stock,avaliable_days,daily_consumption,monthly_value,inventory_index,week_value",
+		SplitTypeKey: "",
+		DataType:     nil,
+		IsSeason:     1,
+	}
+
+	// 将结构体转换为 JSON 字符串
+	paramsJSON, _ := json.Marshal(params)
+
+	// 返回为 map[string]string 类型
+	return map[string]string{
+		"params": string(paramsJSON),
+	}
+}
+
+// CokingCoalInventorySocietyProcessor 炼焦煤库存-全社会
+type CokingCoalInventorySocietyProcessor struct {
+	BaseProcessor
+}
+
+func (p *CokingCoalInventorySocietyProcessor) GenerateRequestParams(currentTime string) map[string]string {
+	params := models.RequestParams{
+		Category: "79",
+		CheckedDims: map[string][]string{
+			"product_item_code": {"FW8005D-1"},
+		},
+		DateRange:    "20190820-" + currentTime,
+		ProductCode:  "FW8005D",
+		QuotaName:    "stock,monthly_value,inventory_index,week_value",
+		SplitTypeKey: "",
+		DataType:     nil,
+		IsSeason:     1,
+	}
+
+	// 将结构体转换为 JSON 字符串
+	paramsJSON, _ := json.Marshal(params)
+
+	// 返回为 map[string]string 类型
+	return map[string]string{
+		"params": string(paramsJSON),
+	}
+}
+
+// CokingCoalInventoryProductionProcessor 炼焦煤库存-生产企业
+type CokingCoalInventoryProductionProcessor struct {
+	BaseProcessor
+}
+
+func (p *CokingCoalInventoryProductionProcessor) GenerateRequestParams(currentTime string) map[string]string {
+	params := models.RequestParams{
+		Category: "116",
+		CheckedDims: map[string][]string{
+			"product_item_code": {"FW8005D-2"},
+		},
+		DateRange:    "20190820-" + currentTime,
+		ProductCode:  "FW8005D",
+		QuotaName:    "stock,monthly_value,inventory_index,week_value",
+		SplitTypeKey: "",
+		DataType:     nil,
+		IsSeason:     1,
+	}
+
+	// 将结构体转换为 JSON 字符串
+	paramsJSON, _ := json.Marshal(params)
+
+	// 返回为 map[string]string 类型
+	return map[string]string{
+		"params": string(paramsJSON),
+	}
+}
+
+// CokingCoalInventoryDownstreamProcessor 炼焦煤库存-下游企业
+type CokingCoalInventoryDownstreamProcessor struct {
+	BaseProcessor
+}
+
+func (p *CokingCoalInventoryDownstreamProcessor) GenerateRequestParams(currentTime string) map[string]string {
+	params := models.RequestParams{
+		Category: "117",
+		CheckedDims: map[string][]string{
+			"product_item_code": {"FW8005D-3", "FW8005D-5"},
+		},
+		DateRange:    "20190820-" + currentTime,
+		ProductCode:  "FW8005D",
+		QuotaName:    "stock,monthly_value,inventory_index,week_value",
+		SplitTypeKey: "",
+		DataType:     nil,
+		IsSeason:     1,
+	}
+
+	// 将结构体转换为 JSON 字符串
+	paramsJSON, _ := json.Marshal(params)
+
+	// 返回为 map[string]string 类型
+	return map[string]string{
+		"params": string(paramsJSON),
+	}
+}
+
+// NationalCoalMineInventoryProcessor 全国煤矿库存
+type NationalCoalMineInventoryProcessor struct {
+	BaseProcessor
+}
+
+func (p *NationalCoalMineInventoryProcessor) GenerateRequestParams(currentTime string) map[string]string {
+	params := models.RequestParams{
+		Category:     nil,
+		CheckedDims:  map[string][]string{},
+		DateRange:    "20190820-" + currentTime,
+		ProductCode:  "FW8001D",
+		QuotaName:    "monthly_value",
+		SplitTypeKey: "",
+		IsTotal:      0,
+		DataType:     nil,
+		IsSeason:     1,
+	}
+
+	// 将结构体转换为 JSON 字符串
+	paramsJSON, _ := json.Marshal(params)
+
+	// 返回为 map[string]string 类型
+	return map[string]string{
+		"params": string(paramsJSON),
+	}
+}
+
+// StateOwnedKeyCoalMineInventoryProcessor 国有重点煤矿库存
+type StateOwnedKeyCoalMineInventoryProcessor struct {
+	BaseProcessor
+}
+
+func (p *StateOwnedKeyCoalMineInventoryProcessor) GenerateRequestParams(currentTime string) map[string]string {
+	params := models.RequestParams{
+		Category: nil,
+		CheckedDims: map[string][]string{
+			"province": {"20", "16", "21", "6", "19", "23", "8", "1", "7", "17", "9", "15", "11", "22", "24", "4", "25", "12", "14", "13", "2", "27", "28"},
+		},
+		DateRange:    "20190820-" + currentTime,
+		ProductCode:  "FW8003D",
+		QuotaName:    "monthly_value",
+		SplitTypeKey: "province",
+		IsTotal:      0,
+		DataType:     nil,
+		IsSeason:     1,
+	}
+
+	// 将结构体转换为 JSON 字符串
+	paramsJSON, _ := json.Marshal(params)
+
+	// 返回为 map[string]string 类型
+	return map[string]string{
+		"params": string(paramsJSON),
+	}
+}
+
+// CokeInventoryProcessor 焦炭库存
+type CokeInventoryProcessor struct {
+	BaseProcessor
+}
+
+func (p *CokeInventoryProcessor) GenerateRequestParams(currentTime string) map[string]string {
+	params := models.RequestParams{
+		Category: nil,
+		CheckedDims: map[string][]string{
+			"product_item_code": {"FW1405D-1", "FW1405D-2", "FW1405D-3", "FW1405D-4", "FW1405D-5", "FW1405D-6", "FW1405D-7"},
+		},
+		DateRange:    "20190820-" + currentTime,
+		ProductCode:  "FW1405D",
+		QuotaName:    "stock,inventory_index",
+		SplitTypeKey: "",
+		DataType:     nil,
+		IsSeason:     1,
+	}
+
+	// 将结构体转换为 JSON 字符串
+	paramsJSON, _ := json.Marshal(params)
+
+	// 返回为 map[string]string 类型
+	return map[string]string{
+		"params": string(paramsJSON),
+	}
+}
+
+// PortDataInventoryNorthernPortProcessor 港口数据-库存-北方港口
+type PortDataInventoryNorthernPortProcessor struct {
+	BaseProcessor
+}
+
+func (p *PortDataInventoryNorthernPortProcessor) GenerateRequestParams(currentTime string) map[string]string {
+	params := models.RequestParams{
+		Category: "79",
+		CheckedDims: map[string][]string{
+			"code": {"2332", "2333", "2335", "2334", "2337", "2339", "2340", "2341", "2342"},
+		},
+		DateRange:    "20190820-" + currentTime,
+		ProductCode:  "FW1203D",
+		QuotaName:    "stock",
+		SplitTypeKey: "",
+		IsTotal:      0,
+		DataType:     nil,
+		Type:         2,
+		IsSeason:     1,
+	}
+
+	// 将结构体转换为 JSON 字符串
+	paramsJSON, _ := json.Marshal(params)
+
+	// 返回为 map[string]string 类型
+	return map[string]string{
+		"params": string(paramsJSON),
+	}
+}
+
+// PortDataInventoryInlandPortProcessor 港口数据-库存-江内港口
+type PortDataInventoryInlandPortProcessor struct {
+	BaseProcessor
+}
+
+func (p *PortDataInventoryInlandPortProcessor) GenerateRequestParams(currentTime string) map[string]string {
+	params := models.RequestParams{
+		Category: "79",
+		CheckedDims: map[string][]string{
+			"code": {"2321", "2320", "2324", "2323", "2322", "2325"},
+		},
+		DateRange:    "20190820-" + currentTime,
+		ProductCode:  "FW1203D",
+		QuotaName:    "stock",
+		SplitTypeKey: "product_item_code",
+		IsTotal:      0,
+		DataType:     nil,
+		Type:         2,
+		IsSeason:     1,
+	}
+
+	// 将结构体转换为 JSON 字符串
+	paramsJSON, _ := json.Marshal(params)
+
+	// 返回为 map[string]string 类型
+	return map[string]string{
+		"params": string(paramsJSON),
+	}
+}
+
+// PortDataDispatchNorthernPortProcessor 港口数据-调度-北方港口
+type PortDataDispatchNorthernPortProcessor struct {
+	BaseProcessor
+}
+
+func (p *PortDataDispatchNorthernPortProcessor) GenerateRequestParams(currentTime string) map[string]string {
+	params := models.RequestParams{
+		Category: "120",
+		CheckedDims: map[string][]string{
+			"code": {"2364", "2365", "2366", "2363", "2367", "2368"},
+			"port": {"2", "4", "7", "9", "11", "326", "327", "329", "330"},
+		},
+		DateRange:    "20190820-" + currentTime,
+		ProductCode:  "FW1203D",
+		QuotaName:    "vehicle,ship,tons",
+		SplitTypeKey: "product_item_code,port",
+		IsTotal:      1,
+		DataType:     nil,
+		Type:         2,
+		IsSeason:     1,
+	}
+
+	// 将结构体转换为 JSON 字符串
+	paramsJSON, _ := json.Marshal(params)
+
+	// 返回为 map[string]string 类型
+	return map[string]string{
+		"params": string(paramsJSON),
+	}
+}
+
+// PortDataThroughputProcessor 港口数据-运量
+type PortDataThroughputProcessor struct {
+	BaseProcessor
+}
+
+func (p *PortDataThroughputProcessor) GenerateRequestParams(currentTime string) map[string]string {
+	params := models.RequestParams{
+		Category: "30",
+		CheckedDims: map[string][]string{
+			"code":     {"2362"},
+			"port_son": {"44", "43", "42", "41", "40", "39"},
+		},
+		DateRange:    "20190820-" + currentTime,
+		ProductCode:  "FW1203D",
+		QuotaName:    "transport_volume",
+		SplitTypeKey: "port_son",
+		IsTotal:      0,
+		DataType:     nil,
+		Type:         2,
+		IsSeason:     1,
+	}
+
+	// 将结构体转换为 JSON 字符串
+	paramsJSON, _ := json.Marshal(params)
+
+	// 返回为 map[string]string 类型
+	return map[string]string{
+		"params": string(paramsJSON),
+	}
+}
+
+// DaqinLineDailyThroughputProcessor 大秦线日运量
+type DaqinLineDailyThroughputProcessor struct {
+	BaseProcessor
+}
+
+func (p *DaqinLineDailyThroughputProcessor) GenerateRequestParams(currentTime string) map[string]string {
+	params := models.RequestParams{
+		Category:     nil,
+		CheckedDims:  map[string][]string{},
+		DateRange:    "20190820-" + currentTime,
+		ProductCode:  "FW1107D",
+		QuotaName:    "stock",
+		SplitTypeKey: "",
+		DataType:     nil,
+		IsSeason:     1,
+	}
+
+	// 将结构体转换为 JSON 字符串
+	paramsJSON, _ := json.Marshal(params)
+
+	// 返回为 map[string]string 类型
+	return map[string]string{
+		"params": string(paramsJSON),
+	}
+}
+
+// ThermalCoalPortPriceProcessor 动力煤港口价格
+type ThermalCoalPortPriceProcessor struct {
+	BaseProcessor
+}
+
+func (p *ThermalCoalPortPriceProcessor) GenerateRequestParams(currentTime string) map[string]string {
+	params := models.RequestParams{
+		Category: nil,
+		CheckedDims: map[string][]string{
+			"product_item_code": {"FW2001P-1001", "FW2001P-1002", "FW2001P-1004", "FW2001P-1005", "FW2001P-1003", "FW2001P-1171"},
+		},
+		DateRange:    "20190820-" + currentTime,
+		ProductCode:  "FW2310R",
+		QuotaName:    "price_rmb,price_index_mom",
+		SplitTypeKey: "",
+		DataType:     nil,
+		IsSeason:     1,
+	}
+
+	// 将结构体转换为 JSON 字符串
+	paramsJSON, _ := json.Marshal(params)
+
+	// 返回为 map[string]string 类型
+	return map[string]string{
+		"params": string(paramsJSON),
+	}
+}
+
+// ThermalCoalConsumptionProcessor 动力煤消费量
+type ThermalCoalConsumptionProcessor struct {
+	BaseProcessor
+}
+
+func (p *ThermalCoalConsumptionProcessor) GenerateRequestParams(currentTime string) map[string]string {
+	params := models.RequestParams{
+		Category: nil,
+		CheckedDims: map[string][]string{
+			"product_item_code": {"FW5002D-1", "FW5002D-2", "FW5002D-3", "FW5002D-4", "FW5002D-5", "FW5002D-6", "FW5002D-7"},
+		},
+		DateRange:    "20190820-" + currentTime,
+		ProductCode:  "FW5002D",
+		QuotaName:    "monthly_value,monthly_accumulation",
+		SplitTypeKey: "",
+		DataType:     nil,
+		IsSeason:     1,
+	}
+
+	// 将结构体转换为 JSON 字符串
+	paramsJSON, _ := json.Marshal(params)
+
+	// 返回为 map[string]string 类型
+	return map[string]string{
+		"params": string(paramsJSON),
+	}
+}
+
+// CokingCleanCoalConsumptionProcessor 炼焦精煤消费量
+type CokingCleanCoalConsumptionProcessor struct {
+	BaseProcessor
+}
+
+func (p *CokingCleanCoalConsumptionProcessor) GenerateRequestParams(currentTime string) map[string]string {
+	params := models.RequestParams{
+		Category:     nil,
+		CheckedDims:  map[string][]string{},
+		DateRange:    "20190820-" + currentTime,
+		ProductCode:  "FW5001D",
+		QuotaName:    "monthly_value,monthly_accumulation",
+		SplitTypeKey: "",
+		DataType:     nil,
+		IsSeason:     1,
+	}
+
+	// 将结构体转换为 JSON 字符串
+	paramsJSON, _ := json.Marshal(params)
+
+	// 返回为 map[string]string 类型
+	return map[string]string{
+		"params": string(paramsJSON),
+	}
+}

services/fenwei/processor_factory.go

@@ -0,0 +1,106 @@
+// Package fenwei
+// @Author gmy 2024/8/20 14:50:00
+package fenwei
+
+const (
+	url   = "https://www.sxcoal.com/api/coalresource-adhoc/queryV1/data"
+	byUrl = "https://www.sxcoal.com/api/coalresource-adhoc/queryV1/byData"
+)
+
+type ProcessorFactory struct{}
+
+func (f *ProcessorFactory) CreateProcessor(module string) DataProcessor {
+	switch module {
+	case "动力煤供应量":
+		return &ThermalCoalSupplyProcessor{
+			BaseProcessor{URL: url},
+		}
+	case "炼焦精煤供应量":
+		return &CokingCleanCoalSupplyProcessor{
+			BaseProcessor{URL: url},
+		}
+	case "原煤分省分煤种产量":
+		return &RawCoalProvinceProductionProcessor{
+			BaseProcessor{URL: url},
+		}
+	case "国有重点煤矿原煤产量":
+		return &StateOwnedKeyCoalMineRawCoalProductionProcessor{
+			BaseProcessor{URL: url},
+		}
+	case "炼焦烟煤分煤种产量":
+		return &CokingBituminousCoalProductionProcessor{
+			BaseProcessor{URL: url},
+		}
+	case "动力煤库存-全社会":
+		return &ThermalCoalInventorySocietyProcessor{
+			BaseProcessor{URL: url},
+		}
+	case "动力煤库存-生产企业":
+		return &ThermalCoalInventoryProductionProcessor{
+			BaseProcessor{URL: url},
+		}
+	case "动力煤库存-六大电厂":
+		return &ThermalCoalInventorySixPowerPlantProcessor{
+			BaseProcessor{URL: url},
+		}
+	case "炼焦煤库存-全社会":
+		return &CokingCoalInventorySocietyProcessor{
+			BaseProcessor{URL: url},
+		}
+	case "炼焦煤库存-生产企业":
+		return &CokingCoalInventoryProductionProcessor{
+			BaseProcessor{URL: url},
+		}
+	case "炼焦煤库存-下游企业":
+		return &CokingCoalInventoryDownstreamProcessor{
+			BaseProcessor{URL: url},
+		}
+	case "全国煤矿库存":
+		return &NationalCoalMineInventoryProcessor{
+			BaseProcessor{URL: url},
+		}
+	case "国有重点煤矿库存":
+		return &StateOwnedKeyCoalMineInventoryProcessor{
+			BaseProcessor{URL: url},
+		}
+	case "焦炭库存":
+		return &CokeInventoryProcessor{
+			BaseProcessor{URL: url},
+		}
+	case "港口数据-库存-北方港口":
+		return &PortDataInventoryNorthernPortProcessor{
+			BaseProcessor{URL: byUrl},
+		}
+	case "港口数据-库存-江内港口":
+		return &PortDataInventoryInlandPortProcessor{
+			BaseProcessor{URL: byUrl},
+		}
+	case "港口数据-调度-北方港口":
+		return &PortDataDispatchNorthernPortProcessor{
+			BaseProcessor{URL: byUrl},
+		}
+	case "港口数据-运量":
+		return &PortDataThroughputProcessor{
+			BaseProcessor{URL: byUrl},
+		}
+	case "大秦线日运量":
+		return &DaqinLineDailyThroughputProcessor{
+			BaseProcessor{URL: url},
+		}
+	case "动力煤港口价格":
+		return &ThermalCoalPortPriceProcessor{
+			BaseProcessor{URL: url},
+		}
+	case "动力煤消费量":
+		return &ThermalCoalConsumptionProcessor{
+			BaseProcessor{URL: url},
+		}
+	case "炼焦精煤消费量":
+		return &CokingCleanCoalConsumptionProcessor{
+			BaseProcessor{URL: url},
+		}
+
+	default:
+		return nil
+	}
+}

static/fen_wei_net_data_json.json

@@ -0,0 +1,26 @@
+{
+  "data": [
+    "动力煤供应量",
+    "炼焦精煤供应量",
+    "原煤分省分煤种产量",
+    "国有重点煤矿原煤产量",
+    "炼焦烟煤分煤种产量",
+    "动力煤库存-全社会",
+    "动力煤库存-生产企业",
+    "动力煤库存-六大电厂",
+    "炼焦煤库存-全社会",
+    "炼焦煤库存-生产企业",
+    "炼焦煤库存-下游企业",
+    "全国煤矿库存",
+    "国有重点煤矿库存",
+    "焦炭库存",
+    "港口数据-库存-北方港口",
+    "港口数据-库存-江内港口",
+    "港口数据-调度-北方港口",
+    "港口数据-运量",
+    "大秦线日运量",
+    "动力煤港口价格",
+    "动力煤消费量",
+    "炼焦精煤消费量"
+  ]
+}

utils/common.go

@@ -1272,3 +1272,5 @@ func MkDir(dirPath string) error {
 	}
 	return nil
 }
+
+// 根据传入时间，将

utils/config.go

@@ -59,6 +59,10 @@ var (
 	FenweiFileDir    string // excel文件目录
 	FenweiOldFileDir string // 已读取过的excel文件目录
 	FenweiOpen       string // 是否配置汾渭数据源
+
+	FenweiNetOpen     string // 是否配置汾渭网页数据源
+	FenweiNetUseName  string // 汾渭登录账号
+	FenweiNetPassword string // 汾渭登录密码
 )
 
 // 煤炭江湖
@@ -144,6 +148,8 @@ func init() {
 		FenweiOpen = config["fenwei_open"]
 		FenweiFileDir = config["fenwei_file_dir"]
 		FenweiOldFileDir = config["fenwei_old_file_dir"]
+
+		FenweiNetOpen = config["fenwei_net_open"]
 	}
 
 	//煤炭江湖文件夹配置