Merge branch 'refs/heads/eta_2.0.7_fenwei_0820@guomengyuan'

# Conflicts:
#	services/task.go
#	utils/common.go
#	utils/constants.go

models/base_from_fenwei.go

@@ -58,3 +58,102 @@ type EdbLibFenweiIndexListResponse struct {
 	ErrCode string
 	Data    []BaseFromFenweiIndex
 }
+
+// RequestParams 接口爬取汾渭网页数据
+type RequestParams struct {
+	Category     interface{}         `json:"category"`
+	CheckedDims  map[string][]string `json:"checkedDims"`
+	DateRange    string              `json:"dateRange"`
+	ProductCode  string              `json:"productCode"`
+	QuotaName    string              `json:"quotaName"`
+	SplitTypeKey string              `json:"splitTypeKey"`
+	IsTotal      interface{}         `json:"isTotal"`
+	DataType     interface{}         `json:"dataType"`
+	Type         int                 `json:"type"`
+	IsSeason     int                 `json:"isSeason"`
+	splitTypeKey string              `json:"splitTypeKey"`
+}
+
+type Response struct {
+	Code    int    `json:"code"`
+	Message string `json:"message"`
+}
+
+type JsonConfig struct {
+	Data []string `json:"data"`
+}
+
+type FenWeiNetResponse struct {
+	MonthlyAccumulation        float64     `json:"monthly_accumulation"`
+	ProductItemCode            string      `json:"product_item_code"`
+	MonthlyValueChnName        string      `json:"monthly_value_chn_name"`
+	MonthlyAccumulationChnName string      `json:"monthly_accumulation_chn_name"`
+	ProductItemName            string      `json:"product_item_name"`
+	DataDate                   string      `json:"data_date"`
+	MonthlyValue               float64     `json:"monthly_value"`
+	MonthlyValueChnUnit        string      `json:"monthly_value_chn_unit"`
+	ProductCode                string      `json:"product_code"`
+	ProvinceName               string      `json:"province_name"`
+	MonthlyAccumulationChnUnit string      `json:"monthly_accumulation_chn_unit"`
+	PortName                   string      `json:"port_name"`
+	PortSonName                string      `json:"port_son_name"`
+	Stock                      float64     `json:"stock"`
+	StockChnName               string      `json:"stock_chn_name"`
+	StockChnUnit               string      `json:"stock_chn_unit"`
+	CoalTypeName               string      `json:"coal_type_name"`
+	WeekValueChnName           string      `json:"week_value_chn_name"`
+	WeekValue                  float64     `json:"week_value"`
+	AvaliableDaysChnName       string      `json:"avaliable_days_chn_name"`
+	AvaliableDaysChnUnit       string      `json:"avaliable_days_chn_unit"`
+	AvaliableDays              float64     `json:"avaliable_days"`
+	DailyConsumptionChnName    string      `json:"daily_consumption_chn_name"`
+	DailyConsumptionChnUnit    string      `json:"daily_consumption_chn_unit"`
+	DailyConsumption           float64     `json:"daily_consumption"`
+	InventoryIndexChnUnit      string      `json:"inventory_index_chn_unit"`
+	InventoryIndex             float64     `json:"inventory_index"`
+	TonsChnUnit                string      `json:"tons_chn_unit"`
+	Tons                       float64     `json:"tons"`
+	TransportVolumeChnUnit     string      `json:"transport_volume_chn_unit"`
+	TransportVolume            float64     `json:"transport_volume"`
+	PriceRmbChnName            string      `json:"price_rmb_chn_name"`
+	PriceIndexMomChnName       string      `json:"price_index_mom_chn_name"`
+	PriceRmbChnUnit            string      `json:"price_rmb_chn_unit"`
+	PriceIndexMomChnUnit       string      `json:"price_index_mom_chn_unit"`
+	PriceRmb                   interface{} `json:"price_rmb"`
+	PriceIndexMom              interface{} `json:"price_index_mom"`
+	VehicleChnUnit             string      `json:"vehicle_chn_unit"`
+	ShipChnUnit                string      `json:"ship_chn_unit"`
+	WeekValueChnUnit           string      `json:"week_value_chn_unit"`
+}
+
+type FenWeiNetResponseWrapper struct {
+	Data struct {
+		Data []FenWeiNetResponse `json:"data"`
+	} `json:"data"`
+}
+
+type FenWeiNetResponseMapWrapper struct {
+	Data struct {
+		Data map[string][]FenWeiNetResponse `json:"data"`
+	} `json:"data"`
+}
+
+type FenWeiNetResponseAStratumMapWrapper struct {
+	Data map[string][]FenWeiNetResponse `json:"data"`
+}
+
+type FenWeiNetResponseResult struct {
+	Product            string              `json:"product"`
+	FenWeiNetResponses []FenWeiNetResponse `json:"fenWeiNetResponses"`
+}
+
+type FenWeiNetIndexInfo struct {
+	IndexName string `description:"指标名称"`
+	//IndexCode    string  `description:"指标编码"`
+	Unit         string      `description:"单位"`
+	Frequency    string      `description:"频度"`
+	TerminalCode string      `description:"编码"`
+	ClassifyName string      `description:"分类名称"`
+	DataTime     string      `description:"数据时间"`
+	Value        interface{} `description:"数据值"`
+}

services/fenwei/base_from_fenwei_service.go

@@ -0,0 +1,52 @@
+// Package fenwei
+// @Author gmy 2024/8/20 15:06:00
+package fenwei
+
+import (
+	"context"
+	"encoding/json"
+	"eta/eta_data_analysis/models"
+	"eta/eta_data_analysis/utils"
+	"fmt"
+	"os"
+)
+
+// FenWeiNetDataDeal 汾渭网络数据处理
+func FenWeiNetDataDeal(context.Context) (err error) {
+	defer func() {
+		if err != nil {
+			fmt.Println("FenWeiNetDataDeal Err:" + err.Error())
+			utils.FileLog.Info(fmt.Sprintf("FenWeiNetDataDeal Err: %s", err.Error()))
+		}
+	}()
+	utils.FileLog.Info("FenWeiNetDataDeal start")
+	// 读取配置
+	configFile, err := os.ReadFile(utils.FenweiNetJsonPath)
+	if err != nil {
+		utils.FileLog.Info(fmt.Sprintf("读取配置文件错误: %v", err))
+		return
+	}
+
+	// 定义通用的 map 结构体来解析 JSON
+	var config models.JsonConfig
+
+	// 解析 JSON 文件内容
+	err = json.Unmarshal(configFile, &config)
+	if err != nil {
+		utils.FileLog.Info(fmt.Sprintf("解析配置文件错误: %v", err))
+		return
+	}
+
+	factory := ProcessorFactory{}
+	// 遍历调用对应的处理方法
+	for _, v := range config.Data {
+		processor := factory.CreateProcessor(v)
+		err = processor.FetchAndProcess(processor)
+		if err != nil {
+			utils.FileLog.Info(fmt.Sprintf("处理数据错误: %v", err))
+			return
+		}
+	}
+	utils.FileLog.Info("FenWeiNetDataDeal end")
+	return
+}

services/fenwei/data_processor.go

@@ -0,0 +1,224 @@
+// Package fenwei
+// @Author gmy 2024/8/20 14:47:00
+package fenwei
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"eta/eta_data_analysis/models"
+	"eta/eta_data_analysis/services"
+	"eta/eta_data_analysis/utils"
+	"fmt"
+	"github.com/chromedp/cdproto/network"
+	"github.com/chromedp/chromedp"
+	"io"
+	"log"
+	"net/http"
+	"sync"
+	"time"
+)
+
+type DataProcessor interface {
+	FetchAndProcess(DataProcessor) error
+	GenerateRequestParams(currentTime string) map[string]string
+	ProcessResponse(data string) ([]models.FenWeiNetIndexInfo, error)
+}
+
+type BaseProcessor struct {
+	URL string
+}
+
+var (
+	authorization string
+	authLock      sync.RWMutex
+)
+
+func (p *BaseProcessor) FetchAndProcess(processor DataProcessor) error {
+
+	// 获取当前时间 yyyy-MM-dd
+	now := time.Now()
+	currentTime := now.Format(utils.FormatDateUnSpace)
+
+	// 请求参数
+	params := processor.GenerateRequestParams(currentTime)
+	// 保存请求参数
+	originalRequestBody := params["params"] // 保存原始请求数据
+
+	requestBody := bytes.NewBufferString(originalRequestBody)
+	req, err := http.NewRequest("POST", p.URL, requestBody)
+	if err != nil {
+		return err
+	}
+
+	// 设置请求头
+	req.Header.Set("Content-Type", "application/json")
+	req.Header.Set("accept-language", "zh-CN,zh;q=0.9")
+
+	authLock.RLock()
+	req.Header.Set("Authorization", authorization)
+	authLock.RUnlock()
+
+	client := &http.Client{}
+	resp, err := client.Do(req)
+	if err != nil {
+		return err
+	}
+	defer resp.Body.Close()
+
+	body, err := io.ReadAll(resp.Body)
+	if err != nil {
+		return err
+	}
+	if checkResp(string(body)) {
+		authLock.Lock()
+
+		// 登录获取Authorization
+		authorization, err = getAuthorizationByChrome()
+		if err != nil {
+			authLock.Unlock()
+			return err
+		}
+		authLock.Unlock()
+
+		// 重新创建请求对象
+		requestBody = bytes.NewBufferString(originalRequestBody)
+		req, err = http.NewRequest("POST", p.URL, requestBody)
+		if err != nil {
+			return err
+		}
+
+		// 重新设置请求头
+		req.Header.Set("Content-Type", "application/json")
+		req.Header.Set("accept-language", "zh-CN,zh;q=0.9")
+		req.Header.Set("Authorization", authorization)
+
+		// 重新请求
+		resp, err = client.Do(req)
+		if err != nil {
+			return err
+		}
+		defer resp.Body.Close()
+
+		body, err = io.ReadAll(resp.Body)
+		if err != nil {
+			return err
+		}
+	}
+
+	// 数据处理
+	response, err := processor.ProcessResponse(string(body))
+	if err != nil {
+		return err
+	}
+	log.Printf("response size: %v", len(response))
+	utils.FileLog.Info(fmt.Sprintf("response: %v", response))
+
+	// 请求lib应用入库
+	paramsLib := make(map[string]interface{})
+	paramsLib["List"] = response
+	paramsLib["TerminalCode"] = utils.TerminalCode
+	postEdbLib, err := services.PostEdbLib(paramsLib, utils.LIB_ROUTE_FENWEI_NET_DATA_HANDLE)
+	if err != nil {
+		// 有错误就不继续执行
+		log.Printf("postEdbLib err: %v", err)
+		return err
+	}
+	log.Printf("postEdbLib size: %v", len(postEdbLib))
+	utils.FileLog.Info(fmt.Sprintf("postEdbLib: %v", string(postEdbLib)))
+	return nil
+}
+
+// resp响应参数检测 code or message 判断是否需要重新登录
+func checkResp(resp string) bool {
+	if resp == "" {
+		return true
+	}
+	var responseObj models.Response
+	err := json.Unmarshal([]byte(resp), &responseObj)
+	if err != nil {
+		return false
+	}
+	if responseObj.Code != 200 || responseObj.Message != "成功!" {
+		return true
+	}
+
+	return false
+}
+
+// GenerateRequestParams 让子类来实现这个方法
+func (p *BaseProcessor) GenerateRequestParams(currentTime string) map[string]string {
+	return map[string]string{}
+}
+
+func (p *BaseProcessor) ProcessResponse(data string) ([]models.FenWeiNetIndexInfo, error) {
+
+	return nil, nil
+}
+
+// GetAuthorizationByChrome 获取Authorization
+func getAuthorizationByChrome() (authorization string, err error) {
+	// 检查账号和密码是否设置
+	if utils.FenweiNetUseName == "" {
+		return "", fmt.Errorf("汾渭账号未设置")
+	}
+	if utils.FenweiNetPassword == "" {
+		return "", fmt.Errorf("汾渭密码未设置")
+	}
+
+	opts := append(
+		chromedp.DefaultExecAllocatorOptions[:],
+		chromedp.Flag("headless", false),
+	)
+	allocCtx, cancel1 := chromedp.NewExecAllocator(context.Background(), opts...)
+	defer cancel1()
+
+	// 创建chrome实例
+	ctx, cancel2 := chromedp.NewContext(
+		allocCtx,
+		chromedp.WithLogf(log.Printf),
+	)
+	defer cancel2()
+
+	// 提前设置监听器
+	authorizationChan := make(chan string, 1) // 使用channel来确保监听到的Authorization可以安全传递
+	chromedp.ListenTarget(ctx, func(ev interface{}) {
+		if ev, ok := ev.(*network.EventRequestWillBeSent); ok {
+			if authHeader, found := ev.Request.Headers["Authorization"]; found {
+				if authStr, ok := authHeader.(string); ok {
+					select {
+					case authorizationChan <- authStr: // 将Authorization放入channel
+					default:
+					}
+					utils.FileLog.Info("Authorization header found: " + authStr)
+				}
+			}
+		}
+	})
+
+	// 运行浏览器操作
+	err = chromedp.Run(ctx,
+		chromedp.Navigate(`https://www.sxcoal.com/`),
+		chromedp.Click(`.pc_content__jO_mq`, chromedp.ByQuery),
+		chromedp.Sleep(2*time.Second),
+
+		chromedp.SetValue(`div.Sign_username__7eYwE input[type="text"]`, utils.FenweiNetUseName, chromedp.ByQuery),
+		chromedp.SetValue(`div.Sign_password__dwxMn input[type="password"]`, utils.FenweiNetPassword, chromedp.ByQuery),
+		chromedp.Sleep(2*time.Second),
+
+		// 定位并点击指定按钮（class属性为 'Button_btn__xbZjp Button_black__X_jwF Button_mediu__ZVHO_'，并且 span 标签内容为 '登录'）
+		chromedp.Click(`//button[contains(@class, 'Button_btn__xbZjp') and contains(@class, 'Button_black__X_jwF') and contains(@class, 'Button_mediu__ZVHO_')]/span[text()='登录']`, chromedp.BySearch),
+
+		// 添加延迟，确保捕获到请求头
+		chromedp.Sleep(8*time.Second),
+	)
+
+	// 从channel中获取authorization
+	select {
+	case authorization = <-authorizationChan:
+	case <-time.After(10 * time.Second): // 超时时间，可以根据实际情况调整
+		err = fmt.Errorf("未能获取到Authorization")
+	}
+
+	return
+}

services/fenwei/processor_business_logic.go

@@ -0,0 +1,1378 @@
+// Package fenwei
+// @Author gmy 2024/8/20 14:47:00
+package fenwei
+
+import (
+	"encoding/json"
+	"eta/eta_data_analysis/models"
+	"eta/eta_data_analysis/utils"
+	"fmt"
+	"strings"
+)
+
+// ThermalCoalSupplyProcessor 动力煤供应量
+type ThermalCoalSupplyProcessor struct {
+	BaseProcessor
+}
+
+func (p *ThermalCoalSupplyProcessor) GenerateRequestParams(currentTime string) map[string]string {
+	params := models.RequestParams{
+		Category: nil,
+		CheckedDims: map[string][]string{
+			"province": {"20", "16", "21", "6", "19", "23", "8", "1", "7", "17", "9", "15", "11", "22", "24", "4", "25", "12", "14", "13", "18", "3", "10", "5", "2", "37"},
+		},
+		DateRange:    "20190820-" + currentTime,
+		ProductCode:  "FW4002D",
+		QuotaName:    "monthly_value,monthly_accumulation",
+		SplitTypeKey: "province",
+		IsTotal:      0,
+		DataType:     nil,
+		IsSeason:     1,
+	}
+
+	// 将结构体转换为 JSON 字符串
+	paramsJSON, _ := json.Marshal(params)
+
+	// 返回为 map[string]string 类型
+	return map[string]string{
+		"params": string(paramsJSON),
+	}
+}
+
+func (p *ThermalCoalSupplyProcessor) ProcessResponse(data string) ([]models.FenWeiNetIndexInfo, error) {
+	responseData := fillFenWeiNetResponseData(data)
+
+	var indexInfoList []models.FenWeiNetIndexInfo
+	productName := "动力煤供应量"
+	for _, data := range responseData {
+		var frequency string
+		if data.MonthlyValueChnName != "" && strings.Contains(data.MonthlyValueChnName, "月度") {
+			frequency = "月度"
+		} else if data.WeekValueChnName != "" && strings.Contains(data.WeekValueChnName, "周度") {
+			frequency = "周度"
+		} else {
+			frequency = "日度"
+		}
+
+		indexInfoList = append(indexInfoList, models.FenWeiNetIndexInfo{
+			IndexName:    productName + data.ProvinceName + "/汾渭",
+			Unit:         data.MonthlyValueChnUnit,
+			Frequency:    frequency,
+			TerminalCode: utils.TerminalCode,
+			ClassifyName: productName,
+			DataTime:     data.DataDate,
+			Value:        data.MonthlyValue,
+		})
+	}
+
+	return indexInfoList, nil
+}
+
+// CokingCleanCoalSupplyProcessor 炼焦精煤供应量
+type CokingCleanCoalSupplyProcessor struct {
+	BaseProcessor
+}
+
+func (p *CokingCleanCoalSupplyProcessor) GenerateRequestParams(currentTime string) map[string]string {
+	params := models.RequestParams{
+		Category: nil,
+		CheckedDims: map[string][]string{
+			"province":  {"20", "16", "21", "6", "19", "23", "8", "1", "7", "17", "9", "15", "11", "22", "24", "4", "25", "12", "14", "13", "18", "3", "10", "5", "2", "37"},
+			"coal_type": {"13", "10", "9", "8", "7", "5", "3"},
+		},
+		DateRange:    "20190820-" + currentTime,
+		ProductCode:  "FW4001D",
+		QuotaName:    "monthly_value,monthly_accumulation",
+		SplitTypeKey: "coal_type",
+		IsTotal:      0,
+		DataType:     nil,
+		IsSeason:     1,
+	}
+
+	// 将结构体转换为 JSON 字符串
+	paramsJSON, _ := json.Marshal(params)
+
+	// 返回为 map[string]string 类型
+	return map[string]string{
+		"params": string(paramsJSON),
+	}
+}
+
+func (p *CokingCleanCoalSupplyProcessor) ProcessResponse(data string) ([]models.FenWeiNetIndexInfo, error) {
+	responseData := fillFenWeiNetResponseData(data)
+
+	var indexInfoList []models.FenWeiNetIndexInfo
+	productName := "炼焦精煤供应量"
+	for _, data := range responseData {
+		var frequency string
+		if data.MonthlyValueChnName != "" && strings.Contains(data.MonthlyValueChnName, "月度") {
+			frequency = "月度"
+		} else if data.WeekValueChnName != "" && strings.Contains(data.WeekValueChnName, "周度") {
+			frequency = "周度"
+		} else {
+			frequency = data.MonthlyValueChnName
+		}
+
+		indexInfoList = append(indexInfoList, models.FenWeiNetIndexInfo{
+			IndexName:    productName + data.CoalTypeName + "/汾渭",
+			Unit:         data.MonthlyValueChnUnit,
+			Frequency:    frequency,
+			TerminalCode: utils.TerminalCode,
+			ClassifyName: productName,
+			DataTime:     data.DataDate,
+			Value:        data.MonthlyValue,
+		})
+	}
+
+	return indexInfoList, nil
+}
+
+// RawCoalProvinceProductionProcessor 原煤分省分煤种产量
+type RawCoalProvinceProductionProcessor struct {
+	BaseProcessor
+}
+
+func (p *RawCoalProvinceProductionProcessor) GenerateRequestParams(currentTime string) map[string]string {
+	params := models.RequestParams{
+		Category: nil,
+		CheckedDims: map[string][]string{
+			"province":  {"20", "16", "21", "6", "19", "23", "8", "1", "7", "17", "9", "15", "11", "22", "24", "4", "25", "12", "14", "13", "18", "3", "10", "5", "2", "37"},
+			"coal_type": {"14", "6", "40", "21"},
+		},
+		DateRange:    "20190820-" + currentTime,
+		ProductCode:  "FW3050D",
+		QuotaName:    "monthly_value,monthly_accumulation",
+		SplitTypeKey: "coal_type",
+		IsTotal:      0,
+		DataType:     nil,
+		IsSeason:     1,
+	}
+
+	// 将结构体转换为 JSON 字符串
+	paramsJSON, _ := json.Marshal(params)
+
+	// 返回为 map[string]string 类型
+	return map[string]string{
+		"params": string(paramsJSON),
+	}
+}
+
+func (p *RawCoalProvinceProductionProcessor) ProcessResponse(data string) ([]models.FenWeiNetIndexInfo, error) {
+	responseData := fillFenWeiNetResponseData(data)
+
+	var indexInfoList []models.FenWeiNetIndexInfo
+	productName := "原煤分省分煤种产量"
+	for _, data := range responseData {
+		var frequency string
+		if data.MonthlyValueChnName != "" && strings.Contains(data.MonthlyValueChnName, "月度") {
+			frequency = "月度"
+		} else if data.WeekValueChnName != "" && strings.Contains(data.WeekValueChnName, "周度") {
+			frequency = "周度"
+		} else {
+			frequency = data.MonthlyValueChnName
+		}
+
+		indexInfoList = append(indexInfoList, models.FenWeiNetIndexInfo{
+			IndexName:    productName + data.CoalTypeName + "/汾渭",
+			Unit:         data.MonthlyValueChnUnit,
+			Frequency:    frequency,
+			TerminalCode: utils.TerminalCode,
+			ClassifyName: productName,
+			DataTime:     data.DataDate,
+			Value:        data.MonthlyValue,
+		})
+	}
+
+	return indexInfoList, nil
+}
+
+// StateOwnedKeyCoalMineRawCoalProductionProcessor 国有重点煤矿原煤产量
+type StateOwnedKeyCoalMineRawCoalProductionProcessor struct {
+	BaseProcessor
+}
+
+func (p *StateOwnedKeyCoalMineRawCoalProductionProcessor) GenerateRequestParams(currentTime string) map[string]string {
+	params := models.RequestParams{
+		Category: nil,
+		CheckedDims: map[string][]string{
+			"province": {"20", "16", "21", "6", "19", "23", "8", "1", "7", "17", "9", "15", "11", "22", "24", "4", "25", "12", "14", "13", "2", "27", "28"},
+		},
+		DateRange:    "20190820-" + currentTime,
+		ProductCode:  "FW3007D",
+		QuotaName:    "monthly_value,monthly_accumulation",
+		SplitTypeKey: "province",
+		IsTotal:      0,
+		DataType:     nil,
+		IsSeason:     1,
+	}
+
+	// 将结构体转换为 JSON 字符串
+	paramsJSON, _ := json.Marshal(params)
+
+	// 返回为 map[string]string 类型
+	return map[string]string{
+		"params": string(paramsJSON),
+	}
+}
+
+func (p *StateOwnedKeyCoalMineRawCoalProductionProcessor) ProcessResponse(data string) ([]models.FenWeiNetIndexInfo, error) {
+	responseData := fillFenWeiNetResponseData(data)
+
+	var indexInfoList []models.FenWeiNetIndexInfo
+	productName := "国有重点煤矿原煤产量"
+	for _, data := range responseData {
+		var frequency string
+		if data.MonthlyValueChnName != "" && strings.Contains(data.MonthlyValueChnName, "月度") {
+			frequency = "月度"
+		} else if data.WeekValueChnName != "" && strings.Contains(data.WeekValueChnName, "周度") {
+			frequency = "周度"
+		} else {
+			frequency = data.MonthlyValueChnName
+		}
+
+		indexInfoList = append(indexInfoList, models.FenWeiNetIndexInfo{
+			IndexName:    productName + data.ProvinceName + "/汾渭",
+			Unit:         data.MonthlyValueChnUnit,
+			Frequency:    frequency,
+			TerminalCode: utils.TerminalCode,
+			ClassifyName: productName,
+			DataTime:     data.DataDate,
+			Value:        data.MonthlyValue,
+		})
+	}
+
+	return indexInfoList, nil
+}
+
+// CokingBituminousCoalProductionProcessor 炼焦烟煤分煤种产量
+type CokingBituminousCoalProductionProcessor struct {
+	BaseProcessor
+}
+
+func (p *CokingBituminousCoalProductionProcessor) GenerateRequestParams(currentTime string) map[string]string {
+	params := models.RequestParams{
+		Category: nil,
+		CheckedDims: map[string][]string{
+			"province":  {"20", "16", "21", "6", "19", "23", "8", "1", "7", "17", "9", "15", "11", "22", "24", "4", "25", "12", "14", "13", "18", "3", "10", "5", "2", "37"},
+			"coal_type": {"13", "10", "9", "8", "7", "5", "3"},
+		},
+		DateRange:    "20190820-" + currentTime,
+		ProductCode:  "FW3001D",
+		QuotaName:    "monthly_value,monthly_accumulation",
+		SplitTypeKey: "coal_type",
+		IsTotal:      0,
+		DataType:     nil,
+		IsSeason:     1,
+	}
+
+	// 将结构体转换为 JSON 字符串
+	paramsJSON, _ := json.Marshal(params)
+
+	// 返回为 map[string]string 类型
+	return map[string]string{
+		"params": string(paramsJSON),
+	}
+}
+
+func (p *CokingBituminousCoalProductionProcessor) ProcessResponse(data string) ([]models.FenWeiNetIndexInfo, error) {
+	responseData := fillFenWeiNetResponseData(data)
+
+	var indexInfoList []models.FenWeiNetIndexInfo
+	productName := "炼焦烟煤分煤种产量"
+	for _, data := range responseData {
+		var frequency string
+		if data.MonthlyValueChnName != "" && strings.Contains(data.MonthlyValueChnName, "月度") {
+			frequency = "月度"
+		} else if data.WeekValueChnName != "" && strings.Contains(data.WeekValueChnName, "周度") {
+			frequency = "周度"
+		} else {
+			frequency = data.MonthlyValueChnName
+		}
+
+		indexInfoList = append(indexInfoList, models.FenWeiNetIndexInfo{
+			IndexName:    productName + data.CoalTypeName + "/汾渭",
+			Unit:         data.MonthlyValueChnUnit,
+			Frequency:    frequency,
+			TerminalCode: utils.TerminalCode,
+			ClassifyName: productName,
+			DataTime:     data.DataDate,
+			Value:        data.MonthlyValue,
+		})
+	}
+
+	return indexInfoList, nil
+}
+
+// ThermalCoalInventorySocietyProcessor 动力煤库存-全社会
+type ThermalCoalInventorySocietyProcessor struct {
+	BaseProcessor
+}
+
+func (p *ThermalCoalInventorySocietyProcessor) GenerateRequestParams(currentTime string) map[string]string {
+	params := models.RequestParams{
+		Category: "79",
+		CheckedDims: map[string][]string{
+			"product_item_code": {"FW8004D-1"},
+		},
+		DateRange:    "20190820-" + currentTime,
+		ProductCode:  "FW8004D",
+		QuotaName:    "stock,avaliable_days,daily_consumption,monthly_value,inventory_index,week_value",
+		SplitTypeKey: "",
+		DataType:     nil,
+		IsSeason:     1,
+	}
+
+	// 将结构体转换为 JSON 字符串
+	paramsJSON, _ := json.Marshal(params)
+
+	// 返回为 map[string]string 类型
+	return map[string]string{
+		"params": string(paramsJSON),
+	}
+}
+
+func (p *ThermalCoalInventorySocietyProcessor) ProcessResponse(data string) ([]models.FenWeiNetIndexInfo, error) {
+	responseDataMap := fillFenWeiNetResponseMapData(data)
+
+	var indexInfoList []models.FenWeiNetIndexInfo
+	productName := "动力煤库存-全社会"
+
+	for productCategoryName, responseDataList := range responseDataMap {
+		if productCategoryName != productName {
+			continue
+		}
+
+		for _, data := range responseDataList {
+			var frequency string
+			if data.MonthlyValueChnName != "" && strings.Contains(data.MonthlyValueChnName, "月度") {
+				frequency = "月度"
+			} else if data.WeekValueChnName != "" && strings.Contains(data.WeekValueChnName, "周度") {
+				frequency = "周度"
+			} else {
+				frequency = data.MonthlyValueChnName
+			}
+
+			indexInfoList = append(indexInfoList, models.FenWeiNetIndexInfo{
+				IndexName:    productName + "/汾渭",
+				Unit:         data.MonthlyValueChnUnit,
+				Frequency:    frequency,
+				TerminalCode: utils.TerminalCode,
+				ClassifyName: productName,
+				DataTime:     data.DataDate,
+				Value:        data.MonthlyValue,
+			})
+		}
+	}
+
+	return indexInfoList, nil
+}
+
+// ThermalCoalInventoryProductionProcessor 动力煤库存-生产企业
+type ThermalCoalInventoryProductionProcessor struct {
+	BaseProcessor
+}
+
+func (p *ThermalCoalInventoryProductionProcessor) GenerateRequestParams(currentTime string) map[string]string {
+	params := models.RequestParams{
+		Category: "116",
+		CheckedDims: map[string][]string{
+			"product_item_code": {"FW8004D-2"},
+		},
+		DateRange:    "20190820-" + currentTime,
+		ProductCode:  "FW8004D",
+		QuotaName:    "stock,avaliable_days,daily_consumption,monthly_value,inventory_index,week_value",
+		SplitTypeKey: "",
+		DataType:     nil,
+		IsSeason:     1,
+	}
+
+	// 将结构体转换为 JSON 字符串
+	paramsJSON, _ := json.Marshal(params)
+
+	// 返回为 map[string]string 类型
+	return map[string]string{
+		"params": string(paramsJSON),
+	}
+}
+
+func (p *ThermalCoalInventoryProductionProcessor) ProcessResponse(data string) ([]models.FenWeiNetIndexInfo, error) {
+	responseDataMap := fillFenWeiNetResponseMapData(data)
+
+	var indexInfoList []models.FenWeiNetIndexInfo
+	productName := "动力煤库存-生产企业"
+
+	for productCategoryName, responseDataList := range responseDataMap {
+		if productCategoryName != productName {
+			continue
+		}
+
+		for _, data := range responseDataList {
+			var frequency string
+			if data.MonthlyValueChnName != "" && strings.Contains(data.MonthlyValueChnName, "月度") {
+				frequency = "月度"
+			} else if data.WeekValueChnName != "" && strings.Contains(data.WeekValueChnName, "周度") {
+				frequency = "周度"
+			} else {
+				frequency = data.MonthlyValueChnName
+			}
+
+			indexInfoList = append(indexInfoList, models.FenWeiNetIndexInfo{
+				IndexName:    productName + "/汾渭",
+				Unit:         data.WeekValueChnUnit,
+				Frequency:    frequency,
+				TerminalCode: utils.TerminalCode,
+				ClassifyName: productName,
+				DataTime:     data.DataDate,
+				Value:        data.MonthlyValue,
+			})
+		}
+	}
+
+	return indexInfoList, nil
+}
+
+// ThermalCoalInventorySixPowerPlantProcessor 动力煤库存-六大电厂
+type ThermalCoalInventorySixPowerPlantProcessor struct {
+	BaseProcessor
+}
+
+func (p *ThermalCoalInventorySixPowerPlantProcessor) GenerateRequestParams(currentTime string) map[string]string {
+	params := models.RequestParams{
+		Category: "118",
+		CheckedDims: map[string][]string{
+			"product_item_code": {"FW8004D-125500", "FW8004D-112534", "FW8004D-112548", "FW8004D-112549", "FW8004D-1129", "FW8004D-125", "FW8004D-125382", "FW8004D-125383"},
+		},
+		DateRange:    "20190820-" + currentTime,
+		ProductCode:  "FW8004D",
+		QuotaName:    "stock,avaliable_days,daily_consumption,monthly_value,inventory_index,week_value",
+		SplitTypeKey: "",
+		DataType:     nil,
+		IsSeason:     1,
+	}
+
+	// 将结构体转换为 JSON 字符串
+	paramsJSON, _ := json.Marshal(params)
+
+	// 返回为 map[string]string 类型
+	return map[string]string{
+		"params": string(paramsJSON),
+	}
+}
+
+func (p *ThermalCoalInventorySixPowerPlantProcessor) ProcessResponse(data string) ([]models.FenWeiNetIndexInfo, error) {
+	responseDataMap := fillFenWeiNetResponseMapData(data)
+
+	var indexInfoList []models.FenWeiNetIndexInfo
+	productName := "动力煤库存"
+
+	for productCategoryName, responseDataList := range responseDataMap {
+
+		for _, data := range responseDataList {
+			var frequency string
+			if data.MonthlyValueChnName != "" && strings.Contains(data.MonthlyValueChnName, "月度") {
+				frequency = "月度"
+			} else if data.WeekValueChnName != "" && strings.Contains(data.WeekValueChnName, "周度") {
+				frequency = "周度"
+			} else {
+				frequency = "日度"
+			}
+
+			classifyName := "动力煤库存-六大电厂"
+
+			// 库存
+			indexInfoList = append(indexInfoList, models.FenWeiNetIndexInfo{
+				IndexName:    productName + productCategoryName + data.StockChnName + "/汾渭",
+				Unit:         data.StockChnUnit,
+				Frequency:    frequency,
+				TerminalCode: utils.TerminalCode,
+				ClassifyName: classifyName,
+				DataTime:     data.DataDate,
+				Value:        data.Stock,
+			})
+			// 可用天数
+			indexInfoList = append(indexInfoList, models.FenWeiNetIndexInfo{
+				IndexName:    productName + productCategoryName + data.AvaliableDaysChnName + "/汾渭",
+				Unit:         data.AvaliableDaysChnUnit,
+				Frequency:    frequency,
+				TerminalCode: utils.TerminalCode,
+				ClassifyName: classifyName,
+				DataTime:     data.DataDate,
+				Value:        data.AvaliableDays,
+			})
+			// 日耗
+			indexInfoList = append(indexInfoList, models.FenWeiNetIndexInfo{
+				IndexName:    productName + productCategoryName + data.DailyConsumptionChnName + "/汾渭",
+				Unit:         data.DailyConsumptionChnUnit,
+				Frequency:    frequency,
+				TerminalCode: utils.TerminalCode,
+				ClassifyName: classifyName,
+				DataTime:     data.DataDate,
+				Value:        data.DailyConsumption,
+			})
+		}
+	}
+
+	return indexInfoList, nil
+}
+
+// CokingCoalInventorySocietyProcessor 炼焦煤库存-全社会
+type CokingCoalInventorySocietyProcessor struct {
+	BaseProcessor
+}
+
+func (p *CokingCoalInventorySocietyProcessor) GenerateRequestParams(currentTime string) map[string]string {
+	params := models.RequestParams{
+		Category: "79",
+		CheckedDims: map[string][]string{
+			"product_item_code": {"FW8005D-1"},
+		},
+		DateRange:    "20190820-" + currentTime,
+		ProductCode:  "FW8005D",
+		QuotaName:    "stock,monthly_value,inventory_index,week_value",
+		SplitTypeKey: "",
+		DataType:     nil,
+		IsSeason:     1,
+	}
+
+	// 将结构体转换为 JSON 字符串
+	paramsJSON, _ := json.Marshal(params)
+
+	// 返回为 map[string]string 类型
+	return map[string]string{
+		"params": string(paramsJSON),
+	}
+}
+
+func (p *CokingCoalInventorySocietyProcessor) ProcessResponse(data string) ([]models.FenWeiNetIndexInfo, error) {
+	responseDataMap := fillFenWeiNetResponseMapData(data)
+
+	var indexInfoList []models.FenWeiNetIndexInfo
+	productName := "炼焦煤库存-全社会"
+
+	for productCategoryName, responseDataList := range responseDataMap {
+		if productCategoryName == "合计" {
+			continue
+		}
+		for _, data := range responseDataList {
+			var frequency string
+			if data.MonthlyValueChnName != "" && strings.Contains(data.MonthlyValueChnName, "月度") {
+				frequency = "月度"
+			} else if data.WeekValueChnName != "" && strings.Contains(data.WeekValueChnName, "周度") {
+				frequency = "周度"
+			} else {
+				frequency = data.MonthlyValueChnName
+			}
+
+			indexInfoList = append(indexInfoList, models.FenWeiNetIndexInfo{
+				IndexName:    productCategoryName + "/汾渭",
+				Unit:         data.MonthlyValueChnUnit,
+				Frequency:    frequency,
+				TerminalCode: utils.TerminalCode,
+				ClassifyName: productName,
+				DataTime:     data.DataDate,
+				Value:        data.MonthlyValue,
+			})
+		}
+	}
+
+	return indexInfoList, nil
+}
+
+// CokingCoalInventoryProductionProcessor 炼焦煤库存-生产企业
+type CokingCoalInventoryProductionProcessor struct {
+	BaseProcessor
+}
+
+func (p *CokingCoalInventoryProductionProcessor) GenerateRequestParams(currentTime string) map[string]string {
+	params := models.RequestParams{
+		Category: "116",
+		CheckedDims: map[string][]string{
+			"product_item_code": {"FW8005D-2"},
+		},
+		DateRange:    "20190820-" + currentTime,
+		ProductCode:  "FW8005D",
+		QuotaName:    "stock,monthly_value,inventory_index,week_value",
+		SplitTypeKey: "",
+		DataType:     nil,
+		IsSeason:     1,
+	}
+
+	// 将结构体转换为 JSON 字符串
+	paramsJSON, _ := json.Marshal(params)
+
+	// 返回为 map[string]string 类型
+	return map[string]string{
+		"params": string(paramsJSON),
+	}
+}
+
+func (p *CokingCoalInventoryProductionProcessor) ProcessResponse(data string) ([]models.FenWeiNetIndexInfo, error) {
+	responseDataMap := fillFenWeiNetResponseMapData(data)
+
+	var indexInfoList []models.FenWeiNetIndexInfo
+	productName := "炼焦煤库存-生产企业"
+
+	for productCategoryName, responseDataList := range responseDataMap {
+		if productCategoryName == "合计" {
+			continue
+		}
+		for _, data := range responseDataList {
+			var frequency string
+			if data.MonthlyValueChnName != "" && strings.Contains(data.MonthlyValueChnName, "月度") {
+				frequency = "月度"
+			} else if data.WeekValueChnName != "" && strings.Contains(data.WeekValueChnName, "周度") {
+				frequency = "周度"
+			} else {
+				frequency = data.MonthlyValueChnName
+			}
+
+			indexInfoList = append(indexInfoList, models.FenWeiNetIndexInfo{
+				IndexName:    productCategoryName + "/汾渭",
+				Unit:         data.WeekValueChnUnit,
+				Frequency:    frequency,
+				TerminalCode: utils.TerminalCode,
+				ClassifyName: productName,
+				DataTime:     data.DataDate,
+				Value:        data.WeekValue,
+			})
+		}
+	}
+
+	return indexInfoList, nil
+}
+
+// CokingCoalInventoryDownstreamProcessor 炼焦煤库存-下游企业
+type CokingCoalInventoryDownstreamProcessor struct {
+	BaseProcessor
+}
+
+func (p *CokingCoalInventoryDownstreamProcessor) GenerateRequestParams(currentTime string) map[string]string {
+	params := models.RequestParams{
+		Category: "117",
+		CheckedDims: map[string][]string{
+			"product_item_code": {"FW8005D-3", "FW8005D-5"},
+		},
+		DateRange:    "20190820-" + currentTime,
+		ProductCode:  "FW8005D",
+		QuotaName:    "stock,monthly_value,inventory_index,week_value",
+		SplitTypeKey: "",
+		DataType:     nil,
+		IsSeason:     1,
+	}
+
+	// 将结构体转换为 JSON 字符串
+	paramsJSON, _ := json.Marshal(params)
+
+	// 返回为 map[string]string 类型
+	return map[string]string{
+		"params": string(paramsJSON),
+	}
+}
+
+func (p *CokingCoalInventoryDownstreamProcessor) ProcessResponse(data string) ([]models.FenWeiNetIndexInfo, error) {
+	responseDataMap := fillFenWeiNetResponseMapData(data)
+
+	var indexInfoList []models.FenWeiNetIndexInfo
+	productName := "炼焦煤库存-下游企业"
+
+	for productCategoryName, responseDataList := range responseDataMap {
+		if productCategoryName == "合计" {
+			continue
+		}
+		for _, data := range responseDataList {
+			frequency := "周度"
+
+			indexInfoList = append(indexInfoList, models.FenWeiNetIndexInfo{
+				IndexName:    productCategoryName + "/汾渭",
+				Unit:         data.StockChnUnit,
+				Frequency:    frequency,
+				TerminalCode: utils.TerminalCode,
+				ClassifyName: productName,
+				DataTime:     data.DataDate,
+				Value:        data.Stock,
+			})
+		}
+	}
+
+	return indexInfoList, nil
+}
+
+// NationalCoalMineInventoryProcessor 全国煤矿库存
+type NationalCoalMineInventoryProcessor struct {
+	BaseProcessor
+}
+
+func (p *NationalCoalMineInventoryProcessor) GenerateRequestParams(currentTime string) map[string]string {
+	params := models.RequestParams{
+		Category:     nil,
+		CheckedDims:  map[string][]string{},
+		DateRange:    "20190820-" + currentTime,
+		ProductCode:  "FW8001D",
+		QuotaName:    "monthly_value",
+		SplitTypeKey: "",
+		DataType:     nil,
+		IsSeason:     1,
+	}
+
+	// 将结构体转换为 JSON 字符串
+	paramsJSON, _ := json.Marshal(params)
+
+	// 返回为 map[string]string 类型
+	return map[string]string{
+		"params": string(paramsJSON),
+	}
+}
+
+func (p *NationalCoalMineInventoryProcessor) ProcessResponse(data string) ([]models.FenWeiNetIndexInfo, error) {
+	responseData := fillFenWeiNetResponseData(data)
+
+	var indexInfoList []models.FenWeiNetIndexInfo
+	productName := "全国煤矿库存"
+
+	for _, data := range responseData {
+		var frequency string
+		if data.MonthlyValueChnName != "" && strings.Contains(data.MonthlyValueChnName, "月度") {
+			frequency = "月度"
+		} else if data.WeekValueChnName != "" && strings.Contains(data.WeekValueChnName, "周度") {
+			frequency = "周度"
+		} else {
+			frequency = "日度"
+		}
+
+		indexInfoList = append(indexInfoList, models.FenWeiNetIndexInfo{
+			IndexName:    productName + "/汾渭",
+			Unit:         data.MonthlyValueChnUnit,
+			Frequency:    frequency,
+			TerminalCode: utils.TerminalCode,
+			ClassifyName: productName,
+			DataTime:     data.DataDate,
+			Value:        data.MonthlyValue,
+		})
+	}
+
+	return indexInfoList, nil
+}
+
+// StateOwnedKeyCoalMineInventoryProcessor 国有重点煤矿库存
+type StateOwnedKeyCoalMineInventoryProcessor struct {
+	BaseProcessor
+}
+
+func (p *StateOwnedKeyCoalMineInventoryProcessor) GenerateRequestParams(currentTime string) map[string]string {
+	params := models.RequestParams{
+		Category: nil,
+		CheckedDims: map[string][]string{
+			"province": {"20", "16", "21", "6", "19", "23", "8", "1", "7", "17", "9", "15", "11", "22", "24", "4", "25", "12", "14", "13", "2", "27", "28"},
+		},
+		DateRange:    "20190820-" + currentTime,
+		ProductCode:  "FW8003D",
+		QuotaName:    "monthly_value",
+		SplitTypeKey: "province",
+		IsTotal:      "",
+		DataType:     nil,
+		IsSeason:     1,
+	}
+
+	// 将结构体转换为 JSON 字符串
+	paramsJSON, _ := json.Marshal(params)
+
+	// 返回为 map[string]string 类型
+	return map[string]string{
+		"params": string(paramsJSON),
+	}
+}
+
+func (p *StateOwnedKeyCoalMineInventoryProcessor) ProcessResponse(data string) ([]models.FenWeiNetIndexInfo, error) {
+	responseData := fillFenWeiNetResponseData(data)
+
+	var indexInfoList []models.FenWeiNetIndexInfo
+	productName := "国有重点煤矿库存"
+
+	for _, data := range responseData {
+		var frequency string
+		if data.MonthlyValueChnName != "" && strings.Contains(data.MonthlyValueChnName, "月度") {
+			frequency = "月度"
+		} else if data.WeekValueChnName != "" && strings.Contains(data.WeekValueChnName, "周度") {
+			frequency = "周度"
+		} else {
+			frequency = "日度"
+		}
+
+		indexInfoList = append(indexInfoList, models.FenWeiNetIndexInfo{
+			IndexName:    productName + data.ProvinceName + "/汾渭",
+			Unit:         data.MonthlyValueChnUnit,
+			Frequency:    frequency,
+			TerminalCode: utils.TerminalCode,
+			ClassifyName: productName,
+			DataTime:     data.DataDate,
+			Value:        data.MonthlyValue,
+		})
+	}
+
+	return indexInfoList, nil
+}
+
+// CokeInventoryProcessor 焦炭库存
+type CokeInventoryProcessor struct {
+	BaseProcessor
+}
+
+func (p *CokeInventoryProcessor) GenerateRequestParams(currentTime string) map[string]string {
+	params := models.RequestParams{
+		Category: nil,
+		CheckedDims: map[string][]string{
+			"product_item_code": {"FW1405D-1", "FW1405D-2", "FW1405D-3", "FW1405D-4", "FW1405D-5", "FW1405D-6", "FW1405D-7"},
+		},
+		DateRange:    "20190820-" + currentTime,
+		ProductCode:  "FW1405D",
+		QuotaName:    "stock,inventory_index",
+		SplitTypeKey: "",
+		DataType:     nil,
+		IsSeason:     1,
+	}
+
+	// 将结构体转换为 JSON 字符串
+	paramsJSON, _ := json.Marshal(params)
+
+	// 返回为 map[string]string 类型
+	return map[string]string{
+		"params": string(paramsJSON),
+	}
+}
+
+func (p *CokeInventoryProcessor) ProcessResponse(data string) ([]models.FenWeiNetIndexInfo, error) {
+	responseDataMap := fillFenWeiNetResponseMapData(data)
+
+	var indexInfoList []models.FenWeiNetIndexInfo
+	productName := "焦炭库存"
+	for productCategoryName, responseDataList := range responseDataMap {
+		utils.FileLog.Info(fmt.Sprintf("productName: %s, responseDataList:size: %v", productName, len(responseDataList)))
+		for _, data := range responseDataList {
+			var frequency string
+			productCodeMap := map[string]struct{}{
+				"FW1405D-1": {},
+				"FW1405D-2": {},
+				"FW1405D-3": {},
+				"FW1405D-6": {},
+				"FW1405D-7": {},
+			}
+			if _, ok := productCodeMap[data.ProductItemCode]; ok {
+				frequency = "周度"
+			} else {
+				frequency = "日度"
+			}
+
+			var unit string
+			if data.InventoryIndexChnUnit != "" {
+				unit = data.InventoryIndexChnUnit
+			} else {
+				unit = data.StockChnUnit
+			}
+
+			var value float64
+			if data.InventoryIndex != 0 {
+				value = data.InventoryIndex
+			} else {
+				value = data.Stock
+			}
+
+			indexInfoList = append(indexInfoList, models.FenWeiNetIndexInfo{
+				IndexName:    productName + productCategoryName + "/汾渭",
+				Unit:         unit,
+				Frequency:    frequency,
+				TerminalCode: utils.TerminalCode,
+				ClassifyName: productName,
+				DataTime:     data.DataDate,
+				Value:        value,
+			})
+		}
+	}
+
+	return indexInfoList, nil
+}
+
+// PortDataInventoryNorthernPortProcessor 港口数据-库存-北方港口
+type PortDataInventoryNorthernPortProcessor struct {
+	BaseProcessor
+}
+
+func (p *PortDataInventoryNorthernPortProcessor) GenerateRequestParams(currentTime string) map[string]string {
+	params := models.RequestParams{
+		Category: "79",
+		CheckedDims: map[string][]string{
+			"code": {"2332", "2333", "2335", "2334", "2337", "2339", "2340", "2341", "2342"},
+		},
+		DateRange:    "20190820-" + currentTime,
+		ProductCode:  "FW1203D",
+		QuotaName:    "stock",
+		SplitTypeKey: "product_item_code",
+		IsTotal:      0,
+		DataType:     nil,
+		Type:         2,
+		IsSeason:     1,
+	}
+
+	// 将结构体转换为 JSON 字符串
+	paramsJSON, _ := json.Marshal(params)
+
+	// 返回为 map[string]string 类型
+	return map[string]string{
+		"params": string(paramsJSON),
+	}
+}
+
+func (p *PortDataInventoryNorthernPortProcessor) ProcessResponse(data string) ([]models.FenWeiNetIndexInfo, error) {
+	responseData := fillFenWeiNetResponseData(data)
+
+	var indexInfoList []models.FenWeiNetIndexInfo
+	productName := "港口数据-库存-北方港口"
+
+	for _, data := range responseData {
+		var frequency string
+		if data.MonthlyValueChnName != "" && strings.Contains(data.MonthlyValueChnName, "月度") {
+			frequency = "月度"
+		} else if data.WeekValueChnName != "" && strings.Contains(data.WeekValueChnName, "周度") {
+			frequency = "周度"
+		} else {
+			frequency = "日度"
+		}
+
+		indexInfoList = append(indexInfoList, models.FenWeiNetIndexInfo{
+			IndexName:    productName + data.ProductItemName + "/汾渭",
+			Unit:         data.StockChnUnit,
+			Frequency:    frequency,
+			TerminalCode: utils.TerminalCode,
+			ClassifyName: productName,
+			DataTime:     data.DataDate,
+			Value:        data.Stock,
+		})
+	}
+
+	return indexInfoList, nil
+}
+
+// PortDataInventoryInlandPortProcessor 港口数据-库存-江内港口
+type PortDataInventoryInlandPortProcessor struct {
+	BaseProcessor
+}
+
+func (p *PortDataInventoryInlandPortProcessor) GenerateRequestParams(currentTime string) map[string]string {
+	params := models.RequestParams{
+		Category: "79",
+		CheckedDims: map[string][]string{
+			"code": {"2321", "2320", "2324", "2323", "2322", "2325"},
+		},
+		DateRange:    "20190820-" + currentTime,
+		ProductCode:  "FW1203D",
+		QuotaName:    "stock",
+		SplitTypeKey: "product_item_code",
+		IsTotal:      0,
+		DataType:     nil,
+		Type:         2,
+		IsSeason:     1,
+	}
+
+	// 将结构体转换为 JSON 字符串
+	paramsJSON, _ := json.Marshal(params)
+
+	// 返回为 map[string]string 类型
+	return map[string]string{
+		"params": string(paramsJSON),
+	}
+}
+
+func (p *PortDataInventoryInlandPortProcessor) ProcessResponse(data string) ([]models.FenWeiNetIndexInfo, error) {
+	responseData := fillFenWeiNetResponseData(data)
+
+	var indexInfoList []models.FenWeiNetIndexInfo
+	productName := "港口数据-库存-江内港口"
+
+	for _, data := range responseData {
+		var frequency = "周度"
+
+		indexInfoList = append(indexInfoList, models.FenWeiNetIndexInfo{
+			IndexName:    productName + data.ProductItemName + "/汾渭",
+			Unit:         data.StockChnUnit,
+			Frequency:    frequency,
+			TerminalCode: utils.TerminalCode,
+			ClassifyName: productName,
+			DataTime:     data.DataDate,
+			Value:        data.Stock,
+		})
+	}
+
+	return indexInfoList, nil
+}
+
+// PortDataDispatchNorthernPortProcessor 港口数据-调度-北方港口
+type PortDataDispatchNorthernPortProcessor struct {
+	BaseProcessor
+}
+
+func (p *PortDataDispatchNorthernPortProcessor) GenerateRequestParams(currentTime string) map[string]string {
+	params := models.RequestParams{
+		Category: "120",
+		CheckedDims: map[string][]string{
+			"code": {"2364", "2365", "2366", "2363", "2367", "2368"},
+			"port": {"2", "4", "7", "9", "11", "326", "327", "329", "330"},
+		},
+		DateRange:    "20190820-" + currentTime,
+		ProductCode:  "FW1203D",
+		QuotaName:    "vehicle,ship,tons",
+		SplitTypeKey: "product_item_code,port",
+		IsTotal:      1,
+		DataType:     nil,
+		Type:         2,
+		IsSeason:     1,
+	}
+
+	// 将结构体转换为 JSON 字符串
+	paramsJSON, _ := json.Marshal(params)
+
+	// 返回为 map[string]string 类型
+	return map[string]string{
+		"params": string(paramsJSON),
+	}
+}
+
+func (p *PortDataDispatchNorthernPortProcessor) ProcessResponse(data string) ([]models.FenWeiNetIndexInfo, error) {
+	responseData := fillFenWeiNetResponseData(data)
+
+	var indexInfoList []models.FenWeiNetIndexInfo
+	productName := "港口数据-调度-北方港口"
+
+	for _, data := range responseData {
+		var frequency = "日度"
+		var unit string
+		if data.VehicleChnUnit != "" {
+			unit = data.VehicleChnUnit
+		} else if data.TonsChnUnit != "" {
+			unit = data.TonsChnUnit
+		} else {
+			unit = data.ShipChnUnit
+		}
+
+		indexInfoList = append(indexInfoList, models.FenWeiNetIndexInfo{
+			IndexName:    productName + "-" + data.PortName + data.ProductItemName + "/汾渭",
+			Unit:         unit,
+			Frequency:    frequency,
+			TerminalCode: utils.TerminalCode,
+			ClassifyName: productName,
+			DataTime:     data.DataDate,
+			Value:        data.Tons,
+		})
+	}
+
+	return indexInfoList, nil
+}
+
+// PortDataThroughputProcessor 港口数据-运量
+type PortDataThroughputProcessor struct {
+	BaseProcessor
+}
+
+func (p *PortDataThroughputProcessor) GenerateRequestParams(currentTime string) map[string]string {
+	params := models.RequestParams{
+		Category: "30",
+		CheckedDims: map[string][]string{
+			"code":     {"2362"},
+			"port_son": {"44", "43", "42", "41", "40", "39"},
+		},
+		DateRange:    "20190820-" + currentTime,
+		ProductCode:  "FW1203D",
+		QuotaName:    "transport_volume",
+		SplitTypeKey: "port_son",
+		IsTotal:      0,
+		DataType:     nil,
+		Type:         2,
+		IsSeason:     1,
+	}
+
+	// 将结构体转换为 JSON 字符串
+	paramsJSON, _ := json.Marshal(params)
+
+	// 返回为 map[string]string 类型
+	return map[string]string{
+		"params": string(paramsJSON),
+	}
+}
+
+func (p *PortDataThroughputProcessor) ProcessResponse(data string) ([]models.FenWeiNetIndexInfo, error) {
+	responseData := fillFenWeiNetResponseData(data)
+
+	var indexInfoList []models.FenWeiNetIndexInfo
+	productName := "港口数据-运量"
+
+	for _, data := range responseData {
+		var frequency = "日度"
+
+		indexInfoList = append(indexInfoList, models.FenWeiNetIndexInfo{
+			IndexName:    productName + data.PortSonName + "/汾渭",
+			Unit:         data.TransportVolumeChnUnit,
+			Frequency:    frequency,
+			TerminalCode: utils.TerminalCode,
+			ClassifyName: productName,
+			DataTime:     data.DataDate,
+			Value:        data.TransportVolume,
+		})
+	}
+
+	return indexInfoList, nil
+}
+
+// DaqinLineDailyThroughputProcessor 大秦线日运量
+type DaqinLineDailyThroughputProcessor struct {
+	BaseProcessor
+}
+
+func (p *DaqinLineDailyThroughputProcessor) GenerateRequestParams(currentTime string) map[string]string {
+	params := models.RequestParams{
+		Category:     nil,
+		CheckedDims:  map[string][]string{},
+		DateRange:    "20190820-" + currentTime,
+		ProductCode:  "FW1107D",
+		QuotaName:    "stock",
+		SplitTypeKey: "",
+		DataType:     nil,
+		IsSeason:     1,
+	}
+
+	// 将结构体转换为 JSON 字符串
+	paramsJSON, _ := json.Marshal(params)
+
+	// 返回为 map[string]string 类型
+	return map[string]string{
+		"params": string(paramsJSON),
+	}
+}
+
+func (p *DaqinLineDailyThroughputProcessor) ProcessResponse(data string) ([]models.FenWeiNetIndexInfo, error) {
+	responseData := fillFenWeiNetResponseData(data)
+
+	var indexInfoList []models.FenWeiNetIndexInfo
+	productName := "大秦线日运量"
+
+	for _, data := range responseData {
+		var frequency = "日度"
+
+		indexInfoList = append(indexInfoList, models.FenWeiNetIndexInfo{
+			IndexName:    productName + data.StockChnName + "/汾渭",
+			Unit:         data.StockChnUnit,
+			Frequency:    frequency,
+			TerminalCode: utils.TerminalCode,
+			ClassifyName: productName,
+			DataTime:     data.DataDate,
+			Value:        data.Stock,
+		})
+	}
+
+	return indexInfoList, nil
+}
+
+// ThermalCoalPortPriceProcessor 动力煤港口价格
+type ThermalCoalPortPriceProcessor struct {
+	BaseProcessor
+}
+
+func (p *ThermalCoalPortPriceProcessor) GenerateRequestParams(currentTime string) map[string]string {
+	params := models.RequestParams{
+		Category: nil,
+		CheckedDims: map[string][]string{
+			"product_item_code": {"FW2001P-1001", "FW2001P-1002", "FW2001P-1004", "FW2001P-1005", "FW2001P-1003", "FW2001P-1171"},
+		},
+		DateRange:    "20190820-" + currentTime,
+		ProductCode:  "FW2310R",
+		QuotaName:    "price_rmb,price_index_mom",
+		SplitTypeKey: "",
+		DataType:     nil,
+		IsSeason:     1,
+	}
+
+	// 将结构体转换为 JSON 字符串
+	paramsJSON, _ := json.Marshal(params)
+
+	// 返回为 map[string]string 类型
+	return map[string]string{
+		"params": string(paramsJSON),
+	}
+}
+
+func (p *ThermalCoalPortPriceProcessor) ProcessResponse(data string) ([]models.FenWeiNetIndexInfo, error) {
+	responseDataMap := fillFenWeiNetResponseAStratumMapData(data)
+
+	var indexInfoList []models.FenWeiNetIndexInfo
+	productName := "动力煤港口价格"
+	for productCategoryName, responseDataList := range responseDataMap {
+		utils.FileLog.Info(fmt.Sprintf("productName: %s, responseDataList:size: %v", productName, len(responseDataList)))
+		for _, data := range responseDataList {
+			var frequency = "日度"
+
+			indexInfoList = append(indexInfoList, models.FenWeiNetIndexInfo{
+				IndexName:    productName + productCategoryName + "-" + data.PriceRmbChnName + "/汾渭",
+				Unit:         data.PriceRmbChnUnit,
+				Frequency:    frequency,
+				TerminalCode: utils.TerminalCode,
+				ClassifyName: productName,
+				DataTime:     data.DataDate,
+				Value:        data.PriceRmb,
+			})
+			indexInfoList = append(indexInfoList, models.FenWeiNetIndexInfo{
+				IndexName:    productName + productCategoryName + "-" + data.PriceIndexMomChnName + "/汾渭",
+				Unit:         data.PriceIndexMomChnUnit,
+				Frequency:    frequency,
+				TerminalCode: utils.TerminalCode,
+				ClassifyName: productName,
+				DataTime:     data.DataDate,
+				Value:        data.PriceIndexMom,
+			})
+		}
+	}
+
+	return indexInfoList, nil
+}
+
+// ThermalCoalConsumptionProcessor 动力煤消费量
+type ThermalCoalConsumptionProcessor struct {
+	BaseProcessor
+}
+
+func (p *ThermalCoalConsumptionProcessor) GenerateRequestParams(currentTime string) map[string]string {
+	params := models.RequestParams{
+		Category: nil,
+		CheckedDims: map[string][]string{
+			"product_item_code": {"FW5002D-1", "FW5002D-2", "FW5002D-3", "FW5002D-4", "FW5002D-5", "FW5002D-6", "FW5002D-7"},
+		},
+		DateRange:    "20190820-" + currentTime,
+		ProductCode:  "FW5002D",
+		QuotaName:    "monthly_value,monthly_accumulation",
+		SplitTypeKey: "",
+		DataType:     nil,
+		IsSeason:     1,
+	}
+
+	// 将结构体转换为 JSON 字符串
+	paramsJSON, _ := json.Marshal(params)
+
+	// 返回为 map[string]string 类型
+	return map[string]string{
+		"params": string(paramsJSON),
+	}
+}
+
+func (p *ThermalCoalConsumptionProcessor) ProcessResponse(data string) ([]models.FenWeiNetIndexInfo, error) {
+	responseDataMap := fillFenWeiNetResponseMapData(data)
+
+	var indexInfoList []models.FenWeiNetIndexInfo
+	productName := "动力煤消费量"
+	for productCategoryName, responseDataList := range responseDataMap {
+		utils.FileLog.Info(fmt.Sprintf("productName: %s, responseDataList:size: %v", productName, len(responseDataList)))
+		for _, data := range responseDataList {
+			var frequency string
+			if data.MonthlyValueChnName != "" && strings.Contains(data.MonthlyValueChnName, "月度") {
+				frequency = "月度"
+			} else if data.WeekValueChnName != "" && strings.Contains(data.WeekValueChnName, "周度") {
+				frequency = "周度"
+			} else {
+				frequency = "日度"
+			}
+
+			indexInfoList = append(indexInfoList, models.FenWeiNetIndexInfo{
+				IndexName:    productName + productCategoryName + "/汾渭",
+				Unit:         data.MonthlyValueChnUnit,
+				Frequency:    frequency,
+				TerminalCode: utils.TerminalCode,
+				ClassifyName: productName,
+				DataTime:     data.DataDate,
+				Value:        data.MonthlyValue,
+			})
+		}
+	}
+
+	return indexInfoList, nil
+}
+
+// CokingCleanCoalConsumptionProcessor 炼焦精煤消费量
+type CokingCleanCoalConsumptionProcessor struct {
+	BaseProcessor
+}
+
+func (p *CokingCleanCoalConsumptionProcessor) GenerateRequestParams(currentTime string) map[string]string {
+	params := models.RequestParams{
+		Category:     nil,
+		CheckedDims:  map[string][]string{},
+		DateRange:    "20190820-" + currentTime,
+		ProductCode:  "FW5001D",
+		QuotaName:    "monthly_value,monthly_accumulation",
+		SplitTypeKey: "",
+		DataType:     nil,
+		IsSeason:     1,
+	}
+
+	// 将结构体转换为 JSON 字符串
+	paramsJSON, _ := json.Marshal(params)
+
+	// 返回为 map[string]string 类型
+	return map[string]string{
+		"params": string(paramsJSON),
+	}
+}
+
+func (p *CokingCleanCoalConsumptionProcessor) ProcessResponse(data string) ([]models.FenWeiNetIndexInfo, error) {
+	responseData := fillFenWeiNetResponseData(data)
+
+	var indexInfoList []models.FenWeiNetIndexInfo
+	productName := "炼焦精煤消费量"
+
+	for _, data := range responseData {
+		var frequency string
+		if data.MonthlyValueChnName != "" && strings.Contains(data.MonthlyValueChnName, "月度") {
+			frequency = "月度"
+		} else if data.WeekValueChnName != "" && strings.Contains(data.WeekValueChnName, "周度") {
+			frequency = "周度"
+		} else {
+			frequency = "日度"
+		}
+
+		indexInfoList = append(indexInfoList, models.FenWeiNetIndexInfo{
+			IndexName:    productName + "/汾渭",
+			Unit:         data.MonthlyValueChnUnit,
+			Frequency:    frequency,
+			TerminalCode: utils.TerminalCode,
+			ClassifyName: productName,
+			DataTime:     data.DataDate,
+			Value:        data.MonthlyValue,
+		})
+	}
+
+	return indexInfoList, nil
+}
+
+func fillFenWeiNetResponseData(data string) []models.FenWeiNetResponse {
+	var result models.FenWeiNetResponseWrapper
+	err := json.Unmarshal([]byte(data), &result)
+	if err != nil {
+		return nil
+	}
+	responseData := result.Data.Data
+	return responseData
+}
+
+func fillFenWeiNetResponseMapData(data string) map[string][]models.FenWeiNetResponse {
+	var result models.FenWeiNetResponseMapWrapper
+	err := json.Unmarshal([]byte(data), &result)
+	if err != nil {
+		return nil
+	}
+	responseData := result.Data.Data
+	return responseData
+}
+
+func fillFenWeiNetResponseAStratumMapData(data string) map[string][]models.FenWeiNetResponse {
+	var result models.FenWeiNetResponseAStratumMapWrapper
+	err := json.Unmarshal([]byte(data), &result)
+	if err != nil {
+		return nil
+	}
+	responseData := result.Data
+	return responseData
+}

services/fenwei/processor_factory.go

@@ -0,0 +1,106 @@
+// Package fenwei
+// @Author gmy 2024/8/20 14:50:00
+package fenwei
+
+const (
+	fenWeiUrl   = "https://www.sxcoal.com/api/coalresource-adhoc/queryV1/data"
+	fenWeiByUrl = "https://www.sxcoal.com/api/coalresource-adhoc/queryV1/byData"
+)
+
+type ProcessorFactory struct{}
+
+func (f *ProcessorFactory) CreateProcessor(module string) DataProcessor {
+	switch module {
+	case "动力煤供应量":
+		return &ThermalCoalSupplyProcessor{
+			BaseProcessor{URL: fenWeiUrl},
+		}
+	case "炼焦精煤供应量":
+		return &CokingCleanCoalSupplyProcessor{
+			BaseProcessor{URL: fenWeiUrl},
+		}
+	case "原煤分省分煤种产量":
+		return &RawCoalProvinceProductionProcessor{
+			BaseProcessor{URL: fenWeiUrl},
+		}
+	case "国有重点煤矿原煤产量":
+		return &StateOwnedKeyCoalMineRawCoalProductionProcessor{
+			BaseProcessor{URL: fenWeiUrl},
+		}
+	case "炼焦烟煤分煤种产量":
+		return &CokingBituminousCoalProductionProcessor{
+			BaseProcessor{URL: fenWeiUrl},
+		}
+	case "动力煤库存-全社会":
+		return &ThermalCoalInventorySocietyProcessor{
+			BaseProcessor{URL: fenWeiUrl},
+		}
+	case "动力煤库存-生产企业":
+		return &ThermalCoalInventoryProductionProcessor{
+			BaseProcessor{URL: fenWeiUrl},
+		}
+	case "动力煤库存-六大电厂":
+		return &ThermalCoalInventorySixPowerPlantProcessor{
+			BaseProcessor{URL: fenWeiUrl},
+		}
+	case "炼焦煤库存-全社会":
+		return &CokingCoalInventorySocietyProcessor{
+			BaseProcessor{URL: fenWeiUrl},
+		}
+	case "炼焦煤库存-生产企业":
+		return &CokingCoalInventoryProductionProcessor{
+			BaseProcessor{URL: fenWeiUrl},
+		}
+	case "炼焦煤库存-下游企业":
+		return &CokingCoalInventoryDownstreamProcessor{
+			BaseProcessor{URL: fenWeiUrl},
+		}
+	case "全国煤矿库存":
+		return &NationalCoalMineInventoryProcessor{
+			BaseProcessor{URL: fenWeiUrl},
+		}
+	case "国有重点煤矿库存":
+		return &StateOwnedKeyCoalMineInventoryProcessor{
+			BaseProcessor{URL: fenWeiUrl},
+		}
+	case "焦炭库存":
+		return &CokeInventoryProcessor{
+			BaseProcessor{URL: fenWeiUrl},
+		}
+	case "港口数据-库存-北方港口":
+		return &PortDataInventoryNorthernPortProcessor{
+			BaseProcessor{URL: fenWeiByUrl},
+		}
+	case "港口数据-库存-江内港口":
+		return &PortDataInventoryInlandPortProcessor{
+			BaseProcessor{URL: fenWeiByUrl},
+		}
+	case "港口数据-调度-北方港口":
+		return &PortDataDispatchNorthernPortProcessor{
+			BaseProcessor{URL: fenWeiByUrl},
+		}
+	case "港口数据-运量":
+		return &PortDataThroughputProcessor{
+			BaseProcessor{URL: fenWeiByUrl},
+		}
+	case "大秦线日运量":
+		return &DaqinLineDailyThroughputProcessor{
+			BaseProcessor{URL: fenWeiUrl},
+		}
+	case "动力煤港口价格":
+		return &ThermalCoalPortPriceProcessor{
+			BaseProcessor{URL: fenWeiUrl},
+		}
+	case "动力煤消费量":
+		return &ThermalCoalConsumptionProcessor{
+			BaseProcessor{URL: fenWeiUrl},
+		}
+	case "炼焦精煤消费量":
+		return &CokingCleanCoalConsumptionProcessor{
+			BaseProcessor{URL: fenWeiUrl},
+		}
+
+	default:
+		return nil
+	}
+}

services/task.go

@@ -3,6 +3,7 @@ package services
 import (
 	ccfService "eta/eta_data_analysis/services/base_from_ccf"
 	oilchemService "eta/eta_data_analysis/services/base_from_oilchem"
+	"eta/eta_data_analysis/services/fenwei"
 	"eta/eta_data_analysis/utils"
 	"fmt"
 	"github.com/beego/beego/v2/task"
@@ -48,6 +49,13 @@ func Task() {
 		task.AddTask("汾渭数据指标文件检测", fenWeiReadWatchIndexFile)
 	}
 
+	// 汾渭网络数据
+	if utils.FenweiNetOpen == "1" {
+		// 汾渭网络数据处理 每天16点和19点 执行爬取
+		fenWeiNetDataDeal := task.NewTask("fenWeiNetDataDeal", "0 0 16,19 * * *", fenwei.FenWeiNetDataDeal)
+		task.AddTask("汾渭网络数据处理", fenWeiNetDataDeal)
+	}
+
 	if utils.MtjhOpen == "1" {
 		c := cron.New(cron.WithSeconds())
 		//每2分钟检测一次指标文件是否更新

static/fen_wei_net_data_json.json

@@ -0,0 +1,26 @@
+{
+  "data": [
+    "动力煤供应量",
+    "炼焦精煤供应量",
+    "原煤分省分煤种产量",
+    "国有重点煤矿原煤产量",
+    "炼焦烟煤分煤种产量",
+    "动力煤库存-全社会",
+    "动力煤库存-生产企业",
+    "动力煤库存-六大电厂",
+    "炼焦煤库存-全社会",
+    "炼焦煤库存-生产企业",
+    "炼焦煤库存-下游企业",
+    "全国煤矿库存",
+    "国有重点煤矿库存",
+    "焦炭库存",
+    "港口数据-库存-北方港口",
+    "港口数据-库存-江内港口",
+    "港口数据-调度-北方港口",
+    "港口数据-运量",
+    "大秦线日运量",
+    "动力煤港口价格",
+    "动力煤消费量",
+    "炼焦精煤消费量"
+  ]
+}

utils/config.go

@@ -59,6 +59,11 @@ var (
 	FenweiFileDir    string // excel文件目录
 	FenweiOldFileDir string // 已读取过的excel文件目录
 	FenweiOpen       string // 是否配置汾渭数据源
+
+	FenweiNetOpen     string // 是否配置汾渭网页数据源
+	FenweiNetUseName  string // 汾渭登录账号
+	FenweiNetPassword string // 汾渭登录密码
+	FenweiNetJsonPath string // 汾渭json文件地址
 )
 
 // 煤炭江湖
@@ -152,6 +157,11 @@ func init() {
 		FenweiOpen = config["fenwei_open"]
 		FenweiFileDir = config["fenwei_file_dir"]
 		FenweiOldFileDir = config["fenwei_old_file_dir"]
+
+		FenweiNetOpen = config["fenwei_net_open"]
+		FenweiNetUseName = config["fenwei_net_username"]
+		FenweiNetPassword = config["fenwei_net_password"]
+		FenweiNetJsonPath = config["fenwei_net_json_path"]
 	}
 
 	//煤炭江湖文件夹配置

utils/constants.go

@@ -248,4 +248,5 @@ const (
 	LIB_ROUTE_CCF_EDB_HANDLE            = "ccf/handle/edb_data"        // CCF化纤信息指标入库接口地址
 	LIB_ROUTE_CCF_TABLE_HANDLE          = "ccf/handle/table_data"      // CCF化纤信息装置表格入库接口地址
 	LIB_ROUTE_OILCHEM_TABLE_HANDLE      = "oilchem/handle/edb_data"  // 隆众资讯入库接口地址
+	LIB_ROUTE_FENWEI_NET_DATA_HANDLE    = "/fenwei/net/data/handle"    // 汾渭网页数据处理
 )