汾渭数据源-接口数据爬取

models/base_from_fenwei.go

@@ -59,7 +59,7 @@ type EdbLibFenweiIndexListResponse struct {
 	Data    []BaseFromFenweiIndex
 }
 
-// 接口爬取汾渭网页数据
+// RequestParams 接口爬取汾渭网页数据
 type RequestParams struct {
 	Category     interface{}         `json:"category"`
 	CheckedDims  map[string][]string `json:"checkedDims"`
@@ -73,3 +73,80 @@ type RequestParams struct {
 	IsSeason     int                 `json:"isSeason"`
 	splitTypeKey string              `json:"splitTypeKey"`
 }
+
+type Response struct {
+	Code    int    `json:"code"`
+	Message string `json:"message"`
+}
+
+type JsonConfig struct {
+	Data []string `json:"data"`
+}
+
+type FenWeiNetResponse struct {
+	MonthlyAccumulation        float64 `json:"monthly_accumulation"`
+	ProductItemCode            string  `json:"product_item_code"`
+	MonthlyValueChnName        string  `json:"monthly_value_chn_name"`
+	MonthlyAccumulationChnName string  `json:"monthly_accumulation_chn_name"`
+	ProductItemName            string  `json:"product_item_name"`
+	DataDate                   string  `json:"data_date"`
+	MonthlyValue               float64 `json:"monthly_value"`
+	MonthlyValueChnUnit        string  `json:"monthly_value_chn_unit"`
+	ProductCode                string  `json:"product_code"`
+	ProvinceName               string  `json:"province_name"`
+	MonthlyAccumulationChnUnit string  `json:"monthly_accumulation_chn_unit"`
+	PortName                   string  `json:"port_name"`
+	PortSonName                string  `json:"port_son_name"`
+	Stock                      float64 `json:"stock"`
+	StockChnName               string  `json:"stock_chn_name"`
+	StockChnUnit               string  `json:"stock_chn_unit"`
+	CoalTypeName               string  `json:"coal_type_name"`
+	WeekValueChnName           string  `json:"week_value_chn_name"`
+	WeekValue                  float64 `json:"week_value"`
+	AvaliableDaysChnName       string  `json:"avaliable_days_chn_name"`
+	AvaliableDaysChnUnit       string  `json:"avaliable_days_chn_unit"`
+	AvaliableDays              float64 `json:"avaliable_days"`
+	DailyConsumptionChnName    string  `json:"daily_consumption_chn_name"`
+	DailyConsumptionChnUnit    string  `json:"daily_consumption_chn_unit"`
+	DailyConsumption           float64 `json:"daily_consumption"`
+	InventoryIndexChnUnit      string  `json:"inventory_index_chn_unit"`
+	InventoryIndex             float64 `json:"inventory_index"`
+	TonsChnUnit                string  `json:"tons_chn_unit"`
+	Tons                       float64 `json:"tons"`
+	TransportVolumeChnUnit     string  `json:"transport_volume_chn_unit"`
+	TransportVolume            float64 `json:"transport_volume"`
+	PriceRmbChnName            string  `json:"price_rmb_chn_name"`
+	PriceIndexMomChnName       string  `json:"price_index_mom_chn_name"`
+	PriceRmbChnUnit            string  `json:"price_rmb_chn_unit"`
+	PriceIndexMomChnUnit       string  `json:"price_index_mom_chn_unit"`
+	PriceRmb                   float64 `json:"price_rmb"`
+	PriceIndexMom              float64 `json:"price_index_mom"`
+}
+
+type FenWeiNetResponseWrapper struct {
+	Data struct {
+		Data []FenWeiNetResponse `json:"data"`
+	} `json:"data"`
+}
+
+type FenWeiNetResponseMapWrapper struct {
+	Data struct {
+		Data map[string][]FenWeiNetResponse `json:"data"`
+	} `json:"data"`
+}
+
+type FenWeiNetResponseResult struct {
+	Product            string              `json:"product"`
+	FenWeiNetResponses []FenWeiNetResponse `json:"fenWeiNetResponses"`
+}
+
+type FenWeiNetIndexInfo struct {
+	IndexName string `description:"指标名称"`
+	//IndexCode    string  `description:"指标编码"`
+	Unit         string  `description:"单位"`
+	Frequency    string  `description:"频度"`
+	TerminalCode string  `description:"编码"`
+	ClassifyName string  `description:"分类名称"`
+	DataTime     string  `description:"数据时间"`
+	Value        float64 `description:"数据值"`
+}

services/fenwei/base_from_fenwei_service.go

@@ -3,20 +3,42 @@
 package fenwei
 
 import (
+	"encoding/json"
+	"eta/eta_data_analysis/models"
 	"eta/eta_data_analysis/utils"
 	"fmt"
+	"os"
 )
 
 // FenWeiNetDataDeal 汾渭网络数据处理
 func FenWeiNetDataDeal(err error) {
 	defer func() {
 		if err != nil {
-			fmt.Println("FenweiReadWatchIndexFile Err:" + err.Error())
-			utils.FileLog.Info(fmt.Sprintf("FenweiReadWatchIndexFile Err: %s", err.Error()))
+			fmt.Println("FenWeiNetDataDeal Err:" + err.Error())
+			utils.FileLog.Info(fmt.Sprintf("FenWeiNetDataDeal Err: %s", err.Error()))
 		}
 	}()
-	if utils.FenweiFileDir == "" {
-		utils.FileLog.Info("文件目录未配置")
+	utils.FileLog.Info("FenWeiNetDataDeal start")
+	// 读取配置
+	configFile, err := os.ReadFile(utils.FenweiNetJsonPath)
+	if err != nil {
+		utils.FileLog.Info(fmt.Sprintf("读取配置文件错误: %v", err))
 		return
 	}
+
+	// 定义通用的 map 结构体来解析 JSON
+	var config models.JsonConfig
+
+	// 解析 JSON 文件内容
+	err = json.Unmarshal(configFile, &config)
+	if err != nil {
+		utils.FileLog.Info(fmt.Sprintf("解析配置文件错误: %v", err))
+		return
+	}
+
+	factory := ProcessorFactory{}
+	// 遍历调用对应的处理方法
+	for _, v := range config.Data {
+		factory.CreateProcessor(v).FetchAndProcess()
+	}
 }

services/fenwei/data_processor.go

@@ -5,6 +5,9 @@ package fenwei
 import (
 	"bytes"
 	"context"
+	"encoding/json"
+	"eta/eta_data_analysis/models"
+	"eta/eta_data_analysis/services"
 	"eta/eta_data_analysis/utils"
 	"fmt"
 	"github.com/chromedp/cdproto/network"
@@ -12,11 +15,12 @@ import (
 	"io"
 	"log"
 	"net/http"
+	"sync"
 	"time"
 )
 
 type DataProcessor interface {
-	FetchAndProcess() (string, error)
+	FetchAndProcess() error
 	GenerateRequestParams(string) map[string]string
 }
 
@@ -24,36 +28,93 @@ type BaseProcessor struct {
 	URL string
 }
 
-func (p *BaseProcessor) FetchAndProcess() (string, error) {
+var (
+	authorization string
+	authLock      sync.RWMutex
+)
+
+func (p *BaseProcessor) FetchAndProcess() error {
 
 	// 获取当前时间 yyyy-MM-dd
 	now := time.Now()
 	currentTime := now.Format(utils.FormatDateUnSpace)
 
+	// 请求参数
 	params := p.GenerateRequestParams(currentTime)
 
 	req, err := http.NewRequest("POST", p.URL, bytes.NewBufferString(params["params"]))
 	if err != nil {
-		return "", err
+		return err
 	}
 
 	// 设置请求头
 	req.Header.Set("Content-Type", "application/json")
 	req.Header.Set("accept-language", "zh-CN,zh;q=0.9")
 
+	authLock.RLock()
+	req.Header.Set("Authorization", authorization)
+	authLock.RUnlock()
+
 	client := &http.Client{}
 	resp, err := client.Do(req)
 	if err != nil {
-		return "", err
+		return err
 	}
 	defer resp.Body.Close()
 
 	body, err := io.ReadAll(resp.Body)
 	if err != nil {
-		return "", err
+		return err
+	}
+	if checkResp(string(body)) {
+		authLock.Lock()
+
+		// 登录获取Authorization
+		authorization, err = getAuthorizationByChrome()
+		if err != nil {
+			authLock.Unlock()
+			return err
+		}
+		authLock.Unlock()
+
+		// 重新请求
+		req.Header.Set("Authorization", authorization)
+		resp, err = client.Do(req)
+		if err != nil {
+			return err
+		}
+	}
+
+	// 数据处理
+	response, err := p.ProcessResponse(string(body))
+	if err != nil {
+		return err
+	}
+	utils.FileLog.Info(fmt.Sprintf("response: %v", response))
+
+	// 请求lib应用入库
+	paramsLib := make(map[string]interface{})
+	paramsLib["List"] = response
+	paramsLib["TerminalCode"] = utils.TerminalCode
+	services.PostEdbLib(paramsLib, utils.LIB_ROUTE_FENWEI_NET_DATA_HANDLE)
+	return nil
+}
+
+// resp响应参数检测 code or message 判断是否需要重新登录
+func checkResp(resp string) bool {
+	if resp == "" {
+		return true
+	}
+	var responseObj models.Response
+	err := json.Unmarshal([]byte(resp), &responseObj)
+	if err != nil {
+		return false
+	}
+	if responseObj.Code != 200 || responseObj.Message != "成功!" {
+		return true
 	}
 
-	return p.ProcessResponse(string(body)), nil
+	return false
 }
 
 // GenerateRequestParams 让子类来实现这个方法
@@ -61,13 +122,13 @@ func (p *BaseProcessor) GenerateRequestParams(string) map[string]string {
 	return map[string]string{}
 }
 
-func (p *BaseProcessor) ProcessResponse(data string) string {
+func (p *BaseProcessor) ProcessResponse(data string) ([]models.FenWeiNetIndexInfo, error) {
 
-	return "Processed Data: " + data
+	return nil, nil
 }
 
-// GetCookieByChrome 获取cookie
-func getCookieByChrome() (cookieStr string, err error) {
+// GetAuthorizationByChrome 获取Authorization
+func getAuthorizationByChrome() (authorization string, err error) {
 	// 读取Cookie
 	if utils.FenweiNetUseName == "" {
 		err = fmt.Errorf("汾渭账号未设置")
@@ -103,7 +164,7 @@ func getCookieByChrome() (cookieStr string, err error) {
 		chromedp.Click(`.Button_text__D25sy`, chromedp.ByQuery),
 
 		chromedp.ActionFunc(func(ctx context.Context) error {
-			cookies, err := network.GetCookies().Do(ctx)
+			/*cookies, err := network.GetCookies().Do(ctx)
 			if err != nil {
 				return err
 			}
@@ -111,7 +172,19 @@ func getCookieByChrome() (cookieStr string, err error) {
 			for _, v := range cookies {
 				cookieStr = cookieStr + v.Name + "=" + v.Value + ";"
 			}
-			utils.FileLog.Info("header cookie:" + cookieStr)
+			utils.FileLog.Info("header cookie:" + cookieStr)*/
+
+			// 监听并处理请求事件
+			chromedp.ListenTarget(ctx, func(ev interface{}) {
+				if ev, ok := ev.(*network.EventRequestWillBeSent); ok {
+					for _, header := range ev.Request.Headers {
+						if authHeader, ok := header.(string); ok && authHeader == "Authorization" {
+							authorization = authHeader
+							utils.FileLog.Info("Authorization header found: " + authorization)
+						}
+					}
+				}
+			})
 
 			return nil
 		}),
@@ -119,11 +192,3 @@ func getCookieByChrome() (cookieStr string, err error) {
 
 	return
 }
-
-func encodeParams(params map[string]string) string {
-	var result string
-	for key, value := range params {
-		result += fmt.Sprintf("%s=%s&", key, value)
-	}
-	return result[:len(result)-1]
-}

services/fenwei/processor_business_logic.go

@@ -5,6 +5,9 @@ package fenwei
 import (
 	"encoding/json"
 	"eta/eta_data_analysis/models"
+	"eta/eta_data_analysis/utils"
+	"fmt"
+	"strings"
 )
 
 // ThermalCoalSupplyProcessor 动力煤供应量
@@ -36,6 +39,35 @@ func (p *ThermalCoalSupplyProcessor) GenerateRequestParams(currentTime string) m
 	}
 }
 
+func (p *ThermalCoalSupplyProcessor) ProcessResponse(data string) ([]models.FenWeiNetIndexInfo, error) {
+	responseData := fillFenWeiNetResponseData(data)
+
+	var indexInfoList []models.FenWeiNetIndexInfo
+	productName := "动力煤供应量"
+	for _, data := range responseData {
+		var frequency string
+		if data.MonthlyValueChnName != "" && strings.Contains(data.MonthlyValueChnName, "月度") {
+			frequency = "月度"
+		} else if data.WeekValueChnName != "" && strings.Contains(data.WeekValueChnName, "月度") {
+			frequency = "周度"
+		} else {
+			frequency = "日度"
+		}
+
+		indexInfoList = append(indexInfoList, models.FenWeiNetIndexInfo{
+			IndexName:    productName + data.ProvinceName + "/汾渭",
+			Unit:         data.MonthlyValueChnUnit,
+			Frequency:    frequency,
+			TerminalCode: utils.TerminalCode,
+			ClassifyName: productName,
+			DataTime:     data.DataDate,
+			Value:        data.MonthlyValue,
+		})
+	}
+
+	return indexInfoList, nil
+}
+
 // CokingCleanCoalSupplyProcessor 炼焦精煤供应量
 type CokingCleanCoalSupplyProcessor struct {
 	BaseProcessor
@@ -66,6 +98,35 @@ func (p *CokingCleanCoalSupplyProcessor) GenerateRequestParams(currentTime strin
 	}
 }
 
+func (p *CokingCleanCoalSupplyProcessor) ProcessResponse(data string) ([]models.FenWeiNetIndexInfo, error) {
+	responseData := fillFenWeiNetResponseData(data)
+
+	var indexInfoList []models.FenWeiNetIndexInfo
+	productName := "炼焦精煤供应量"
+	for _, data := range responseData {
+		var frequency string
+		if data.MonthlyValueChnName != "" && strings.Contains(data.MonthlyValueChnName, "月度") {
+			frequency = "月度"
+		} else if data.WeekValueChnName != "" && strings.Contains(data.WeekValueChnName, "月度") {
+			frequency = "周度"
+		} else {
+			frequency = data.MonthlyValueChnName
+		}
+
+		indexInfoList = append(indexInfoList, models.FenWeiNetIndexInfo{
+			IndexName:    productName + data.CoalTypeName + "/汾渭",
+			Unit:         data.MonthlyValueChnUnit,
+			Frequency:    frequency,
+			TerminalCode: utils.TerminalCode,
+			ClassifyName: productName,
+			DataTime:     data.DataDate,
+			Value:        data.MonthlyValue,
+		})
+	}
+
+	return indexInfoList, nil
+}
+
 // RawCoalProvinceProductionProcessor 原煤分省分煤种产量
 type RawCoalProvinceProductionProcessor struct {
 	BaseProcessor
@@ -96,6 +157,35 @@ func (p *RawCoalProvinceProductionProcessor) GenerateRequestParams(currentTime s
 	}
 }
 
+func (p *RawCoalProvinceProductionProcessor) ProcessResponse(data string) ([]models.FenWeiNetIndexInfo, error) {
+	responseData := fillFenWeiNetResponseData(data)
+
+	var indexInfoList []models.FenWeiNetIndexInfo
+	productName := "原煤分省分煤种产量"
+	for _, data := range responseData {
+		var frequency string
+		if data.MonthlyValueChnName != "" && strings.Contains(data.MonthlyValueChnName, "月度") {
+			frequency = "月度"
+		} else if data.WeekValueChnName != "" && strings.Contains(data.WeekValueChnName, "月度") {
+			frequency = "周度"
+		} else {
+			frequency = data.MonthlyValueChnName
+		}
+
+		indexInfoList = append(indexInfoList, models.FenWeiNetIndexInfo{
+			IndexName:    productName + data.CoalTypeName + "/汾渭",
+			Unit:         data.MonthlyValueChnUnit,
+			Frequency:    frequency,
+			TerminalCode: utils.TerminalCode,
+			ClassifyName: productName,
+			DataTime:     data.DataDate,
+			Value:        data.MonthlyValue,
+		})
+	}
+
+	return indexInfoList, nil
+}
+
 // StateOwnedKeyCoalMineRawCoalProductionProcessor 国有重点煤矿原煤产量
 type StateOwnedKeyCoalMineRawCoalProductionProcessor struct {
 	BaseProcessor
@@ -125,6 +215,35 @@ func (p *StateOwnedKeyCoalMineRawCoalProductionProcessor) GenerateRequestParams(
 	}
 }
 
+func (p *StateOwnedKeyCoalMineRawCoalProductionProcessor) ProcessResponse(data string) ([]models.FenWeiNetIndexInfo, error) {
+	responseData := fillFenWeiNetResponseData(data)
+
+	var indexInfoList []models.FenWeiNetIndexInfo
+	productName := "国有重点煤矿原煤产量"
+	for _, data := range responseData {
+		var frequency string
+		if data.MonthlyValueChnName != "" && strings.Contains(data.MonthlyValueChnName, "月度") {
+			frequency = "月度"
+		} else if data.WeekValueChnName != "" && strings.Contains(data.WeekValueChnName, "月度") {
+			frequency = "周度"
+		} else {
+			frequency = data.MonthlyValueChnName
+		}
+
+		indexInfoList = append(indexInfoList, models.FenWeiNetIndexInfo{
+			IndexName:    productName + data.ProvinceName + "/汾渭",
+			Unit:         data.MonthlyValueChnUnit,
+			Frequency:    frequency,
+			TerminalCode: utils.TerminalCode,
+			ClassifyName: productName,
+			DataTime:     data.DataDate,
+			Value:        data.MonthlyValue,
+		})
+	}
+
+	return indexInfoList, nil
+}
+
 // CokingBituminousCoalProductionProcessor 炼焦烟煤分煤种产量
 type CokingBituminousCoalProductionProcessor struct {
 	BaseProcessor
@@ -155,6 +274,35 @@ func (p *CokingBituminousCoalProductionProcessor) GenerateRequestParams(currentT
 	}
 }
 
+func (p *CokingBituminousCoalProductionProcessor) ProcessResponse(data string) ([]models.FenWeiNetIndexInfo, error) {
+	responseData := fillFenWeiNetResponseData(data)
+
+	var indexInfoList []models.FenWeiNetIndexInfo
+	productName := "炼焦烟煤分煤种产量"
+	for _, data := range responseData {
+		var frequency string
+		if data.MonthlyValueChnName != "" && strings.Contains(data.MonthlyValueChnName, "月度") {
+			frequency = "月度"
+		} else if data.WeekValueChnName != "" && strings.Contains(data.WeekValueChnName, "月度") {
+			frequency = "周度"
+		} else {
+			frequency = data.MonthlyValueChnName
+		}
+
+		indexInfoList = append(indexInfoList, models.FenWeiNetIndexInfo{
+			IndexName:    productName + data.CoalTypeName + "/汾渭",
+			Unit:         data.MonthlyValueChnUnit,
+			Frequency:    frequency,
+			TerminalCode: utils.TerminalCode,
+			ClassifyName: productName,
+			DataTime:     data.DataDate,
+			Value:        data.MonthlyValue,
+		})
+	}
+
+	return indexInfoList, nil
+}
+
 // ThermalCoalInventorySocietyProcessor 动力煤库存-全社会
 type ThermalCoalInventorySocietyProcessor struct {
 	BaseProcessor
@@ -183,6 +331,35 @@ func (p *ThermalCoalInventorySocietyProcessor) GenerateRequestParams(currentTime
 	}
 }
 
+func (p *ThermalCoalInventorySocietyProcessor) ProcessResponse(data string) ([]models.FenWeiNetIndexInfo, error) {
+	responseData := fillFenWeiNetResponseData(data)
+
+	var indexInfoList []models.FenWeiNetIndexInfo
+	productName := "动力煤库存-全社会"
+	for _, data := range responseData {
+		var frequency string
+		if data.MonthlyValueChnName != "" && strings.Contains(data.MonthlyValueChnName, "月度") {
+			frequency = "月度"
+		} else if data.WeekValueChnName != "" && strings.Contains(data.WeekValueChnName, "月度") {
+			frequency = "周度"
+		} else {
+			frequency = data.MonthlyValueChnName
+		}
+
+		indexInfoList = append(indexInfoList, models.FenWeiNetIndexInfo{
+			IndexName:    productName + "/汾渭",
+			Unit:         data.MonthlyValueChnUnit,
+			Frequency:    frequency,
+			TerminalCode: utils.TerminalCode,
+			ClassifyName: productName,
+			DataTime:     data.DataDate,
+			Value:        data.MonthlyValue,
+		})
+	}
+
+	return indexInfoList, nil
+}
+
 // ThermalCoalInventoryProductionProcessor 动力煤库存-生产企业
 type ThermalCoalInventoryProductionProcessor struct {
 	BaseProcessor
@@ -211,6 +388,35 @@ func (p *ThermalCoalInventoryProductionProcessor) GenerateRequestParams(currentT
 	}
 }
 
+func (p *ThermalCoalInventoryProductionProcessor) ProcessResponse(data string) ([]models.FenWeiNetIndexInfo, error) {
+	responseData := fillFenWeiNetResponseData(data)
+
+	var indexInfoList []models.FenWeiNetIndexInfo
+	productName := "动力煤库存-生产企业"
+	for _, data := range responseData {
+		var frequency string
+		if data.MonthlyValueChnName != "" && strings.Contains(data.MonthlyValueChnName, "月度") {
+			frequency = "月度"
+		} else if data.WeekValueChnName != "" && strings.Contains(data.WeekValueChnName, "月度") {
+			frequency = "周度"
+		} else {
+			frequency = data.MonthlyValueChnName
+		}
+
+		indexInfoList = append(indexInfoList, models.FenWeiNetIndexInfo{
+			IndexName:    productName + "/汾渭",
+			Unit:         data.MonthlyValueChnUnit,
+			Frequency:    frequency,
+			TerminalCode: utils.TerminalCode,
+			ClassifyName: productName,
+			DataTime:     data.DataDate,
+			Value:        data.MonthlyValue,
+		})
+	}
+
+	return indexInfoList, nil
+}
+
 // ThermalCoalInventorySixPowerPlantProcessor 动力煤库存-六大电厂
 type ThermalCoalInventorySixPowerPlantProcessor struct {
 	BaseProcessor
@@ -239,6 +445,58 @@ func (p *ThermalCoalInventorySixPowerPlantProcessor) GenerateRequestParams(curre
 	}
 }
 
+func (p *ThermalCoalInventorySixPowerPlantProcessor) ProcessResponse(data string) ([]models.FenWeiNetIndexInfo, error) {
+	responseData := fillFenWeiNetResponseData(data)
+
+	var indexInfoList []models.FenWeiNetIndexInfo
+	productName := "动力煤库存"
+	for _, data := range responseData {
+		var frequency string
+		if data.MonthlyValueChnName != "" && strings.Contains(data.MonthlyValueChnName, "月度") {
+			frequency = "月度"
+		} else if data.WeekValueChnName != "" && strings.Contains(data.WeekValueChnName, "月度") {
+			frequency = "周度"
+		} else {
+			frequency = "日度"
+		}
+
+		classifyName := "动力煤库存-六大电厂"
+
+		// 库存
+		indexInfoList = append(indexInfoList, models.FenWeiNetIndexInfo{
+			IndexName:    productName + data.ProductItemName + data.StockChnName + "/汾渭",
+			Unit:         data.StockChnUnit,
+			Frequency:    frequency,
+			TerminalCode: utils.TerminalCode,
+			ClassifyName: classifyName,
+			DataTime:     data.DataDate,
+			Value:        data.Stock,
+		})
+		// 可用天数
+		indexInfoList = append(indexInfoList, models.FenWeiNetIndexInfo{
+			IndexName:    productName + data.ProductItemName + data.AvaliableDaysChnName + "/汾渭",
+			Unit:         data.AvaliableDaysChnUnit,
+			Frequency:    frequency,
+			TerminalCode: utils.TerminalCode,
+			ClassifyName: classifyName,
+			DataTime:     data.DataDate,
+			Value:        data.AvaliableDays,
+		})
+		// 日耗
+		indexInfoList = append(indexInfoList, models.FenWeiNetIndexInfo{
+			IndexName:    productName + data.ProductItemName + data.DailyConsumptionChnName + "/汾渭",
+			Unit:         data.DailyConsumptionChnUnit,
+			Frequency:    frequency,
+			TerminalCode: utils.TerminalCode,
+			ClassifyName: classifyName,
+			DataTime:     data.DataDate,
+			Value:        data.DailyConsumption,
+		})
+	}
+
+	return indexInfoList, nil
+}
+
 // CokingCoalInventorySocietyProcessor 炼焦煤库存-全社会
 type CokingCoalInventorySocietyProcessor struct {
 	BaseProcessor
@@ -267,6 +525,36 @@ func (p *CokingCoalInventorySocietyProcessor) GenerateRequestParams(currentTime
 	}
 }
 
+func (p *CokingCoalInventorySocietyProcessor) ProcessResponse(data string) ([]models.FenWeiNetIndexInfo, error) {
+	responseData := fillFenWeiNetResponseData(data)
+
+	var indexInfoList []models.FenWeiNetIndexInfo
+	productName := "炼焦煤库存-全社会"
+
+	for _, data := range responseData {
+		var frequency string
+		if data.MonthlyValueChnName != "" && strings.Contains(data.MonthlyValueChnName, "月度") {
+			frequency = "月度"
+		} else if data.WeekValueChnName != "" && strings.Contains(data.WeekValueChnName, "月度") {
+			frequency = "周度"
+		} else {
+			frequency = data.MonthlyValueChnName
+		}
+
+		indexInfoList = append(indexInfoList, models.FenWeiNetIndexInfo{
+			IndexName:    data.ProductItemName + "/汾渭",
+			Unit:         data.MonthlyValueChnUnit,
+			Frequency:    frequency,
+			TerminalCode: utils.TerminalCode,
+			ClassifyName: productName,
+			DataTime:     data.DataDate,
+			Value:        data.MonthlyValue,
+		})
+	}
+
+	return indexInfoList, nil
+}
+
 // CokingCoalInventoryProductionProcessor 炼焦煤库存-生产企业
 type CokingCoalInventoryProductionProcessor struct {
 	BaseProcessor
@@ -295,6 +583,36 @@ func (p *CokingCoalInventoryProductionProcessor) GenerateRequestParams(currentTi
 	}
 }
 
+func (p *CokingCoalInventoryProductionProcessor) ProcessResponse(data string) ([]models.FenWeiNetIndexInfo, error) {
+	responseData := fillFenWeiNetResponseData(data)
+
+	var indexInfoList []models.FenWeiNetIndexInfo
+	productName := "炼焦煤库存-生产企业"
+
+	for _, data := range responseData {
+		var frequency string
+		if data.MonthlyValueChnName != "" && strings.Contains(data.MonthlyValueChnName, "月度") {
+			frequency = "月度"
+		} else if data.WeekValueChnName != "" && strings.Contains(data.WeekValueChnName, "月度") {
+			frequency = "周度"
+		} else {
+			frequency = data.MonthlyValueChnName
+		}
+
+		indexInfoList = append(indexInfoList, models.FenWeiNetIndexInfo{
+			IndexName:    data.ProductItemName + "/汾渭",
+			Unit:         data.MonthlyValueChnUnit,
+			Frequency:    frequency,
+			TerminalCode: utils.TerminalCode,
+			ClassifyName: productName,
+			DataTime:     data.DataDate,
+			Value:        data.WeekValue,
+		})
+	}
+
+	return indexInfoList, nil
+}
+
 // CokingCoalInventoryDownstreamProcessor 炼焦煤库存-下游企业
 type CokingCoalInventoryDownstreamProcessor struct {
 	BaseProcessor
@@ -323,6 +641,29 @@ func (p *CokingCoalInventoryDownstreamProcessor) GenerateRequestParams(currentTi
 	}
 }
 
+func (p *CokingCoalInventoryDownstreamProcessor) ProcessResponse(data string) ([]models.FenWeiNetIndexInfo, error) {
+	responseData := fillFenWeiNetResponseData(data)
+
+	var indexInfoList []models.FenWeiNetIndexInfo
+	productName := "炼焦煤库存-下游企业"
+
+	for _, data := range responseData {
+		frequency := "周度"
+
+		indexInfoList = append(indexInfoList, models.FenWeiNetIndexInfo{
+			IndexName:    data.ProductItemName + "/汾渭",
+			Unit:         data.StockChnUnit,
+			Frequency:    frequency,
+			TerminalCode: utils.TerminalCode,
+			ClassifyName: productName,
+			DataTime:     data.DataDate,
+			Value:        data.Stock,
+		})
+	}
+
+	return indexInfoList, nil
+}
+
 // NationalCoalMineInventoryProcessor 全国煤矿库存
 type NationalCoalMineInventoryProcessor struct {
 	BaseProcessor
@@ -350,6 +691,36 @@ func (p *NationalCoalMineInventoryProcessor) GenerateRequestParams(currentTime s
 	}
 }
 
+func (p *NationalCoalMineInventoryProcessor) ProcessResponse(data string) ([]models.FenWeiNetIndexInfo, error) {
+	responseData := fillFenWeiNetResponseData(data)
+
+	var indexInfoList []models.FenWeiNetIndexInfo
+	productName := "全国煤矿库存"
+
+	for _, data := range responseData {
+		var frequency string
+		if data.MonthlyValueChnName != "" && strings.Contains(data.MonthlyValueChnName, "月度") {
+			frequency = "月度"
+		} else if data.WeekValueChnName != "" && strings.Contains(data.WeekValueChnName, "月度") {
+			frequency = "周度"
+		} else {
+			frequency = "日度"
+		}
+
+		indexInfoList = append(indexInfoList, models.FenWeiNetIndexInfo{
+			IndexName:    productName + "/汾渭",
+			Unit:         data.MonthlyValueChnUnit,
+			Frequency:    frequency,
+			TerminalCode: utils.TerminalCode,
+			ClassifyName: productName,
+			DataTime:     data.DataDate,
+			Value:        data.MonthlyValue,
+		})
+	}
+
+	return indexInfoList, nil
+}
+
 // StateOwnedKeyCoalMineInventoryProcessor 国有重点煤矿库存
 type StateOwnedKeyCoalMineInventoryProcessor struct {
 	BaseProcessor
@@ -379,6 +750,36 @@ func (p *StateOwnedKeyCoalMineInventoryProcessor) GenerateRequestParams(currentT
 	}
 }
 
+func (p *StateOwnedKeyCoalMineInventoryProcessor) ProcessResponse(data string) ([]models.FenWeiNetIndexInfo, error) {
+	responseData := fillFenWeiNetResponseData(data)
+
+	var indexInfoList []models.FenWeiNetIndexInfo
+	productName := "国有重点煤矿库存"
+
+	for _, data := range responseData {
+		var frequency string
+		if data.MonthlyValueChnName != "" && strings.Contains(data.MonthlyValueChnName, "月度") {
+			frequency = "月度"
+		} else if data.WeekValueChnName != "" && strings.Contains(data.WeekValueChnName, "月度") {
+			frequency = "周度"
+		} else {
+			frequency = "日度"
+		}
+
+		indexInfoList = append(indexInfoList, models.FenWeiNetIndexInfo{
+			IndexName:    productName + data.ProvinceName + "/汾渭",
+			Unit:         data.MonthlyValueChnUnit,
+			Frequency:    frequency,
+			TerminalCode: utils.TerminalCode,
+			ClassifyName: productName,
+			DataTime:     data.DataDate,
+			Value:        data.MonthlyValue,
+		})
+	}
+
+	return indexInfoList, nil
+}
+
 // CokeInventoryProcessor 焦炭库存
 type CokeInventoryProcessor struct {
 	BaseProcessor
@@ -407,6 +808,57 @@ func (p *CokeInventoryProcessor) GenerateRequestParams(currentTime string) map[s
 	}
 }
 
+func (p *CokeInventoryProcessor) ProcessResponse(data string) ([]models.FenWeiNetIndexInfo, error) {
+	responseDataMap := fillFenWeiNetResponseMapData(data)
+
+	var indexInfoList []models.FenWeiNetIndexInfo
+	productName := "焦炭库存"
+	for productCategoryName, responseDataList := range responseDataMap {
+		utils.FileLog.Info(fmt.Sprintf("productName: %s, responseDataList:size: %v", productName, len(responseDataList)))
+		for _, data := range responseDataList {
+			var frequency string
+			productCodeMap := map[string]struct{}{
+				"FW1405D-1": {},
+				"FW1405D-2": {},
+				"FW1405D-3": {},
+				"FW1405D-6": {},
+				"FW1405D-7": {},
+			}
+			if _, ok := productCodeMap[data.ProductItemCode]; ok {
+				frequency = "周度"
+			} else {
+				frequency = "日度"
+			}
+
+			var unit string
+			if data.InventoryIndexChnUnit != "" {
+				unit = data.InventoryIndexChnUnit
+			} else {
+				unit = data.StockChnUnit
+			}
+
+			var value float64
+			if data.InventoryIndex != 0 {
+				value = data.InventoryIndex
+			} else {
+				value = data.Stock
+			}
+
+			indexInfoList = append(indexInfoList, models.FenWeiNetIndexInfo{
+				IndexName:    productName + productCategoryName + "/汾渭",
+				Unit:         unit,
+				Frequency:    frequency,
+				TerminalCode: utils.TerminalCode,
+				ClassifyName: productName,
+				DataTime:     data.DataDate,
+				Value:        value,
+			})
+		}
+	}
+
+	return indexInfoList, nil
+}
+
 // PortDataInventoryNorthernPortProcessor 港口数据-库存-北方港口
 type PortDataInventoryNorthernPortProcessor struct {
 	BaseProcessor
@@ -437,6 +889,36 @@ func (p *PortDataInventoryNorthernPortProcessor) GenerateRequestParams(currentTi
 	}
 }
 
+func (p *PortDataInventoryNorthernPortProcessor) ProcessResponse(data string) ([]models.FenWeiNetIndexInfo, error) {
+	responseData := fillFenWeiNetResponseData(data)
+
+	var indexInfoList []models.FenWeiNetIndexInfo
+	productName := "港口数据-库存-北方港口"
+
+	for _, data := range responseData {
+		var frequency string
+		if data.MonthlyValueChnName != "" && strings.Contains(data.MonthlyValueChnName, "月度") {
+			frequency = "月度"
+		} else if data.WeekValueChnName != "" && strings.Contains(data.WeekValueChnName, "月度") {
+			frequency = "周度"
+		} else {
+			frequency = "日度"
+		}
+
+		indexInfoList = append(indexInfoList, models.FenWeiNetIndexInfo{
+			IndexName:    productName + data.ProductItemName + "/汾渭",
+			Unit:         data.StockChnName,
+			Frequency:    frequency,
+			TerminalCode: utils.TerminalCode,
+			ClassifyName: productName,
+			DataTime:     data.DataDate,
+			Value:        data.Stock,
+		})
+	}
+
+	return indexInfoList, nil
+}
+
 // PortDataInventoryInlandPortProcessor 港口数据-库存-江内港口
 type PortDataInventoryInlandPortProcessor struct {
 	BaseProcessor
@@ -467,6 +949,29 @@ func (p *PortDataInventoryInlandPortProcessor) GenerateRequestParams(currentTime
 	}
 }
 
+func (p *PortDataInventoryInlandPortProcessor) ProcessResponse(data string) ([]models.FenWeiNetIndexInfo, error) {
+	responseData := fillFenWeiNetResponseData(data)
+
+	var indexInfoList []models.FenWeiNetIndexInfo
+	productName := "港口数据-库存-江内港口"
+
+	for _, data := range responseData {
+		var frequency = "周度"
+
+		indexInfoList = append(indexInfoList, models.FenWeiNetIndexInfo{
+			IndexName:    productName + data.ProductItemName + "/汾渭",
+			Unit:         data.StockChnName,
+			Frequency:    frequency,
+			TerminalCode: utils.TerminalCode,
+			ClassifyName: productName,
+			DataTime:     data.DataDate,
+			Value:        data.Stock,
+		})
+	}
+
+	return indexInfoList, nil
+}
+
 // PortDataDispatchNorthernPortProcessor 港口数据-调度-北方港口
 type PortDataDispatchNorthernPortProcessor struct {
 	BaseProcessor
@@ -498,6 +1003,29 @@ func (p *PortDataDispatchNorthernPortProcessor) GenerateRequestParams(currentTim
 	}
 }
 
+func (p *PortDataDispatchNorthernPortProcessor) ProcessResponse(data string) ([]models.FenWeiNetIndexInfo, error) {
+	responseData := fillFenWeiNetResponseData(data)
+
+	var indexInfoList []models.FenWeiNetIndexInfo
+	productName := "港口数据-调度-北方港口"
+
+	for _, data := range responseData {
+		var frequency = "日度"
+
+		indexInfoList = append(indexInfoList, models.FenWeiNetIndexInfo{
+			IndexName:    productName + "-" + data.ProductItemName + "-" + data.PortName + "/汾渭",
+			Unit:         data.TonsChnUnit,
+			Frequency:    frequency,
+			TerminalCode: utils.TerminalCode,
+			ClassifyName: productName,
+			DataTime:     data.DataDate,
+			Value:        data.Tons,
+		})
+	}
+
+	return indexInfoList, nil
+}
+
 // PortDataThroughputProcessor 港口数据-运量
 type PortDataThroughputProcessor struct {
 	BaseProcessor
@@ -529,6 +1057,29 @@ func (p *PortDataThroughputProcessor) GenerateRequestParams(currentTime string)
 	}
 }
 
+func (p *PortDataThroughputProcessor) ProcessResponse(data string) ([]models.FenWeiNetIndexInfo, error) {
+	responseData := fillFenWeiNetResponseData(data)
+
+	var indexInfoList []models.FenWeiNetIndexInfo
+	productName := "港口数据-运量"
+
+	for _, data := range responseData {
+		var frequency = "日度"
+
+		indexInfoList = append(indexInfoList, models.FenWeiNetIndexInfo{
+			IndexName:    productName + data.PortSonName + "/汾渭",
+			Unit:         data.TransportVolumeChnUnit,
+			Frequency:    frequency,
+			TerminalCode: utils.TerminalCode,
+			ClassifyName: productName,
+			DataTime:     data.DataDate,
+			Value:        data.TransportVolume,
+		})
+	}
+
+	return indexInfoList, nil
+}
+
 // DaqinLineDailyThroughputProcessor 大秦线日运量
 type DaqinLineDailyThroughputProcessor struct {
 	BaseProcessor
@@ -555,6 +1106,29 @@ func (p *DaqinLineDailyThroughputProcessor) GenerateRequestParams(currentTime st
 	}
 }
 
+func (p *DaqinLineDailyThroughputProcessor) ProcessResponse(data string) ([]models.FenWeiNetIndexInfo, error) {
+	responseData := fillFenWeiNetResponseData(data)
+
+	var indexInfoList []models.FenWeiNetIndexInfo
+	productName := "大秦线日运量"
+
+	for _, data := range responseData {
+		var frequency = "日度"
+
+		indexInfoList = append(indexInfoList, models.FenWeiNetIndexInfo{
+			IndexName:    productName + data.StockChnName + "/汾渭",
+			Unit:         data.StockChnUnit,
+			Frequency:    frequency,
+			TerminalCode: utils.TerminalCode,
+			ClassifyName: productName,
+			DataTime:     data.DataDate,
+			Value:        data.Stock,
+		})
+	}
+
+	return indexInfoList, nil
+}
+
 // ThermalCoalPortPriceProcessor 动力煤港口价格
 type ThermalCoalPortPriceProcessor struct {
 	BaseProcessor
@@ -583,6 +1157,40 @@ func (p *ThermalCoalPortPriceProcessor) GenerateRequestParams(currentTime string
 	}
 }
 
+func (p *ThermalCoalPortPriceProcessor) ProcessResponse(data string) ([]models.FenWeiNetIndexInfo, error) {
+	responseDataMap := fillFenWeiNetResponseMapData(data)
+
+	var indexInfoList []models.FenWeiNetIndexInfo
+	productName := "动力煤港口价格"
+	for productCategoryName, responseDataList := range responseDataMap {
+		utils.FileLog.Info(fmt.Sprintf("productName: %s, responseDataList:size: %v", productName, len(responseDataList)))
+		for _, data := range responseDataList {
+			var frequency = "日度"
+
+			indexInfoList = append(indexInfoList, models.FenWeiNetIndexInfo{
+				IndexName:    productName + productCategoryName + "-" + data.PriceRmbChnName + "/汾渭",
+				Unit:         data.PriceRmbChnUnit,
+				Frequency:    frequency,
+				TerminalCode: utils.TerminalCode,
+				ClassifyName: productName,
+				DataTime:     data.DataDate,
+				Value:        data.PriceRmb,
+			})
+			indexInfoList = append(indexInfoList, models.FenWeiNetIndexInfo{
+				IndexName:    productName + productCategoryName + "-" + data.PriceIndexMomChnName + "/汾渭",
+				Unit:         data.PriceIndexMomChnUnit,
+				Frequency:    frequency,
+				TerminalCode: utils.TerminalCode,
+				ClassifyName: productName,
+				DataTime:     data.DataDate,
+				Value:        data.PriceIndexMom,
+			})
+		}
+	}
+
+	return indexInfoList, nil
+}
+
 // ThermalCoalConsumptionProcessor 动力煤消费量
 type ThermalCoalConsumptionProcessor struct {
 	BaseProcessor
@@ -611,6 +1219,38 @@ func (p *ThermalCoalConsumptionProcessor) GenerateRequestParams(currentTime stri
 	}
 }
 
+func (p *ThermalCoalConsumptionProcessor) ProcessResponse(data string) ([]models.FenWeiNetIndexInfo, error) {
+	responseDataMap := fillFenWeiNetResponseMapData(data)
+
+	var indexInfoList []models.FenWeiNetIndexInfo
+	productName := "动力煤消费量"
+	for productCategoryName, responseDataList := range responseDataMap {
+		utils.FileLog.Info(fmt.Sprintf("productName: %s, responseDataList:size: %v", productName, len(responseDataList)))
+		for _, data := range responseDataList {
+			var frequency string
+			if data.MonthlyValueChnName != "" && strings.Contains(data.MonthlyValueChnName, "月度") {
+				frequency = "月度"
+			} else if data.WeekValueChnName != "" && strings.Contains(data.WeekValueChnName, "月度") {
+				frequency = "周度"
+			} else {
+				frequency = "日度"
+			}
+
+			indexInfoList = append(indexInfoList, models.FenWeiNetIndexInfo{
+				IndexName:    productName + productCategoryName + "/汾渭",
+				Unit:         data.MonthlyValueChnUnit,
+				Frequency:    frequency,
+				TerminalCode: utils.TerminalCode,
+				ClassifyName: productName,
+				DataTime:     data.DataDate,
+				Value:        data.MonthlyValue,
+			})
+		}
+	}
+
+	return indexInfoList, nil
+}
+
 // CokingCleanCoalConsumptionProcessor 炼焦精煤消费量
 type CokingCleanCoalConsumptionProcessor struct {
 	BaseProcessor
@@ -636,3 +1276,53 @@ func (p *CokingCleanCoalConsumptionProcessor) GenerateRequestParams(currentTime
 		"params": string(paramsJSON),
 	}
 }
+
+func (p *CokingCleanCoalConsumptionProcessor) ProcessResponse(data string) ([]models.FenWeiNetIndexInfo, error) {
+	responseData := fillFenWeiNetResponseData(data)
+
+	var indexInfoList []models.FenWeiNetIndexInfo
+	productName := "炼焦精煤消费量"
+
+	for _, data := range responseData {
+		var frequency string
+		if data.MonthlyValueChnName != "" && strings.Contains(data.MonthlyValueChnName, "月度") {
+			frequency = "月度"
+		} else if data.WeekValueChnName != "" && strings.Contains(data.WeekValueChnName, "月度") {
+			frequency = "周度"
+		} else {
+			frequency = "日度"
+		}
+
+		indexInfoList = append(indexInfoList, models.FenWeiNetIndexInfo{
+			IndexName:    productName + "/汾渭",
+			Unit:         data.MonthlyValueChnUnit,
+			Frequency:    frequency,
+			TerminalCode: utils.TerminalCode,
+			ClassifyName: productName,
+			DataTime:     data.DataDate,
+			Value:        data.MonthlyValue,
+		})
+	}
+
+	return indexInfoList, nil
+}
+
+func fillFenWeiNetResponseData(data string) []models.FenWeiNetResponse {
+	var result models.FenWeiNetResponseWrapper
+	err := json.Unmarshal([]byte(data), &result)
+	if err != nil {
+		return nil
+	}
+	responseData := result.Data.Data
+	return responseData
+}
+
+func fillFenWeiNetResponseMapData(data string) map[string][]models.FenWeiNetResponse {
+	var result models.FenWeiNetResponseMapWrapper
+	err := json.Unmarshal([]byte(data), &result)
+	if err != nil {
+		return nil
+	}
+	responseData := result.Data.Data
+	return responseData
+}

utils/config.go

@@ -63,6 +63,7 @@ var (
 	FenweiNetOpen     string // 是否配置汾渭网页数据源
 	FenweiNetUseName  string // 汾渭登录账号
 	FenweiNetPassword string // 汾渭登录密码
+	FenweiNetJsonPath string // 汾渭json文件地址
 )
 
 // 煤炭江湖
@@ -150,6 +151,9 @@ func init() {
 		FenweiOldFileDir = config["fenwei_old_file_dir"]
 
 		FenweiNetOpen = config["fenwei_net_open"]
+		FenweiNetUseName = config["fenwei_net_username"]
+		FenweiNetPassword = config["fenwei_net_password"]
+		FenweiNetJsonPath = config["fenwei_net_json_path"]
 	}
 
 	//煤炭江湖文件夹配置

utils/constants.go

@@ -246,4 +246,5 @@ const (
 	LIB_ROUTE_COAL_MINE_MTJH            = "/mtjh/data"                 //煤炭江湖数据处理excel数据并入库 数据地址
 	LIB_ROUTE_CCF_EDB_HANDLE            = "ccf/handle/edb_data"        // CCF化纤信息指标入库接口地址
 	LIB_ROUTE_CCF_TABLE_HANDLE          = "ccf/handle/table_data"      // CCF化纤信息装置表格入库接口地址
+	LIB_ROUTE_FENWEI_NET_DATA_HANDLE    = "/fenwei/net/data/handle"    // 汾渭网页数据处理
 )