汾渭数据源-接口数据爬取

models/base_from_fenwei.go

@@ -67,7 +67,7 @@ type RequestParams struct {
 	ProductCode  string              `json:"productCode"`
 	QuotaName    string              `json:"quotaName"`
 	SplitTypeKey string              `json:"splitTypeKey"`
-	IsTotal      int                 `json:"isTotal"`
+	IsTotal      interface{}         `json:"isTotal"`
 	DataType     interface{}         `json:"dataType"`
 	Type         int                 `json:"type"`
 	IsSeason     int                 `json:"isSeason"`
@@ -121,6 +121,8 @@ type FenWeiNetResponse struct {
 	PriceIndexMomChnUnit       string  `json:"price_index_mom_chn_unit"`
 	PriceRmb                   float64 `json:"price_rmb"`
 	PriceIndexMom              float64 `json:"price_index_mom"`
+	VehicleChnUnit             string  `json:"vehicle_chn_unit"`
+	ShipChnUnit                string  `json:"ship_chn_unit"`
 }
 
 type FenWeiNetResponseWrapper struct {
@@ -135,6 +137,10 @@ type FenWeiNetResponseMapWrapper struct {
 	} `json:"data"`
 }
 
+type FenWeiNetResponseAStratumMapWrapper struct {
+	Data map[string][]FenWeiNetResponse `json:"data"`
+}
+
 type FenWeiNetResponseResult struct {
 	Product            string              `json:"product"`
 	FenWeiNetResponses []FenWeiNetResponse `json:"fenWeiNetResponses"`

services/fenwei/processor_business_logic.go

@@ -332,29 +332,36 @@ func (p *ThermalCoalInventorySocietyProcessor) GenerateRequestParams(currentTime
 }
 
 func (p *ThermalCoalInventorySocietyProcessor) ProcessResponse(data string) ([]models.FenWeiNetIndexInfo, error) {
-	responseData := fillFenWeiNetResponseData(data)
+	responseDataMap := fillFenWeiNetResponseMapData(data)
 
 	var indexInfoList []models.FenWeiNetIndexInfo
 	productName := "动力煤库存-全社会"
-	for _, data := range responseData {
-		var frequency string
-		if data.MonthlyValueChnName != "" && strings.Contains(data.MonthlyValueChnName, "月度") {
-			frequency = "月度"
-		} else if data.WeekValueChnName != "" && strings.Contains(data.WeekValueChnName, "月度") {
-			frequency = "周度"
-		} else {
-			frequency = data.MonthlyValueChnName
+
+	for productCategoryName, responseDataList := range responseDataMap {
+		if productCategoryName != productName {
+			continue
 		}
 
-		indexInfoList = append(indexInfoList, models.FenWeiNetIndexInfo{
-			IndexName:    productName + "/汾渭",
-			Unit:         data.MonthlyValueChnUnit,
-			Frequency:    frequency,
-			TerminalCode: utils.TerminalCode,
-			ClassifyName: productName,
-			DataTime:     data.DataDate,
-			Value:        data.MonthlyValue,
-		})
+		for _, data := range responseDataList {
+			var frequency string
+			if data.MonthlyValueChnName != "" && strings.Contains(data.MonthlyValueChnName, "月度") {
+				frequency = "月度"
+			} else if data.WeekValueChnName != "" && strings.Contains(data.WeekValueChnName, "月度") {
+				frequency = "周度"
+			} else {
+				frequency = data.MonthlyValueChnName
+			}
+
+			indexInfoList = append(indexInfoList, models.FenWeiNetIndexInfo{
+				IndexName:    productName + "/汾渭",
+				Unit:         data.MonthlyValueChnUnit,
+				Frequency:    frequency,
+				TerminalCode: utils.TerminalCode,
+				ClassifyName: productName,
+				DataTime:     data.DataDate,
+				Value:        data.MonthlyValue,
+			})
+		}
 	}
 
 	return indexInfoList, nil
@@ -389,29 +396,36 @@ func (p *ThermalCoalInventoryProductionProcessor) GenerateRequestParams(currentT
 }
 
 func (p *ThermalCoalInventoryProductionProcessor) ProcessResponse(data string) ([]models.FenWeiNetIndexInfo, error) {
-	responseData := fillFenWeiNetResponseData(data)
+	responseDataMap := fillFenWeiNetResponseMapData(data)
 
 	var indexInfoList []models.FenWeiNetIndexInfo
 	productName := "动力煤库存-生产企业"
-	for _, data := range responseData {
-		var frequency string
-		if data.MonthlyValueChnName != "" && strings.Contains(data.MonthlyValueChnName, "月度") {
-			frequency = "月度"
-		} else if data.WeekValueChnName != "" && strings.Contains(data.WeekValueChnName, "月度") {
-			frequency = "周度"
-		} else {
-			frequency = data.MonthlyValueChnName
+
+	for productCategoryName, responseDataList := range responseDataMap {
+		if productCategoryName != productName {
+			continue
 		}
 
-		indexInfoList = append(indexInfoList, models.FenWeiNetIndexInfo{
-			IndexName:    productName + "/汾渭",
-			Unit:         data.MonthlyValueChnUnit,
-			Frequency:    frequency,
-			TerminalCode: utils.TerminalCode,
-			ClassifyName: productName,
-			DataTime:     data.DataDate,
-			Value:        data.MonthlyValue,
-		})
+		for _, data := range responseDataList {
+			var frequency string
+			if data.MonthlyValueChnName != "" && strings.Contains(data.MonthlyValueChnName, "月度") {
+				frequency = "月度"
+			} else if data.WeekValueChnName != "" && strings.Contains(data.WeekValueChnName, "月度") {
+				frequency = "周度"
+			} else {
+				frequency = data.MonthlyValueChnName
+			}
+
+			indexInfoList = append(indexInfoList, models.FenWeiNetIndexInfo{
+				IndexName:    productName + "/汾渭",
+				Unit:         data.MonthlyValueChnUnit,
+				Frequency:    frequency,
+				TerminalCode: utils.TerminalCode,
+				ClassifyName: productName,
+				DataTime:     data.DataDate,
+				Value:        data.MonthlyValue,
+			})
+		}
 	}
 
 	return indexInfoList, nil
@@ -446,52 +460,56 @@ func (p *ThermalCoalInventorySixPowerPlantProcessor) GenerateRequestParams(curre
 }
 
 func (p *ThermalCoalInventorySixPowerPlantProcessor) ProcessResponse(data string) ([]models.FenWeiNetIndexInfo, error) {
-	responseData := fillFenWeiNetResponseData(data)
+	responseDataMap := fillFenWeiNetResponseMapData(data)
 
 	var indexInfoList []models.FenWeiNetIndexInfo
 	productName := "动力煤库存"
-	for _, data := range responseData {
-		var frequency string
-		if data.MonthlyValueChnName != "" && strings.Contains(data.MonthlyValueChnName, "月度") {
-			frequency = "月度"
-		} else if data.WeekValueChnName != "" && strings.Contains(data.WeekValueChnName, "月度") {
-			frequency = "周度"
-		} else {
-			frequency = "日度"
-		}
 
-		classifyName := "动力煤库存-六大电厂"
+	for productCategoryName, responseDataList := range responseDataMap {
 
-		// 库存
-		indexInfoList = append(indexInfoList, models.FenWeiNetIndexInfo{
-			IndexName:    productName + data.ProductItemName + data.StockChnName + "/汾渭",
-			Unit:         data.StockChnUnit,
-			Frequency:    frequency,
-			TerminalCode: utils.TerminalCode,
-			ClassifyName: classifyName,
-			DataTime:     data.DataDate,
-			Value:        data.Stock,
-		})
-		// 可用天数
-		indexInfoList = append(indexInfoList, models.FenWeiNetIndexInfo{
-			IndexName:    productName + data.ProductItemName + data.AvaliableDaysChnName + "/汾渭",
-			Unit:         data.AvaliableDaysChnUnit,
-			Frequency:    frequency,
-			TerminalCode: utils.TerminalCode,
-			ClassifyName: classifyName,
-			DataTime:     data.DataDate,
-			Value:        data.AvaliableDays,
-		})
-		// 日耗
-		indexInfoList = append(indexInfoList, models.FenWeiNetIndexInfo{
-			IndexName:    productName + data.ProductItemName + data.DailyConsumptionChnName + "/汾渭",
-			Unit:         data.DailyConsumptionChnUnit,
-			Frequency:    frequency,
-			TerminalCode: utils.TerminalCode,
-			ClassifyName: classifyName,
-			DataTime:     data.DataDate,
-			Value:        data.DailyConsumption,
-		})
+		for _, data := range responseDataList {
+			var frequency string
+			if data.MonthlyValueChnName != "" && strings.Contains(data.MonthlyValueChnName, "月度") {
+				frequency = "月度"
+			} else if data.WeekValueChnName != "" && strings.Contains(data.WeekValueChnName, "月度") {
+				frequency = "周度"
+			} else {
+				frequency = "日度"
+			}
+
+			classifyName := "动力煤库存-六大电厂"
+
+			// 库存
+			indexInfoList = append(indexInfoList, models.FenWeiNetIndexInfo{
+				IndexName:    productName + productCategoryName + data.StockChnName + "/汾渭",
+				Unit:         data.StockChnUnit,
+				Frequency:    frequency,
+				TerminalCode: utils.TerminalCode,
+				ClassifyName: classifyName,
+				DataTime:     data.DataDate,
+				Value:        data.Stock,
+			})
+			// 可用天数
+			indexInfoList = append(indexInfoList, models.FenWeiNetIndexInfo{
+				IndexName:    productName + productCategoryName + data.AvaliableDaysChnName + "/汾渭",
+				Unit:         data.AvaliableDaysChnUnit,
+				Frequency:    frequency,
+				TerminalCode: utils.TerminalCode,
+				ClassifyName: classifyName,
+				DataTime:     data.DataDate,
+				Value:        data.AvaliableDays,
+			})
+			// 日耗
+			indexInfoList = append(indexInfoList, models.FenWeiNetIndexInfo{
+				IndexName:    productName + productCategoryName + data.DailyConsumptionChnName + "/汾渭",
+				Unit:         data.DailyConsumptionChnUnit,
+				Frequency:    frequency,
+				TerminalCode: utils.TerminalCode,
+				ClassifyName: classifyName,
+				DataTime:     data.DataDate,
+				Value:        data.DailyConsumption,
+			})
+		}
 	}
 
 	return indexInfoList, nil
@@ -526,30 +544,35 @@ func (p *CokingCoalInventorySocietyProcessor) GenerateRequestParams(currentTime
 }
 
 func (p *CokingCoalInventorySocietyProcessor) ProcessResponse(data string) ([]models.FenWeiNetIndexInfo, error) {
-	responseData := fillFenWeiNetResponseData(data)
+	responseDataMap := fillFenWeiNetResponseMapData(data)
 
 	var indexInfoList []models.FenWeiNetIndexInfo
 	productName := "炼焦煤库存-全社会"
 
-	for _, data := range responseData {
-		var frequency string
-		if data.MonthlyValueChnName != "" && strings.Contains(data.MonthlyValueChnName, "月度") {
-			frequency = "月度"
-		} else if data.WeekValueChnName != "" && strings.Contains(data.WeekValueChnName, "月度") {
-			frequency = "周度"
-		} else {
-			frequency = data.MonthlyValueChnName
+	for productCategoryName, responseDataList := range responseDataMap {
+		if productCategoryName == "合计" {
+			continue
 		}
+		for _, data := range responseDataList {
+			var frequency string
+			if data.MonthlyValueChnName != "" && strings.Contains(data.MonthlyValueChnName, "月度") {
+				frequency = "月度"
+			} else if data.WeekValueChnName != "" && strings.Contains(data.WeekValueChnName, "月度") {
+				frequency = "周度"
+			} else {
+				frequency = data.MonthlyValueChnName
+			}
 
-		indexInfoList = append(indexInfoList, models.FenWeiNetIndexInfo{
-			IndexName:    data.ProductItemName + "/汾渭",
-			Unit:         data.MonthlyValueChnUnit,
-			Frequency:    frequency,
-			TerminalCode: utils.TerminalCode,
-			ClassifyName: productName,
-			DataTime:     data.DataDate,
-			Value:        data.MonthlyValue,
-		})
+			indexInfoList = append(indexInfoList, models.FenWeiNetIndexInfo{
+				IndexName:    productCategoryName + "/汾渭",
+				Unit:         data.MonthlyValueChnUnit,
+				Frequency:    frequency,
+				TerminalCode: utils.TerminalCode,
+				ClassifyName: productName,
+				DataTime:     data.DataDate,
+				Value:        data.MonthlyValue,
+			})
+		}
 	}
 
 	return indexInfoList, nil
@@ -584,30 +607,35 @@ func (p *CokingCoalInventoryProductionProcessor) GenerateRequestParams(currentTi
 }
 
 func (p *CokingCoalInventoryProductionProcessor) ProcessResponse(data string) ([]models.FenWeiNetIndexInfo, error) {
-	responseData := fillFenWeiNetResponseData(data)
+	responseDataMap := fillFenWeiNetResponseMapData(data)
 
 	var indexInfoList []models.FenWeiNetIndexInfo
 	productName := "炼焦煤库存-生产企业"
 
-	for _, data := range responseData {
-		var frequency string
-		if data.MonthlyValueChnName != "" && strings.Contains(data.MonthlyValueChnName, "月度") {
-			frequency = "月度"
-		} else if data.WeekValueChnName != "" && strings.Contains(data.WeekValueChnName, "月度") {
-			frequency = "周度"
-		} else {
-			frequency = data.MonthlyValueChnName
+	for productCategoryName, responseDataList := range responseDataMap {
+		if productCategoryName == "合计" {
+			continue
 		}
+		for _, data := range responseDataList {
+			var frequency string
+			if data.MonthlyValueChnName != "" && strings.Contains(data.MonthlyValueChnName, "月度") {
+				frequency = "月度"
+			} else if data.WeekValueChnName != "" && strings.Contains(data.WeekValueChnName, "月度") {
+				frequency = "周度"
+			} else {
+				frequency = data.MonthlyValueChnName
+			}
 
-		indexInfoList = append(indexInfoList, models.FenWeiNetIndexInfo{
-			IndexName:    data.ProductItemName + "/汾渭",
-			Unit:         data.MonthlyValueChnUnit,
-			Frequency:    frequency,
-			TerminalCode: utils.TerminalCode,
-			ClassifyName: productName,
-			DataTime:     data.DataDate,
-			Value:        data.WeekValue,
-		})
+			indexInfoList = append(indexInfoList, models.FenWeiNetIndexInfo{
+				IndexName:    productCategoryName + "/汾渭",
+				Unit:         data.MonthlyValueChnUnit,
+				Frequency:    frequency,
+				TerminalCode: utils.TerminalCode,
+				ClassifyName: productName,
+				DataTime:     data.DataDate,
+				Value:        data.WeekValue,
+			})
+		}
 	}
 
 	return indexInfoList, nil
@@ -642,23 +670,28 @@ func (p *CokingCoalInventoryDownstreamProcessor) GenerateRequestParams(currentTi
 }
 
 func (p *CokingCoalInventoryDownstreamProcessor) ProcessResponse(data string) ([]models.FenWeiNetIndexInfo, error) {
-	responseData := fillFenWeiNetResponseData(data)
+	responseDataMap := fillFenWeiNetResponseMapData(data)
 
 	var indexInfoList []models.FenWeiNetIndexInfo
 	productName := "炼焦煤库存-下游企业"
 
-	for _, data := range responseData {
-		frequency := "周度"
+	for productCategoryName, responseDataList := range responseDataMap {
+		if productCategoryName == "合计" {
+			continue
+		}
+		for _, data := range responseDataList {
+			frequency := "周度"
 
-		indexInfoList = append(indexInfoList, models.FenWeiNetIndexInfo{
-			IndexName:    data.ProductItemName + "/汾渭",
-			Unit:         data.StockChnUnit,
-			Frequency:    frequency,
-			TerminalCode: utils.TerminalCode,
-			ClassifyName: productName,
-			DataTime:     data.DataDate,
-			Value:        data.Stock,
-		})
+			indexInfoList = append(indexInfoList, models.FenWeiNetIndexInfo{
+				IndexName:    productCategoryName + "/汾渭",
+				Unit:         data.StockChnUnit,
+				Frequency:    frequency,
+				TerminalCode: utils.TerminalCode,
+				ClassifyName: productName,
+				DataTime:     data.DataDate,
+				Value:        data.Stock,
+			})
+		}
 	}
 
 	return indexInfoList, nil
@@ -677,7 +710,6 @@ func (p *NationalCoalMineInventoryProcessor) GenerateRequestParams(currentTime s
 		ProductCode:  "FW8001D",
 		QuotaName:    "monthly_value",
 		SplitTypeKey: "",
-		IsTotal:      0,
 		DataType:     nil,
 		IsSeason:     1,
 	}
@@ -736,7 +768,7 @@ func (p *StateOwnedKeyCoalMineInventoryProcessor) GenerateRequestParams(currentT
 		ProductCode:  "FW8003D",
 		QuotaName:    "monthly_value",
 		SplitTypeKey: "province",
-		IsTotal:      0,
+		IsTotal:      "",
 		DataType:     nil,
 		IsSeason:     1,
 	}
@@ -873,7 +905,7 @@ func (p *PortDataInventoryNorthernPortProcessor) GenerateRequestParams(currentTi
 		DateRange:    "20190820-" + currentTime,
 		ProductCode:  "FW1203D",
 		QuotaName:    "stock",
-		SplitTypeKey: "",
+		SplitTypeKey: "product_item_code",
 		IsTotal:      0,
 		DataType:     nil,
 		Type:         2,
@@ -907,7 +939,7 @@ func (p *PortDataInventoryNorthernPortProcessor) ProcessResponse(data string) ([
 
 		indexInfoList = append(indexInfoList, models.FenWeiNetIndexInfo{
 			IndexName:    productName + data.ProductItemName + "/汾渭",
-			Unit:         data.StockChnName,
+			Unit:         data.StockChnUnit,
 			Frequency:    frequency,
 			TerminalCode: utils.TerminalCode,
 			ClassifyName: productName,
@@ -960,7 +992,7 @@ func (p *PortDataInventoryInlandPortProcessor) ProcessResponse(data string) ([]m
 
 		indexInfoList = append(indexInfoList, models.FenWeiNetIndexInfo{
 			IndexName:    productName + data.ProductItemName + "/汾渭",
-			Unit:         data.StockChnName,
+			Unit:         data.StockChnUnit,
 			Frequency:    frequency,
 			TerminalCode: utils.TerminalCode,
 			ClassifyName: productName,
@@ -1011,10 +1043,18 @@ func (p *PortDataDispatchNorthernPortProcessor) ProcessResponse(data string) ([]
 
 	for _, data := range responseData {
 		var frequency = "日度"
+		var unit string
+		if data.VehicleChnUnit != "" {
+			unit = data.VehicleChnUnit
+		} else if data.TonsChnUnit != "" {
+			unit = data.TonsChnUnit
+		} else {
+			unit = data.ShipChnUnit
+		}
 
 		indexInfoList = append(indexInfoList, models.FenWeiNetIndexInfo{
-			IndexName:    productName + "-" + data.ProductItemName + "-" + data.PortName + "/汾渭",
-			Unit:         data.TonsChnUnit,
+			IndexName:    productName + "-" + data.PortName + data.ProductItemName + "/汾渭",
+			Unit:         unit,
 			Frequency:    frequency,
 			TerminalCode: utils.TerminalCode,
 			ClassifyName: productName,
@@ -1158,7 +1198,7 @@ func (p *ThermalCoalPortPriceProcessor) GenerateRequestParams(currentTime string
 }
 
 func (p *ThermalCoalPortPriceProcessor) ProcessResponse(data string) ([]models.FenWeiNetIndexInfo, error) {
-	responseDataMap := fillFenWeiNetResponseMapData(data)
+	responseDataMap := fillFenWeiNetResponseAStratumMapData(data)
 
 	var indexInfoList []models.FenWeiNetIndexInfo
 	productName := "动力煤港口价格"
@@ -1326,3 +1366,13 @@ func fillFenWeiNetResponseMapData(data string) map[string][]models.FenWeiNetResp
 	responseData := result.Data.Data
 	return responseData
 }
+
+func fillFenWeiNetResponseAStratumMapData(data string) map[string][]models.FenWeiNetResponse {
+	var result models.FenWeiNetResponseAStratumMapWrapper
+	err := json.Unmarshal([]byte(data), &result)
+	if err != nil {
+		return nil
+	}
+	responseData := result.Data
+	return responseData
+}