数据爬取

services/ruizide/data_processor.go

@@ -272,8 +272,8 @@ func resolverNet() {
 // func fileResolver() {
 func main() {
 	var tableNameList = []string{
-		//"Oil_Demand_Signals_Weekly_Report",
-		"RE_Dashboard_Export",
+		"Oil_Demand_Signals_Weekly_Report",
+		"Supply_Revision_Analysis",
 	}
 	for _, tableName := range tableNameList {
 		var fileName string
@@ -291,11 +291,30 @@ func main() {
 		sheetNames := f.GetSheetList()
 		for _, sheetName := range sheetNames {
 			fmt.Printf("读取工作表: %s\n", sheetName)
+			if strings.Contains(sheetName, "Content") {
+				continue
+			}
+			if strings.Contains(sheetName, "Road Index") {
+				continue
+			}
+			if strings.Contains(sheetName, "Road Active Fleet") {
+				continue
+			}
+			if strings.Contains(sheetName, "Aviation Index") {
+				continue
+			}
+			if strings.Contains(sheetName, "Aviation Active Fleet") {
+				continue
+			}
+			if strings.Contains(sheetName, "Demand - Gasoline") {
+				continue
+			}
 
 			// 获取工作表的最大行数
 			maxRow, err := f.GetRows(sheetName) // 直接获取所有行数据
 			if err != nil {
 				log.Fatalf("获取工作表数据时出错: %v", err)
+				continue
 			}
 
 			// 遍历行并打印内容

services/ruizide/processor_factory.go

@@ -14,13 +14,106 @@ func GetProcessor(tableName string, sheetName string) (ReportProcessor, error) {
 		switch sheetName {
 		case "Road Index":
 			return &RoadIndexProcessor{}, nil
+		case "Road Active Fleet":
+			return &RoadActiveFleetProcessor{}, nil
+		case "Aviation Index":
+			return &AviationIndexProcessor{}, nil
+		case "Aviation Active Fleet":
+			return &AviationActiveFleetProcessor{}, nil
+		case "Demand - Gasoline":
+			return &DemandGasolineProcessor{}, nil
+		case "Demand - Diesel":
+			return &DemandDieselProcessor{}, nil
+		case "Demand - Jet Fuel":
+			return &DemandJetFuelProcessor{}, nil
+		case "Demand - Maritime Bunker":
+			return &DemandMaritimeBunkerProcessor{}, nil
 		default:
 			return nil, fmt.Errorf("unknown sheetName: %s", sheetName)
 		}
-	} else if tableName == "RE_Dashboard_Export" {
+	} else if tableName == "Supply_Revision_Analysis" {
 		switch sheetName {
 		case "Chart1":
-			return &ReDashboardExportOneProcessor{}, nil
+			return &SupplyRevisionAnalysisChartOneProcessor{}, nil
+		case "Chart2":
+			return &SupplyRevisionAnalysisChartTwoProcessor{}, nil
+		case "Chart3":
+			return &SupplyRevisionAnalysisChartThreeProcessor{}, nil
+		case "Chart4":
+			return &SupplyRevisionAnalysisChartFourProcessor{}, nil
+		case "Chart5":
+			return &SupplyRevisionAnalysisChartFiveProcessor{}, nil
+		case "Chart6":
+			return &SupplyRevisionAnalysisChartSixProcessor{}, nil
+		default:
+			return nil, fmt.Errorf("unknown sheetName: %s", sheetName)
+		}
+	} else if tableName == "Oil_Supply_Analysis" {
+		switch sheetName {
+		case "Chart1":
+			return &OilSupplyAnalysisChartOneProcessor{}, nil
+		case "Chart2":
+			return &OilSupplyAnalysisChartTwoProcessor{}, nil
+		case "Chart3":
+			return &OilSupplyAnalysisChartThreeProcessor{}, nil
+		case "Chart4":
+			return &OilSupplyAnalysisChartFourProcessor{}, nil
+		default:
+			return nil, fmt.Errorf("unknown sheetName: %s", sheetName)
+		}
+	} else if tableName == "Oil_Demand_Analysis_Continent" {
+		switch sheetName {
+		case "Chart1":
+			return &OilSupplyAnalysisChartOneProcessor{}, nil
+		default:
+			return nil, fmt.Errorf("unknown sheetName: %s", sheetName)
+		}
+	} else if tableName == "Oil_Demand_Analysis_Region" {
+		switch sheetName {
+		case "Chart1":
+			return &OilSupplyAnalysisChartOneProcessor{}, nil
+		default:
+			return nil, fmt.Errorf("unknown sheetName: %s", sheetName)
+		}
+	} else if tableName == "Oil_Demand_Analysis_Country" {
+		switch sheetName {
+		case "Chart1":
+			return &OilSupplyAnalysisChartOneProcessor{}, nil
+		default:
+			return nil, fmt.Errorf("unknown sheetName: %s", sheetName)
+		}
+	} else if tableName == "Oil_Demand_Analysis_Product_Category" {
+		switch sheetName {
+		case "Chart1":
+			return &OilSupplyAnalysisChartOneProcessor{}, nil
+		default:
+			return nil, fmt.Errorf("unknown sheetName: %s", sheetName)
+		}
+	} else if tableName == "Oil_Demand_Analysis_Product_Detail" {
+		switch sheetName {
+		case "Chart1":
+			return &OilSupplyAnalysisChartOneProcessor{}, nil
+		default:
+			return nil, fmt.Errorf("unknown sheetName: %s", sheetName)
+		}
+	} else if tableName == "Oil_Demand_Analysis_Sector_Category" {
+		switch sheetName {
+		case "Chart1":
+			return &OilSupplyAnalysisChartOneProcessor{}, nil
+		default:
+			return nil, fmt.Errorf("unknown sheetName: %s", sheetName)
+		}
+	} else if tableName == "Oil_Demand_Analysis_Sector_Detail" {
+		switch sheetName {
+		case "Chart1":
+			return &OilSupplyAnalysisChartOneProcessor{}, nil
+		default:
+			return nil, fmt.Errorf("unknown sheetName: %s", sheetName)
+		}
+	} else if tableName == "Oil_Demand_Analysis_Scenario" {
+		switch sheetName {
+		case "Chart1":
+			return &OilSupplyAnalysisChartOneProcessor{}, nil
 		default:
 			return nil, fmt.Errorf("unknown sheetName: %s", sheetName)
 		}

utils/date_util.go

@@ -550,3 +550,62 @@ func ConvertDateFormat2(dataText string) (string, error) {
 
 	return endOfQuarter, nil
 }
+
+// ConvertDateFormat3 转换时间格式 Jan-23 --> 2023-01-31
+func ConvertDateFormat3(dataText string) (string, error) {
+	// 拆分月份和年份
+	parts := strings.Split(dataText, "-")
+	if len(parts) != 2 {
+		return "", fmt.Errorf("日期格式不正确")
+	}
+
+	// 提取月份和年份
+	monthStr := parts[0]
+	yearStr := parts[1]
+
+	// 转换年份
+	year, err := strconv.Atoi(yearStr)
+	if err != nil {
+		return "", fmt.Errorf("无效的年份: %v", err)
+	}
+
+	// 获取月份数字
+	month, err := time.Parse("Jan", monthStr)
+	if err != nil {
+		return "", fmt.Errorf("无效的月份: %v", err)
+	}
+
+	// 计算最后一天
+	lastDay := time.Date(year+2000, month.Month(), 0, 0, 0, 0, 0, time.UTC) // year+2000 将年份转为四位数
+	lastDay = lastDay.AddDate(0, 1, 0)                                      // 加一个月
+	lastDay = lastDay.AddDate(0, 0, -1)                                     // 减去一天，得到当前月份的最后一天
+
+	return lastDay.Format("2006-01-02"), nil
+}
+
+// ConvertDateFormat4 转换时间格式 2022 --> 2022-12-31
+func ConvertDateFormat4(dataText string) (string, error) {
+	// 解析输入年份
+	year, err := strconv.Atoi(dataText)
+	if err != nil {
+		return "", fmt.Errorf("无法解析年份: %v", err)
+	}
+
+	// 创建该年最后一天的日期
+	lastDay := time.Date(year, time.December, 31, 0, 0, 0, 0, time.UTC)
+
+	// 返回格式化后的日期字符串
+	return lastDay.Format("2006-01-02"), nil
+}
+
+// ConvertDateFormat5 转换时间格式 yyyy-MM-dd HH:mm:ss --> yyyy-MM-dd
+func ConvertDateFormat5(dataText string) (string, error) {
+	// 解析输入的日期和时间
+	dateTime, err := time.Parse("2006-01-02 15:04:05", dataText)
+	if err != nil {
+		return "", err
+	}
+
+	// 格式化为输出格式
+	return dateTime.Format("2006-01-02"), nil
+}