eta_crawler/services/liangyou/processor_business_logic.go

// Package liangyou
// @Author gmy 2024/8/6 10:50:00
package liangyou

import (
	"context"
	"eta/eta_crawler/models"
	"eta/eta_crawler/utils"
	"fmt"
	"github.com/PuerkitoBio/goquery"
	"github.com/beego/beego/v2/core/logs"
	"github.com/chromedp/chromedp"
	"log"
	"regexp"
	"strconv"
	"strings"
	"unicode"
)

const (
	lySourceName = "lysww" // 粮油商务网
)

// TableData 用于存储表格的数据
type TableData struct {
	Headers []string   `json:"headers"`
	Rows    [][]string `json:"rows"`
}

// ImportCostProcessor
// @Description: 进口成本处理器
type ImportCostProcessor struct{}

func (p *ImportCostProcessor) Process(ctx context.Context, product string, reportContent string, keywords []string, classifyId int) ([]models.BaseFromLyData, error) {
	logs.Info("Processing import cost...")

	// 解析关键字
	if len(keywords) < 5 {
		return []models.BaseFromLyData{}, fmt.Errorf("ImportCostProcessor Process() : keywords must contain at least 5 elements")
	}

	// 拿到 行关键字和列关键字
	columnName := keywords[len(keywords)-4]
	rowVariety := keywords[0]
	rowPort := keywords[len(keywords)-3]
	indexNamePrefix := keywords[:1]
	indexNameSuffix := keywords[1:]

	// 提取所有表格数据
	tableData := getNoHeadTableData(reportContent)

	// 提取日期信息
	dateText, err := getDateInfo(ctx)
	if err != nil {
		return []models.BaseFromLyData{}, err
	}

	// 时间格式转换
	format, err := utils.ConvertTimeFormat(dateText)
	if err != nil {
		return []models.BaseFromLyData{}, err
	}

	// 解析日期并计算当前月份
	var targetMonths []string
	if product == "油菜籽" {
		targetMonths, err = utils.ParseDateAndMonthColzaOil(format)
	} else {
		targetMonths, err = utils.ParseDateAndMonth(dateText)
	}
	if err != nil {
		return []models.BaseFromLyData{}, fmt.Errorf("ImportCostProcessor Process() : Failed to parse date: %v", err)
	}
	fmt.Printf("Target Month: %s\n", targetMonths)

	// 处理提取的表格数据
	var result []models.BaseFromLyData

	for _, data := range tableData {
		tableHeaders := data.Headers
		tableRows := data.Rows

		// 查找目标列
		columnIdx := -1
		for i, header := range tableHeaders {
			if strings.Contains(header, columnName) {
				columnIdx = i
				break
			}
		}
		if columnIdx == -1 {
			log.Printf("ImportCostProcessor Process() : Column '%s' not found in table", columnName)
			continue
		}

		// 处理表格中的每一行
		//var flag bool = true
		var previousRowVariety string
		var previousRowPort string
		for rowIndex, row := range tableRows {
			if len(row) == len(tableHeaders) {
				previousRowVariety = row[0]
				previousRowPort = row[1]
			} else if len(row) == len(tableHeaders)-1 {
				previousRowPort = row[0]
				row = append([]string{previousRowVariety}, row...)
				tableRows[rowIndex] = row
			} else if len(row) == len(tableHeaders)-2 {
				row = append([]string{previousRowVariety, previousRowPort}, row...)
				tableRows[rowIndex] = row
			}
			for _, targetMonth := range targetMonths {
				if len(row) >= len(tableHeaders) && strings.Contains(rowVariety, row[0]) && row[1] == targetMonth && row[len(row)-1] == rowPort {
					if columnIdx < len(row) {
						// 指标名称
						indexNameList := append(indexNamePrefix, append([]string{targetMonth}, indexNameSuffix...)...)
						indexName := strings.Join(indexNameList[:len(keywords)-2], "：")
						// 指标编码
						indexCode := utils.GenerateIndexCode(lySourceName, indexName)
						// 指标id获取
						indexId, err := getIndexId(indexCode, indexName, classifyId, lySourceName, keywords[len(keywords)-2], keywords[len(keywords)-1])
						if err != nil {
							logs.Error("ImportCostProcessor Process() : Failed to get index id: %v", err)
							continue
						}

						indexData, err := models.GetLyDataByIndexIdAndDataTime(indexId, format)
						if err != nil {
							logs.Error("ImportCostProcessor Process() : Failed to get data by index id and date: %v", err)
							continue
						}
						if len(indexData) > 0 {
							logs.Info("ImportCostProcessor Process() : Data already exists for index %d and date %s", indexId, dateText)
							continue
						}

						valueStr := row[columnIdx]
						value, err := strconv.ParseFloat(valueStr, 64)
						if err != nil {
							return []models.BaseFromLyData{}, fmt.Errorf("failed to parse value '%s': %v", valueStr, err)
						}
						// 创建并添加到结果列表
						baseFromLyData := models.BaseFromLyData{
							DataTime:          format,
							Value:             value,
							BaseFromLyIndexId: indexId,
							IndexCode:         indexCode,
						}
						result = append(result, baseFromLyData)
					} else {
						log.Printf("ImportCostProcessor Process() : Column index out of range for row '%s', '%s'", rowVariety, rowPort)
					}
					break
				}
			}

		}

	}

	return result, nil
}

// ProcessingProfitProcessor
// @Description: 加工利润处理器
type ProcessingProfitProcessor struct{}

func (p *ProcessingProfitProcessor) Process(ctx context.Context, product string, reportContent string, keywords []string, classifyId int) ([]models.BaseFromLyData, error) {
	fmt.Println("Processing processing profit...")
	// 解析关键字
	if len(keywords) < 4 {
		return []models.BaseFromLyData{}, fmt.Errorf("ProcessingProfitProcessor Process() : keywords must contain at least 4 elements")
	}

	// 拿到 行关键字和列关键字
	columnName := keywords[1]
	rowVariety := keywords[0]
	indexNamePrefix := keywords[:1]
	indexNameSuffix := keywords[1:]

	// 提取所有表格数据
	tableData := getNoHeadTableData(reportContent)

	// 提取日期信息
	dateText, err := getDateInfo(ctx)
	if err != nil {
		return []models.BaseFromLyData{}, err
	}

	// 时间格式转换
	format, err := utils.ConvertTimeFormat(dateText)
	if err != nil {
		return []models.BaseFromLyData{}, err
	}

	// 解析日期并计算当前月份 和 后两月
	yearMonths, err := utils.ConvertTimeFormatToYearMonth(format)
	if err != nil {
		return nil, err
	}
	fmt.Printf("Target yearMonth: %s\n", yearMonths)

	// 处理提取的表格数据
	var result []models.BaseFromLyData

	for _, data := range tableData {
		tableHeaders := data.Headers
		tableRows := data.Rows

		// 查找目标列
		columnIdx := -1
		for i, header := range tableHeaders {
			if strings.Contains(columnName, header) {
				columnIdx = i
				break
			}
		}
		if columnIdx == -1 {
			log.Printf("ProcessingProfitProcessor Process() : Column '%s' not found in table", columnName)
			continue
		}

		// 处理表格中的每一行
		var previousRowVariety string
		for rowIndex, row := range tableRows {
			if len(row) == len(tableHeaders) {
				previousRowVariety = row[0]
			} else if len(row) == len(tableHeaders)-1 {
				row = append([]string{previousRowVariety}, row...)
				tableRows[rowIndex] = row
			}

			for _, targetMonth := range yearMonths {
				if len(row) >= len(tableHeaders) && row[0] == rowVariety && row[1] == targetMonth {
					if columnIdx < len(row) {
						// 指标名称
						indexNameList := append(indexNamePrefix, append([]string{targetMonth}, indexNameSuffix...)...)
						indexName := strings.Join(indexNameList[:len(keywords)-1], "：")
						// 指标编码
						indexCode := utils.GenerateIndexCode(lySourceName, indexName)
						// 指标id获取
						indexId, err := getIndexId(indexCode, indexName, classifyId, lySourceName, keywords[len(keywords)-2], keywords[len(keywords)-1])
						if err != nil {
							logs.Error("ProcessingProfitProcessor Process() : Failed to get index id: %v", err)
							continue
						}

						indexData, err := models.GetLyDataByIndexIdAndDataTime(indexId, format)
						if err != nil {
							logs.Error("ProcessingProfitProcessor Process() : Failed to get data by index id and date: %v", err)
							continue
						}
						if len(indexData) > 0 {
							logs.Info("ProcessingProfitProcessor Process() : Data already exists for index %d and date %s", indexId, dateText)
							continue
						}

						valueStr := row[columnIdx]
						value, err := strconv.ParseFloat(valueStr, 64)
						if err != nil {
							return []models.BaseFromLyData{}, fmt.Errorf("failed to parse value '%s': %v", valueStr, err)
						}
						// 创建并添加到结果列表
						baseFromLyData := models.BaseFromLyData{
							DataTime:          format,
							Value:             value,
							BaseFromLyIndexId: indexId,
							IndexCode:         indexCode,
						}
						result = append(result, baseFromLyData)
					} else {
						log.Printf("ProcessingProfitProcessor Process() : Column index out of range for row '%s', '%s'", rowVariety, columnName)
					}
					break
				}
			}

		}
	}

	return result, nil
}

// ShippingCostProcessor
// @Description: 船运费用处理器
type ShippingCostProcessor struct{}

func (p *ShippingCostProcessor) Process(ctx context.Context, product string, reportContent string, keywords []string, classifyId int) ([]models.BaseFromLyData, error) {
	fmt.Println("Processing processing profit...")
	// 解析关键字
	if len(keywords) < 4 {
		return []models.BaseFromLyData{}, fmt.Errorf("ShippingCostProcessor Process() : keywords must contain at least 5 elements")
	}

	// 拿到 行关键字和列关键字
	columnName := keywords[len(keywords)-3]
	rowVariety := keywords[0]
	rowDestination := keywords[1]
	rowShipType := keywords[2]

	// 提取所有表格数据
	tableData := getNoHeadTableData(reportContent)

	// 提取日期信息
	dateText, err := getDateInfo(ctx)
	if err != nil {
		return []models.BaseFromLyData{}, err
	}

	// 时间格式转换
	format, err := utils.ConvertTimeFormat(dateText)
	if err != nil {
		return []models.BaseFromLyData{}, err
	}

	// 处理提取的表格数据
	var result []models.BaseFromLyData

	for _, data := range tableData {
		tableHeaders := data.Headers
		tableRows := data.Rows

		// 查找目标列
		columnIdx := -1
		for i, header := range tableHeaders {
			if strings.Contains(header, columnName) {
				columnIdx = i
				break
			}
		}
		if columnIdx == -1 {
			log.Printf("ShippingCostProcessor Process() : Column '%s' not found in table", columnName)
			continue
		}

		// 处理表格中的每一行
		for rowIndex, row := range tableRows {
			if len(row) == len(tableHeaders)-1 {
				row = append([]string{rowVariety}, row...)
				tableRows[rowIndex] = row
				rowShipType, err = extractValueInParentheses(rowVariety)
				if err != nil {
					logs.Error("ShippingCostProcessor Process() : Failed to extract value in parentheses: %v", err)
					continue
				}

			}
			if len(row) >= len(tableHeaders) && row[0] == rowVariety && (row[1] == rowDestination || strings.Contains(row[0], row[1])) && row[2] == rowShipType {
				if columnIdx < len(row) {
					// 指标名称
					indexName := strings.Join(keywords[:len(keywords)-3], `：`)
					// 指标编码
					indexCode := utils.GenerateIndexCode(lySourceName, indexName)
					// 指标id获取
					indexId, err := getIndexId(indexCode, indexName, classifyId, lySourceName, keywords[len(keywords)-2], keywords[len(keywords)-1])
					if err != nil {
						logs.Error("ShippingCostProcessor Process() : Failed to get index id: %v", err)
						continue
					}

					indexData, err := models.GetLyDataByIndexIdAndDataTime(indexId, format)
					if err != nil {
						logs.Error("ShippingCostProcessor Process() : Failed to get data by index id and date: %v", err)
						continue
					}
					if len(indexData) > 0 {
						logs.Info("ShippingCostProcessor Process() : Data already exists for index %d and date %s", indexId, dateText)
						continue
					}

					valueStr := row[columnIdx]
					value, err := strconv.ParseFloat(valueStr, 64)
					if err != nil {
						return []models.BaseFromLyData{}, fmt.Errorf("failed to parse value '%s': %v", valueStr, err)
					}
					// 创建并添加到结果列表
					baseFromLyData := models.BaseFromLyData{
						DataTime:          format,
						Value:             value,
						BaseFromLyIndexId: indexId,
						IndexCode:         indexCode,
					}
					result = append(result, baseFromLyData)
				} else {
					log.Printf("ShippingCostProcessor Process() : Column index out of range for row '%s', '%s'", rowVariety, columnName)
				}
				break
			}
		}
	}

	return result, nil
}

// SupplyDemandBalanceProcessor
// @Description: 供需平衡处理器
type SupplyDemandBalanceProcessor struct{}

func (p *SupplyDemandBalanceProcessor) Process(ctx context.Context, product string, reportContent string, keywords []string, classifyId int) ([]models.BaseFromLyData, error) {
	// https://www.fao.com.cn/art/gG7gKTCNDHLJNsq9QRYjoQ==.htm
	logs.Info("Processing processing report...")
	// 解析关键字
	if len(keywords) < 4 {
		return []models.BaseFromLyData{}, fmt.Errorf("SupplyDemandBalanceProcessor Process() : keywords must contain at least 4 elements")
	}

	// 拿到 行关键字和列关键字
	var columnName string
	rowVariety := keywords[1]

	// 提取所有表格数据
	tableData := getTableData(reportContent, true)
	logs.Info("SupplyDemandBalanceProcessor Process() : Table data: %v", tableData)

	// 提取日期信息
	dateText, err := getDateInfo(ctx)
	if err != nil {
		return []models.BaseFromLyData{}, err
	}

	// 时间格式转换
	format, err := utils.ConvertTimeFormat(dateText)
	if err != nil {
		return []models.BaseFromLyData{}, err
	}

	currentYearAndNextYear, err := utils.GetCurrentYearAndNextYear(format)
	if err != nil {
		return nil, err
	}

	month, err := utils.GetCurrentMonth(format)
	if err != nil {
		return nil, err
	}
	monthSuffix := "预估"
	logs.Info("SupplyDemandBalanceProcessor Process() : Target Year: %s：%s\n", currentYearAndNextYear, month+monthSuffix)

	// 处理提取的表格数据
	var result []models.BaseFromLyData

	headers := tableData.Headers
	rows := tableData.Rows

	for _, year := range currentYearAndNextYear {
		columnName = year + month + monthSuffix
		isCurrentYear, err := utils.IsCurrentYear(year)
		if err != nil {
			logs.Error("SupplyDemandBalanceProcessor Process() : Failed to determine if year is current year: %v", err)
			continue
		}
		if !isCurrentYear {
			format, err = utils.GetNextYearLastDay(format)
			if err != nil {
				logs.Error("SupplyDemandBalanceProcessor Process() : Failed to get next year last day: %v", err)
				continue
			}
		}

		// 查找目标列
		columnIdx := -1
		for i, header := range headers {
			if strings.Contains(columnName, header) {
				columnIdx = i
				break
			}
		}
		if columnIdx == -1 {
			logs.Error("SupplyDemandBalanceProcessor Process() : Column '%s' not found in table", columnName)
			continue
		}
		// 处理表格中的每一行
		for _, row := range rows {
			if len(row) >= len(headers) && row[0] == rowVariety {
				if columnIdx < len(row) {
					// 指标名称
					indexName := strings.Join(keywords[:len(keywords)-2], "：")
					// 指标编码
					indexCode := utils.GenerateIndexCode(lySourceName, indexName)
					// 指标id获取
					indexId, err := getIndexId(indexCode, indexName, classifyId, lySourceName, keywords[len(keywords)-2], keywords[len(keywords)-1])
					if err != nil {
						logs.Error("SupplyDemandBalanceProcessor Process() : Failed to get index id: %v", err)
						continue
					}
					valueStr := row[columnIdx]
					value, err := strconv.ParseFloat(valueStr, 64)
					if err != nil {
						return []models.BaseFromLyData{}, fmt.Errorf("SupplyDemandBalanceProcessor Process() : failed to parse value '%s': %v", valueStr, err)
					}
					yearMonth, err := utils.GetYearMonth(format)
					if err != nil {
						logs.Error("SupplyDemandBalanceProcessor Process() : Failed to get year month: %v", err)
						continue
					}
					indexData, err := models.GetLyDataByIndexIdAndDataTimeYM(indexId, yearMonth)
					if err != nil {
						logs.Error("SupplyDemandBalanceProcessor Process() : Failed to get data by index id and date: %v", err)
						continue
					}
					if len(indexData) > 0 {
						logs.Info("SupplyDemandBalanceProcessor Process() : Data already exists for index %d and date %s", indexId, dateText)
						// 存在走更新逻辑 主要更新今年在去年的预估值
						indexData := indexData[0]
						if indexData.Value != value {
							time, err := utils.StringToTime(indexData.ModifyTime)
							if err != nil {
								return nil, err
							}

							timeZero, err := utils.StringToTimeZero(format)
							if err != nil {
								return nil, err
							}

							if time.Before(timeZero) {
								// 更新指标数据
								err := models.UpdateLyDataById(indexData.BaseFromLyDataId, value)
								if err != nil {
									logs.Error("SupplyDemandBalanceProcessor Process() : Failed to update data: %v", err)
									continue
								}
								// 更新指标库数据
								edbIndexData, err := models.GetLyEdbDataByIndexCodeAndDataTime(indexData.IndexCode, yearMonth)
								if err != nil {
									return nil, err
								}
								if len(edbIndexData) > 0 {
									err := models.UpdateLyEdbDataById(edbIndexData[0].EdbInfoId, value)
									if err != nil {
										return nil, err
									}
								}
							}
						}
						continue
					}

					// 创建并添加到结果列表
					baseFromLyData := models.BaseFromLyData{
						DataTime:          format,
						Value:             value,
						BaseFromLyIndexId: indexId,
						IndexCode:         indexCode,
					}
					result = append(result, baseFromLyData)
				} else {
					log.Printf("SupplyDemandBalanceProcessor Process() : Column index out of range for row '%s', '%s'", rowVariety, columnName)
				}
				break
			}
		}
	}
	return result, nil
}

// PurchaseShippingProcessor
// @Description: 采购装船处理器
type PurchaseShippingProcessor struct{}

func (p *PurchaseShippingProcessor) Process(ctx context.Context, product string, reportContent string, keywords []string, classifyId int) ([]models.BaseFromLyData, error) {
	logs.Info("Processing purchase shipping...")
	// 解析关键字
	if len(keywords) < 3 {
		return []models.BaseFromLyData{}, fmt.Errorf("PurchaseShippingProcessor Process() : keywords must contain at least 3 elements")
	}

	// 拿到 行关键字和列关键字
	columnName := keywords[len(keywords)-3]

	// 提取所有表格数据
	tableData := getPurchaseShippingTableData(reportContent)
	logs.Info("PurchaseShippingProcessor Process() : Table data: %v", tableData)

	// 提取日期信息
	dateText, err := getDateInfo(ctx)
	if err != nil {
		return []models.BaseFromLyData{}, err
	}

	// 时间格式转换
	format, err := utils.ConvertTimeFormat(dateText)
	if err != nil {
		return []models.BaseFromLyData{}, err
	}

	// 处理提取的表格数据
	var result []models.BaseFromLyData

	headers := tableData.Headers
	rows := tableData.Rows

	// 查找目标列
	columnIdx := -1
	for i, header := range headers {
		if strings.Contains(columnName, header) {
			columnIdx = i
			break
		}
	}
	if columnIdx == -1 {
		log.Printf("PurchaseShippingProcessor Process() : Column '%s' not found in table", columnName)
	} else {
		// 处理表格中的每一行
		for _, row := range rows {
			if len(row) >= len(headers) {
				if columnIdx < len(row) {
					if !isNumber(row[columnIdx]) {
						continue
					}
					// 指标名称
					indexName := strings.Join(keywords[:len(keywords)-3], "：")
					// 指标编码
					indexCode := utils.GenerateIndexCode(lySourceName, indexName)
					// 指标id获取
					indexId, err := getIndexId(indexCode, indexName, classifyId, lySourceName, keywords[len(keywords)-2], keywords[len(keywords)-1])
					if err != nil {
						logs.Error("PurchaseShippingProcessor Process() : Failed to get index id: %v", err)
						continue
					}
					var yearMonth string
					number, err := utils.ConvertMonthToNumber(row[1])
					if err != nil {
						return nil, err
					}
					yearMonth = row[0] + "-" + number
					isSameMonth, err := utils.IsSameMonth(format, yearMonth)
					if err != nil {
						return nil, err
					}
					if isSameMonth {
						yearMonth = format
					} else {
						lastDayOfMonth, err := utils.GetLastDayOfMonth(yearMonth)
						if err != nil {
							return nil, err
						}
						yearMonth = lastDayOfMonth
					}

					valueStr := row[columnIdx]
					value, err := strconv.ParseFloat(valueStr, 64)
					if err != nil {
						return []models.BaseFromLyData{}, fmt.Errorf("PurchaseShippingProcessor Process() : failed to parse value '%s': %v", valueStr, err)
					}

					month, err := utils.GetYearMonth(yearMonth)
					if err != nil {
						return nil, err
					}
					indexData, err := models.GetLyDataByIndexIdAndDataTimeYM(indexId, month)
					if err != nil {
						logs.Error("PurchaseShippingProcessor Process() : Failed to get data by index id and date: %v", err)
						continue
					}
					if len(indexData) > 0 {
						if indexData[0].Value != value {
							logs.Info("PurchaseShippingProcessor Process() : Data already exists for index %d and date %s", indexId, dateText)

							lyData := indexData[0]
							time, err := utils.StringToTime(lyData.ModifyTime)
							if err != nil {
								return nil, err
							}

							timeZero, err := utils.StringToTimeZero(format)
							if err != nil {
								return nil, err
							}

							if time.Before(timeZero) {
								// 更新指标数据
								err := models.UpdateLyDataById(lyData.BaseFromLyDataId, value)
								if err != nil {
									return nil, err
								}

								// 同步更新指标库数据 须根据指标编码和日期更新
								edbIndexData, err := models.GetLyEdbDataByIndexCodeAndDataTime(lyData.IndexCode, month)
								if err != nil {
									return nil, err
								}
								if len(edbIndexData) > 0 {
									err := models.UpdateLyEdbDataById(edbIndexData[0].EdbInfoId, value)
									if err != nil {
										return nil, err
									}
								}
							}
						}
						continue
					}

					// 创建并添加到结果列表
					baseFromLyData := models.BaseFromLyData{
						DataTime:          yearMonth,
						Value:             value,
						BaseFromLyIndexId: indexId,
						IndexCode:         indexCode,
					}
					result = append(result, baseFromLyData)
					continue
				} else {
					log.Printf("PurchaseShippingProcessor Process() : Column index out of range for row '%s'", columnName)
				}
				break
			}
		}
	}

	return result, nil
}

// ProcessingReportProcessor
// @Description: 加工报告处理器
type ProcessingReportProcessor struct {
}

func (p *ProcessingReportProcessor) Process(ctx context.Context, product string, reportContent string, keywords []string, classifyId int) ([]models.BaseFromLyData, error) {
	logs.Info("Processing processing report...")
	// 解析关键字
	if len(keywords) < 3 {
		return []models.BaseFromLyData{}, fmt.Errorf("ProcessingReportProcessor Process() : keywords must contain at least 3 elements")
	}

	// 拿到 行关键字和列关键字
	columnName := keywords[0]
	rowName := keywords[1]

	// 提取所有表格数据
	tableData := getAllTableData(reportContent)

	// 提取日期信息
	dateText, err := getDateInfo(ctx)
	if err != nil {
		return []models.BaseFromLyData{}, err
	}
	indexName := strings.Join(keywords[:len(keywords)-2], "：")
	// 指标编码
	indexCode := utils.GenerateIndexCode(lySourceName, indexName)
	// 指标id获取
	indexId, err := getIndexId(indexCode, indexName, classifyId, lySourceName, keywords[len(keywords)-2], keywords[len(keywords)-1])
	if err != nil {
		return nil, err
	}

	// 校验指标数据是否存在 根据指标id和日期 存在则跳过，不存在正常往下走
	format, err := utils.ConvertTimeFormat(dateText)
	if err != nil {
		return []models.BaseFromLyData{}, err
	}
	indexData, err := models.GetLyDataByIndexIdAndDataTime(indexId, format)
	if err != nil {
		return []models.BaseFromLyData{}, fmt.Errorf("ProcessingReportProcessor Process() : Failed to get data by index id and date: %v", err)
	}
	if len(indexData) > 0 {
		logs.Info("ProcessingReportProcessor Process() : Data already exists for index %d and date %s", indexId, dateText)
		// 不必做更新处理，报告每周刷新，即使本周和上周数据一致，也需要每周记录
		return []models.BaseFromLyData{}, nil
	}

	// 解析日期并计算当前周数
	targetWeek, err := utils.ParseDateAndWeek(dateText)
	if err != nil {
		return []models.BaseFromLyData{}, fmt.Errorf("ProcessingReportProcessor Process() : Failed to parse date: %v", err)
	}

	fmt.Printf("Target Week: %s\n", targetWeek)

	var result []models.BaseFromLyData
	// 处理提取的表格数据
	for _, data := range tableData {
		tableHeaders := data.Headers
		tableRows := data.Rows

		// 查找目标列
		columnIdx := -1
		for i, header := range tableHeaders {
			headerString := extractChinese(header)
			if strings.Contains(columnName, headerString) {
				// 这个表格不是很好处理，这里写的有些僵硬，后续需要优化
				if columnName == "国内大豆开机率" {
					i = i + 2
				}
				columnIdx = i
				break
			}
		}

		if columnIdx == -1 {
			logs.Error("ProcessingReportProcessor Process() : Column '%s' not found in table", columnName)
			continue
		}

		// 查找本周的列位置
		weekIdx := -1
		for i, header := range tableHeaders {
			if strings.Contains(header, targetWeek) && i > columnIdx {
				weekIdx = i
				break
			}
		}

		if weekIdx == -1 {
			fmt.Printf("Week column '%s' not found in table\n", targetWeek)
			continue
		}

		// 查找目标行
		for _, row := range tableRows {
			if strings.Contains(row[0], rowName) {
				if columnIdx < len(row) {
					// 指标名称
					indexName := strings.Join(keywords[:len(keywords)-2], "：")
					// 指标编码
					indexCode := utils.GenerateIndexCode(lySourceName, indexName)
					// 指标id获取
					indexId, err := getIndexId(indexCode, indexName, classifyId, lySourceName, keywords[len(keywords)-2], keywords[len(keywords)-1])
					if err != nil {
						logs.Error("ProcessingReportProcessor Process() : Failed to get index id: %v", err)
						continue
					}

					indexData, err := models.GetLyDataByIndexIdAndDataTime(indexId, format)
					if err != nil {
						logs.Error("ProcessingReportProcessor Process() : Failed to get data by index id and date: %v", err)
						continue
					}
					if len(indexData) > 0 {
						logs.Info("ProcessingReportProcessor Process() : Data already exists for index %d and date %s", indexId, dateText)
						// 无需走更新逻辑，报告每日更新，即使今天和昨天数据一致，也需要每天记录，如果到这里也只是说，今天这个报告被读取了两次
						continue
					}

					valueStr := row[columnIdx]
					value, err := strconv.ParseFloat(valueStr, 64)
					if err != nil {
						return []models.BaseFromLyData{}, fmt.Errorf("ProcessingReportProcessor Process() : failed to parse value '%s': %v", valueStr, err)
					}
					// 创建并添加到结果列表
					baseFromLyData := models.BaseFromLyData{
						DataTime:          format,
						Value:             value,
						BaseFromLyIndexId: indexId,
						IndexCode:         indexCode,
					}
					result = append(result, baseFromLyData)
					continue
				} else {
					log.Printf("ProcessingReportProcessor Process() : Column index out of range for row '%s', '%s'", rowName, columnName)
				}
				break
			}
		}
	}

	return result, nil
}

// InventoryAnalysisProcessor
// @Description: 库存分析处理器
type InventoryAnalysisProcessor struct{}

func (p *InventoryAnalysisProcessor) Process(ctx context.Context, product string, reportContent string, keywords []string, classifyId int) ([]models.BaseFromLyData, error) {
	// https://www.fao.com.cn/art/yg1IKj9FpPEIDv2LefnPhQ==.htm
	logs.Info("Processing inventory analysis...")

	// 解析关键字
	if len(keywords) < 4 {
		return []models.BaseFromLyData{}, fmt.Errorf("InventoryAnalysisProcessor Process() : keywords must contain at least 4 elements")
	}

	// 拿到 行关键字和列关键字
	columnName := keywords[0]
	rowVariety := keywords[1]

	columnSuffix := "本周"
	columnName = columnName + columnSuffix

	// 提取所有表格数据
	tableData := getTableData(reportContent, true)
	logs.Info("InventoryAnalysisProcessor Process() : Table data: %v", tableData)

	// 提取日期信息
	dateText, err := getDateInfo(ctx)
	if err != nil {
		return []models.BaseFromLyData{}, err
	}

	// 时间格式转换
	format, err := utils.ConvertTimeFormat(dateText)
	if err != nil {
		return []models.BaseFromLyData{}, err
	}

	// 处理提取的表格数据
	var result []models.BaseFromLyData

	headers := tableData.Headers
	rows := tableData.Rows

	// 查找目标列
	columnIdx := -1
	for i, header := range headers {
		header := removeParentheses(header)
		if strings.Contains(columnName, header) {
			columnIdx = i
			break
		}
	}
	if columnIdx == -1 {
		logs.Error("InventoryAnalysisProcessor Process() : Column '%s' not found in table", columnName)
	} else {
		// 处理表格中的每一行
		for _, row := range rows {
			if len(row) >= len(headers) && strings.Contains(row[0], rowVariety) {
				if columnIdx < len(row) {
					// 指标名称
					indexName := strings.Join(keywords[:len(keywords)-2], "：")
					indexName = removeParentheses(indexName)
					// 指标编码
					indexCode := utils.GenerateIndexCode(lySourceName, indexName)
					// 指标id获取
					indexId, err := getIndexId(indexCode, indexName, classifyId, lySourceName, keywords[len(keywords)-2], keywords[len(keywords)-1])
					if err != nil {
						logs.Error("InventoryAnalysisProcessor Process() : Failed to get index id: %v", err)
						continue
					}

					indexData, err := models.GetLyDataByIndexIdAndDataTime(indexId, format)
					if err != nil {
						logs.Error("InventoryAnalysisProcessor Process() : Failed to get data by index id and date: %v", err)
						continue
					}
					if len(indexData) > 0 {
						logs.Info("InventoryAnalysisProcessor Process() : Data already exists for index %d and date %s", indexId, dateText)
						// 无需走更新逻辑，报告每周更新，即使本周和上周数据一致，也需要每周记录，如果到这里也只是说，今天这个报告被读取了两次
						continue
					}

					valueStr := row[columnIdx]
					value, err := strconv.ParseFloat(valueStr, 64)
					if err != nil {
						return []models.BaseFromLyData{}, fmt.Errorf("InventoryAnalysisProcessor Process() : failed to parse value '%s': %v", valueStr, err)
					}
					// 创建并添加到结果列表
					baseFromLyData := models.BaseFromLyData{
						DataTime:          format,
						Value:             value,
						BaseFromLyIndexId: indexId,
						IndexCode:         indexCode,
					}
					result = append(result, baseFromLyData)
					continue
				} else {
					log.Printf("InventoryAnalysisProcessor Process() : Column index out of range for row '%s', '%s'", rowVariety, columnName)
				}
				break
			}
		}
	}

	return result, nil
}

// PriceSpreadArbitrageProcessor
// @Description: 价差套利处理器
type PriceSpreadArbitrageProcessor struct{}

func (p *PriceSpreadArbitrageProcessor) Process(ctx context.Context, product string, reportContent string, keywords []string, classifyId int) ([]models.BaseFromLyData, error) {
	fmt.Println("Processing processing profit...")
	// 解析关键字
	if len(keywords) < 4 {
		return []models.BaseFromLyData{}, fmt.Errorf("PriceSpreadArbitrageProcessor Process() : keywords must contain at least 4 elements")
	}

	// 拿到 行关键字和列关键字
	var columnDate string
	rowVariety := keywords[0]
	rowBourse := keywords[1]
	// 提取所有表格数据
	tableData := getNoHeadTableData(reportContent)

	// 提取日期信息
	dateText, err := getDateInfo(ctx)
	if err != nil {
		return []models.BaseFromLyData{}, err
	}

	// 时间格式转换
	format, err := utils.ConvertTimeFormat(dateText)
	if err != nil {
		return []models.BaseFromLyData{}, err
	}
	day, err := utils.ConvertTimeFormatToYearMonthDay(format)
	if err != nil {
		return nil, err
	}
	columnDate = day

	// 处理提取的表格数据
	var result []models.BaseFromLyData

	for _, data := range tableData {
		tableHeaders := data.Headers
		tableRows := data.Rows

		// 查找目标列
		columnIdx := -1
		for i, header := range tableHeaders {
			if strings.Contains(header, columnDate) {
				columnIdx = i
				break
			}
		}
		if columnIdx == -1 {
			log.Printf("PriceSpreadArbitrageProcessor Process() : Column '%s' not found in table", columnDate)
			continue
		}

		// 处理表格中的每一行
		for _, row := range tableRows {
			if len(row) >= len(tableHeaders) && row[0] == rowVariety && row[1] == rowBourse {
				if columnIdx < len(row) {
					// 指标名称
					indexName := strings.Join(keywords[:len(keywords)-2], "：")
					// 指标编码
					indexCode := utils.GenerateIndexCode(lySourceName, indexName)
					// 指标id获取
					indexId, err := getIndexId(indexCode, indexName, classifyId, lySourceName, keywords[len(keywords)-2], keywords[len(keywords)-1])
					if err != nil {
						logs.Error("PriceSpreadArbitrageProcessor Process() : Failed to get index id: %v", err)
						continue
					}

					indexData, err := models.GetLyDataByIndexIdAndDataTime(indexId, format)
					if err != nil {
						logs.Error("PriceSpreadArbitrageProcessor Process() : Failed to get data by index id and date: %v", err)
						continue
					}
					if len(indexData) > 0 {
						logs.Info("PriceSpreadArbitrageProcessor Process() : Data already exists for index %d and date %s", indexId, dateText)
						// 无需走更新逻辑，报告每天更新，即使今天和每天数据一致，也需要每天记录，如果到这里也只是说，今天这个报告被读取了两次
						continue
					}

					valueStr := row[columnIdx]
					value, err := strconv.ParseFloat(valueStr, 64)
					if err != nil {
						return []models.BaseFromLyData{}, fmt.Errorf("failed to parse value '%s': %v", valueStr, err)
					}
					// 创建并添加到结果列表
					baseFromLyData := models.BaseFromLyData{
						DataTime:          format,
						Value:             value,
						BaseFromLyIndexId: indexId,
						IndexCode:         indexCode,
					}
					result = append(result, baseFromLyData)
				} else {
					log.Printf("PriceSpreadArbitrageProcessor Process() : Column index out of range for row '%s', '%s'", rowVariety, columnDate)
				}
				break
			}

		}
	}

	return result, nil
}

// DailyTransactionProcessor
// @Description: 每日成交处理器
type DailyTransactionProcessor struct{}

func (p *DailyTransactionProcessor) Process(ctx context.Context, product string, reportContent string, keywords []string, classifyId int) ([]models.BaseFromLyData, error) {
	fmt.Println("Processing processing profit...")
	// 解析关键字
	if len(keywords) < 4 {
		return []models.BaseFromLyData{}, fmt.Errorf("DailyTransactionProcessor Process() : keywords must contain at least 4 elements")
	}

	// 获取第一个表格
	areaTableData := getNoHeadTableData(reportContent)[0]
	// 获取第二个表格
	blocTableData := getTableData(reportContent, false)
	logs.Info("DailyTransactionProcessor Process() : areaTableData data: %v, blocTableData data: %v", areaTableData, blocTableData)
	// 提取日期信息
	dateText, err := getDateInfo(ctx)
	if err != nil {
		return []models.BaseFromLyData{}, err
	}

	// 时间格式转换
	format, err := utils.ConvertTimeFormat(dateText)
	if err != nil {
		return []models.BaseFromLyData{}, err
	}

	// 处理提取的表格数据
	var result []models.BaseFromLyData

	areaHeaders := areaTableData.Headers
	areaRows := areaTableData.Rows

	// 第一个表格
	// 拿到 行关键字和列关键字
	columnArea := keywords[1]
	var rowAreaMonthDays []string
	rowWeek := "地区总计"

	monthDay, err := utils.GetWeekdaysInSameWeek(format)
	if err != nil {
		return nil, err
	}
	rowAreaMonthDays = monthDay

	// 查找目标列
	areaColumnIdx := -1
	for i, header := range areaHeaders {
		if strings.Contains(header, columnArea) {
			areaColumnIdx = i
			break
		}
	}
	if areaColumnIdx == -1 {
		log.Printf("DailyTransactionProcessor Process() : One Column '%s' not found in table", columnArea)
	} else if !strings.Contains(strings.Join(keywords[:len(keywords)-3], "："), "主要集团") {
		for _, row := range areaRows {
			if len(row) >= len(areaHeaders) && row[0] == rowWeek {
				if areaColumnIdx < len(row) {
					// 指标名称
					indexName := strings.Join(keywords[:len(keywords)-3], "：")
					// 指标编码
					indexCode := utils.GenerateIndexCode(lySourceName, indexName)
					// 指标id获取
					indexId, err := getIndexId(indexCode, indexName, classifyId, lySourceName, keywords[len(keywords)-2], keywords[len(keywords)-1])
					if err != nil {
						logs.Error("DailyTransactionProcessor Process() : Failed to get index id: %v", err)
						continue
					}

					indexData, err := models.GetLyDataByIndexIdAndDataTime(indexId, format)
					if err != nil {
						logs.Error("DailyTransactionProcessor Process() : Failed to get data by index id and date: %v", err)
						continue
					}
					if len(indexData) > 0 {
						logs.Info("DailyTransactionProcessor Process() : Data already exists for index %d and date %s", indexId, dateText)
						// 无需走更新逻辑，报告每周更新，一周出来一周中每天得数据，即使本周和上周数据一致，也需要每天记录，如果到这里也只是说，今天这个报告被读取了两次
						continue
					}

					valueStr := row[areaColumnIdx]
					isChinese := IsChinese(valueStr)
					if isChinese {
						continue
					}
					value, err := strconv.ParseFloat(valueStr, 64)
					if err != nil {
						return []models.BaseFromLyData{}, fmt.Errorf("failed to parse value '%s': %v", valueStr, err)
					}
					// 创建并添加到结果列表
					var dealDate string
					if row[0] == rowWeek {
						dealDate = format
					} else {
						date, err := utils.ConvertToDate(row[0])
						if err != nil {
							return nil, err
						}
						dealDate = date
					}
					baseFromLyData := models.BaseFromLyData{
						DataTime:          dealDate,
						Value:             value,
						BaseFromLyIndexId: indexId,
						IndexCode:         indexCode,
					}

					result = append(result, baseFromLyData)
				} else {
					log.Printf("DailyTransactionProcessor Process() : Column index out of range for row '%s', '%s'", monthDay, columnArea)
				}
				break
			} else {
				for _, monthDay := range rowAreaMonthDays {
					if len(row) >= len(areaHeaders) && (row[0] == monthDay && !strings.Contains(strings.Join(keywords[:len(keywords)-3], "："), "周度")) {
						if areaColumnIdx < len(row) {
							// 指标名称
							indexName := strings.Join(keywords[:len(keywords)-3], "：")
							// 指标编码
							indexCode := utils.GenerateIndexCode(lySourceName, indexName)
							// 指标id获取
							indexId, err := getIndexId(indexCode, indexName, classifyId, lySourceName, keywords[len(keywords)-2], keywords[len(keywords)-1])
							if err != nil {
								logs.Error("DailyTransactionProcessor Process() : Failed to get index id: %v", err)
								continue
							}

							indexData, err := models.GetLyDataByIndexIdAndDataTime(indexId, format)
							if err != nil {
								logs.Error("DailyTransactionProcessor Process() : Failed to get data by index id and date: %v", err)
								continue
							}
							if len(indexData) > 0 {
								logs.Info("DailyTransactionProcessor Process() : Data already exists for index %d and date %s", indexId, dateText)
								// 无需走更新逻辑，报告每周更新，即使本周和上周数据一致，也需要每周记录，如果到这里也只是说，今天这个报告被读取了两次
								continue
							}

							valueStr := row[areaColumnIdx]
							isChinese := IsChinese(valueStr)
							if isChinese {
								continue
							}
							value, err := strconv.ParseFloat(valueStr, 64)
							if err != nil {
								return []models.BaseFromLyData{}, fmt.Errorf("failed to parse value '%s': %v", valueStr, err)
							}
							// 创建并添加到结果列表
							var dealDate string
							if row[0] == rowWeek {
								dealDate = format
							} else {
								date, err := utils.ConvertToDate(row[0])
								if err != nil {
									return nil, err
								}
								dealDate = date
							}
							baseFromLyData := models.BaseFromLyData{
								DataTime:          dealDate,
								Value:             value,
								BaseFromLyIndexId: indexId,
								IndexCode:         indexCode,
							}

							result = append(result, baseFromLyData)
						} else {
							log.Printf("DailyTransactionProcessor Process() : Column index out of range for row '%s', '%s'", monthDay, columnArea)
						}
						break
					}
				}
			}
		}
	}

	// 第二个表格
	// 拿到 行关键字和列关键字
	columnBloc := keywords[len(keywords)-3]
	rowBloc := keywords[1]

	blocHeaders := blocTableData.Headers
	blocRows := blocTableData.Rows

	// 查找目标列
	blocColumnIdx := -1
	for i, header := range blocHeaders {
		if strings.Contains(header, columnBloc) {
			blocColumnIdx = i
			break
		}
	}

	if blocColumnIdx == -1 {
		log.Printf("DailyTransactionProcessor Process() : Two Column '%s' not found in table", columnBloc)
	} else {
		// 处理表格中的每一行
		for _, row := range blocRows {
			if len(row) >= len(blocHeaders) && strings.Contains(row[0], rowBloc) {
				if blocColumnIdx < len(row) {
					// 指标名称
					indexName := strings.Join(keywords[:len(keywords)-3], "：")
					indexName = removeParentheses(indexName)
					// 指标编码
					indexCode := utils.GenerateIndexCode(lySourceName, indexName)
					// 指标id获取
					indexId, err := getIndexId(indexCode, indexName, classifyId, lySourceName, keywords[len(keywords)-2], keywords[len(keywords)-1])
					if err != nil {
						logs.Error("DailyTransactionProcessor Process() : Failed to get index id: %v", err)
						continue
					}

					indexData, err := models.GetLyDataByIndexIdAndDataTime(indexId, format)
					if err != nil {
						logs.Error("DailyTransactionProcessor Process() : Failed to get data by index id and date: %v", err)
						continue
					}
					if len(indexData) > 0 {
						logs.Info("DailyTransactionProcessor Process() : Data already exists for index %d and date %s", indexId, dateText)
						continue
					}

					valueStr := row[blocColumnIdx]
					value, err := strconv.ParseFloat(valueStr, 64)
					if err != nil {
						return []models.BaseFromLyData{}, fmt.Errorf("DailyTransactionProcessor Process() : failed to parse value '%s': %v", valueStr, err)
					}
					// 创建并添加到结果列表
					baseFromLyData := models.BaseFromLyData{
						DataTime:          format,
						Value:             value,
						BaseFromLyIndexId: indexId,
						IndexCode:         indexCode,
					}
					result = append(result, baseFromLyData)
				} else {
					log.Printf("DailyTransactionProcessor Process() : Column index out of range for row '%s', '%s'", rowBloc, columnBloc)
				}
				break
			}
		}
	}

	return result, nil
}

// PalmOilImportCostProcessor 棕榈油进口成本
type PalmOilImportCostProcessor struct{}

func (p *PalmOilImportCostProcessor) Process(ctx context.Context, product string, reportContent string, keywords []string, classifyId int) ([]models.BaseFromLyData, error) {
	logs.Info("Processing palm oil import cost...")
	// 解析关键字
	if len(keywords) < 5 {
		return []models.BaseFromLyData{}, fmt.Errorf("PalmOilImportCostProcessor Process() : keywords must contain at least 5 elements")
	}

	// 拿到 行关键字和列关键字
	columnName := keywords[len(keywords)-4]
	rowVariety := keywords[0]
	rowPort := keywords[len(keywords)-3]
	indexNamePrefix := keywords[:1]
	indexNameSuffix := keywords[1:]

	// 提取所有表格数据
	tableData := getNoHeadTableData(reportContent)

	// 提取日期信息
	dateText, err := getDateInfo(ctx)
	if err != nil {
		return []models.BaseFromLyData{}, err
	}

	// 时间格式转换
	format, err := utils.ConvertTimeFormat(dateText)
	if err != nil {
		return []models.BaseFromLyData{}, err
	}

	// 解析日期并计算当前月份
	targetMonths, err := utils.GetYearMonthNoYear(format)
	if err != nil {
		return []models.BaseFromLyData{}, fmt.Errorf("PalmOilImportCostProcessor Process() : Failed to parse date: %v", err)
	}
	fmt.Printf("Target Month: %s\n", targetMonths)

	// 处理提取的表格数据
	var result []models.BaseFromLyData

	for _, data := range tableData {
		tableHeaders := data.Headers
		tableRows := data.Rows

		// 查找目标列
		columnIdx := -1
		for i, header := range tableHeaders {
			if strings.Contains(header, columnName) {
				columnIdx = i
				break
			}
		}
		if columnIdx == -1 {
			log.Printf("PalmOilImportCostProcessor Process() : Column '%s' not found in table", columnName)
			continue
		}

		// 处理表格中的每一行
		//var flag bool = true
		var previousRowVariety string
		var previousRowPort string
		var previousRowFob string
		for rowIndex, row := range tableRows {
			if len(row) == len(tableHeaders) {
				previousRowVariety = row[0]
				previousRowPort = row[1]
				previousRowFob = row[2]
			} else if len(row) == len(tableHeaders)-1 {
				previousRowPort = row[0]
				previousRowFob = row[1]
				row = append([]string{previousRowVariety}, row...)
				tableRows[rowIndex] = row
			} else if len(row) == len(tableHeaders)-2 {
				// 这段这里不需要。。。先保留吧
				previousRowFob = row[0]
				row = append([]string{previousRowVariety, previousRowPort}, row...)
				tableRows[rowIndex] = row
			} else if len(row) == len(tableHeaders)-3 {
				row = append([]string{previousRowVariety, previousRowPort, previousRowFob}, row...)
				tableRows[rowIndex] = row
			}
			for _, targetMonth := range targetMonths {
				if len(row) >= len(tableHeaders) && strings.Contains(rowVariety, row[0]) && row[1] == targetMonth && row[len(row)-1] == rowPort {
					if columnIdx < len(row) {
						// 指标名称
						indexNameList := append(indexNamePrefix, append([]string{targetMonth}, indexNameSuffix...)...)
						indexName := strings.Join(indexNameList[:len(keywords)-2], "：")
						// 指标编码
						indexCode := utils.GenerateIndexCode(lySourceName, indexName)
						// 指标id获取
						indexId, err := getIndexId(indexCode, indexName, classifyId, lySourceName, keywords[len(keywords)-2], keywords[len(keywords)-1])
						if err != nil {
							logs.Error("PalmOilImportCostProcessor Process() : Failed to get index id: %v", err)
							continue
						}

						indexData, err := models.GetLyDataByIndexIdAndDataTime(indexId, format)
						if err != nil {
							logs.Error("PalmOilImportCostProcessor Process() : Failed to get data by index id and date: %v", err)
							continue
						}
						if len(indexData) > 0 {
							logs.Info("PalmOilImportCostProcessor Process() : Data already exists for index %d and date %s", indexId, dateText)
							continue
						}

						valueStr := row[columnIdx]
						value, err := strconv.ParseFloat(valueStr, 64)
						if err != nil {
							return []models.BaseFromLyData{}, fmt.Errorf("failed to parse value '%s': %v", valueStr, err)
						}
						// 创建并添加到结果列表
						baseFromLyData := models.BaseFromLyData{
							DataTime:          format,
							Value:             value,
							BaseFromLyIndexId: indexId,
							IndexCode:         indexCode,
						}
						result = append(result, baseFromLyData)
					} else {
						log.Printf("PalmOilImportCostProcessor Process() : Column index out of range for row '%s', '%s'", rowVariety, rowPort)
					}
					break
				}
			}

		}

	}

	return result, nil
}

// ImportEstimateProcessor
// @Description: 进口预估处理器
type ImportEstimateProcessor struct{}

func (p *ImportEstimateProcessor) Process(ctx context.Context, product string, reportContent string, keywords []string, classifyId int) ([]models.BaseFromLyData, error) {
	logs.Info("Processing import estimate...")
	// 解析关键字
	if len(keywords) < 4 {
		return []models.BaseFromLyData{}, fmt.Errorf("ImportEstimateProcessor Process() : keywords must contain at least 4 elements")
	}

	// 拿到 行关键字和列关键字
	var columnDates []string
	rowVariety := keywords[1]

	// 提取所有表格数据
	tableData := getNoHeadTableData(reportContent)

	// 提取日期信息
	dateText, err := getDateInfo(ctx)
	if err != nil {
		return []models.BaseFromLyData{}, err
	}

	// 时间格式转换
	format, err := utils.ConvertTimeFormat(dateText)
	if err != nil {
		return []models.BaseFromLyData{}, err
	}

	columnDates, err = utils.GetNextThreeMonthsNoYear(format)
	if err != nil {
		return nil, err
	}

	monthsLastDay, err := utils.GetNextThreeMonthsLastDay(format)
	if err != nil {
		return nil, err
	}

	// 处理提取的表格数据
	var result []models.BaseFromLyData

	for _, data := range tableData {
		tableHeaders := data.Headers
		tableRows := data.Rows

		// 查找目标列
		for _, columnDate := range columnDates {
			columnIdx := -1
			for i, tableHeader := range tableHeaders {
				if strings.Contains(tableHeader, columnDate) {
					columnIdx = i
					break
				}
			}

			if columnIdx == -1 {
				log.Printf("ImportEstimateProcessor Process() : Column '%s' not found in table", columnDate)
				continue
			} else {
				// 处理表格中的每一行
				for _, row := range tableRows {
					if len(row) >= len(tableHeaders) && strings.Contains(row[0], rowVariety) && isNumber(row[columnIdx]) {
						if columnIdx < len(row) {
							// 指标名称
							indexName := strings.Join(keywords[:len(keywords)-2], `：`)
							// 指标编码
							indexCode := utils.GenerateIndexCode(lySourceName, indexName)
							// 指标id获取
							indexId, err := getIndexId(indexCode, indexName, classifyId, lySourceName, keywords[len(keywords)-2], keywords[len(keywords)-1])
							if err != nil {
								logs.Error("ImportEstimateProcessor Process() : Failed to get index id: %v", err)
								continue
							}
							toNumber, err := utils.ConvertMonthToNumber(columnDate)
							if err != nil {
								logs.Error("ImportEstimateProcessor Process() : Failed to convert month to number: %v", err)
								continue
							}
							slice, err := utils.GetElementInSlice(monthsLastDay, toNumber)
							if err != nil {
								logs.Error("ImportEstimateProcessor Process() : Failed to get element in slice: %v", err)
								continue
							}

							indexData, err := models.GetLyDataByIndexIdAndDataTime(indexId, slice)
							if err != nil {
								logs.Error("ImportEstimateProcessor Process() : Failed to get data by index id and date: %v", err)
								continue
							}

							valueStr := row[columnIdx]
							value, err := strconv.ParseFloat(valueStr, 64)
							if err != nil {
								return []models.BaseFromLyData{}, fmt.Errorf("failed to parse value '%s': %v", valueStr, err)
							}

							if len(indexData) > 0 {
								if indexData[0].Value != value {
									logs.Info("ImportEstimateProcessor Process() : Data already exists for index %d and date %s", indexId, dateText)

									lyData := indexData[0]
									time, err := utils.StringToTime(lyData.ModifyTime)
									if err != nil {
										return nil, err
									}

									timeZero, err := utils.StringToTimeZero(format)
									if err != nil {
										return nil, err
									}

									if lyData.Value != value && time.Before(timeZero) {
										// 更新指标数据
										err := models.UpdateLyDataById(lyData.BaseFromLyDataId, value)
										if err != nil {
											return nil, err
										}

										// 同步更新指标库数据
										lyEdbIndexData, err := models.GetLyEdbDataByIndexCodeAndExactDataTime(lyData.IndexCode, lyData.DataTime)
										if err != nil {
											return nil, err
										}
										if len(lyEdbIndexData) > 0 {
											err := models.UpdateLyEdbDataById(lyEdbIndexData[0].EdbInfoId, value)
											if err != nil {
												return nil, err
											}
										}
									}
								}

								continue
							}

							// 创建并添加到结果列表
							baseFromLyData := models.BaseFromLyData{
								DataTime:          slice,
								Value:             value,
								BaseFromLyIndexId: indexId,
								IndexCode:         indexCode,
							}
							result = append(result, baseFromLyData)
						} else {
							log.Printf("ImportEstimateProcessor Process() : Column index out of range for row '%s', '%s'", rowVariety, columnDate)
						}
						break
					}
				}
			}
		}
	}

	return result, nil
}

// InternationalPriceProcessor
// @Description: 国际价格处理器
type InternationalPriceProcessor struct{}

func (p *InternationalPriceProcessor) Process(ctx context.Context, product string, reportContent string, keywords []string, classifyId int) ([]models.BaseFromLyData, error) {
	logs.Info("Processing international price...")
	// 解析关键字
	if len(keywords) < 4 {
		return []models.BaseFromLyData{}, fmt.Errorf("InternationalPriceProcessor Process() : keywords must contain at least 4 elements")
	}

	// 拿到 行关键字和列关键字
	columnName := keywords[1]
	rowVariety := keywords[0]
	indexNamePrefix := keywords[:1]
	indexNameSuffix := keywords[1:]

	// 提取所有表格数据
	tableData := getNoHeadTableData(reportContent)

	// 提取日期信息
	dateText, err := getDateInfo(ctx)
	if err != nil {
		return []models.BaseFromLyData{}, err
	}

	// 时间格式转换
	format, err := utils.ConvertTimeFormat(dateText)
	if err != nil {
		return []models.BaseFromLyData{}, err
	}

	// 解析日期并计算当前月份 和 后两月
	yearMonths, err := utils.ConvertTimeFormatToYearMonth(format)
	if err != nil {
		return nil, err
	}
	fmt.Printf("Target yearMonth: %s\n", yearMonths)

	// 处理提取的表格数据
	var result []models.BaseFromLyData

	for _, data := range tableData {
		tableHeaders := data.Headers
		tableRows := data.Rows

		// 查找目标列
		columnIdx := -1
		for i, header := range tableHeaders {
			if strings.Contains(header, columnName) {
				columnIdx = i
				break
			}
		}
		if columnIdx == -1 {
			log.Printf("InternationalPriceProcessor Process() : Column '%s' not found in table", columnName)
			continue
		}

		// 处理表格中的每一行
		var previousRowVariety string
		for rowIndex, row := range tableRows {
			if len(row) == len(tableHeaders) {
				previousRowVariety = row[0]
			} else if len(row) == len(tableHeaders)-1 {
				row = append([]string{previousRowVariety}, row...)
				tableRows[rowIndex] = row
			}

			for _, targetMonth := range yearMonths {
				if len(row) >= len(tableHeaders) && row[0] == rowVariety && row[1] == targetMonth {
					if columnIdx < len(row) {
						// 指标名称
						indexNameList := append(indexNamePrefix, append([]string{targetMonth}, indexNameSuffix...)...)
						indexName := strings.Join(indexNameList[:len(keywords)-2], "：")
						// 指标编码
						indexCode := utils.GenerateIndexCode(lySourceName, indexName)
						// 指标id获取
						indexId, err := getIndexId(indexCode, indexName, classifyId, lySourceName, keywords[len(keywords)-2], keywords[len(keywords)-1])
						if err != nil {
							logs.Error("InternationalPriceProcessor Process() : Failed to get index id: %v", err)
							continue
						}

						indexData, err := models.GetLyDataByIndexIdAndDataTime(indexId, format)
						if err != nil {
							logs.Error("InternationalPriceProcessor Process() : Failed to get data by index id and date: %v", err)
							continue
						}
						if len(indexData) > 0 {
							logs.Info("InternationalPriceProcessor Process() : Data already exists for index %d and date %s", indexId, dateText)
							// 无需更新 指标展示本月和后两月的数据，报告每天更新，每天的值可能会改变，即使今天和每天数据一致，也需要每天记录，如果到这里也只是说，今天这个报告被读取了两次
							continue
						}

						valueStr := row[columnIdx]
						value, err := strconv.ParseFloat(valueStr, 64)
						if err != nil {
							return []models.BaseFromLyData{}, fmt.Errorf("failed to parse value '%s': %v", valueStr, err)
						}
						// 创建并添加到结果列表
						baseFromLyData := models.BaseFromLyData{
							DataTime:          format,
							Value:             value,
							BaseFromLyIndexId: indexId,
							IndexCode:         indexCode,
						}
						result = append(result, baseFromLyData)
					} else {
						log.Printf("InternationalPriceProcessor Process() : Column index out of range for row '%s', '%s'", rowVariety, columnName)
					}
					break
				}
			}
		}
	}

	return result, nil
}

// CanadaStatisticsBureauProcessor
// @Description: 加拿大统计局处理器
type CanadaStatisticsBureauProcessor struct{}

func (p *CanadaStatisticsBureauProcessor) Process(ctx context.Context, product string, reportContent string, keywords []string, classifyId int) ([]models.BaseFromLyData, error) {
	logs.Info("Processing Canada statistics bureau...")
	// 解析关键字
	if len(keywords) < 4 {
		return []models.BaseFromLyData{}, fmt.Errorf("CanadaStatisticsBureauProcessor Process() : keywords must contain at least 4 elements")
	}

	// 拿到 行关键字和列关键字
	columnDate := "本周"
	rowVariety := keywords[1]
	// 提取所有表格数据
	tableData := getNoHeadTableData(reportContent)

	// 提取日期信息
	dateText, err := getDateInfo(ctx)
	if err != nil {
		return []models.BaseFromLyData{}, err
	}

	// 时间格式转换
	format, err := utils.ConvertTimeFormat(dateText)
	if err != nil {
		return []models.BaseFromLyData{}, err
	}

	// 处理提取的表格数据
	var result []models.BaseFromLyData

	for _, data := range tableData {
		tableHeaders := data.Headers
		tableRows := data.Rows

		// 查找目标列
		columnIdx := -1
		for i, header := range tableHeaders {
			if strings.Contains(header, columnDate) {
				columnIdx = i
				break
			}
		}
		if columnIdx == -1 {
			log.Printf("CanadaStatisticsBureauProcessor Process() : Column '%s' not found in table", columnDate)
			continue
		}

		// 处理表格中的每一行
		for _, row := range tableRows {
			if len(row) >= len(tableHeaders) {
				if columnIdx < len(row) {
					if row[0] != rowVariety {
						continue
					}
					// 指标名称
					indexName := strings.Join(keywords[:len(keywords)-2], "：")
					// 指标编码
					indexCode := utils.GenerateIndexCode(lySourceName, indexName)
					// 指标id获取
					indexId, err := getIndexId(indexCode, indexName, classifyId, lySourceName, keywords[len(keywords)-2], keywords[len(keywords)-1])
					if err != nil {
						logs.Error("CanadaStatisticsBureauProcessor Process() : Failed to get index id: %v", err)
						continue
					}

					indexData, err := models.GetLyDataByIndexIdAndDataTime(indexId, format)
					if err != nil {
						logs.Error("CanadaStatisticsBureauProcessor Process() : Failed to get data by index id and date: %v", err)
						continue
					}
					if len(indexData) > 0 {
						logs.Info("CanadaStatisticsBureauProcessor Process() : Data already exists for index %d and date %s", indexId, dateText)
						// 无需更新 指标展示本周的数据，报告每周更新，即使本周和上周数据一致，也需要每周记录，如果到这里也只是说，今天这个报告被读取了两次
						continue
					}

					valueStr := row[columnIdx]
					value, err := strconv.ParseFloat(valueStr, 64)
					if err != nil {
						return []models.BaseFromLyData{}, fmt.Errorf("failed to parse value '%s': %v", valueStr, err)
					}
					// 创建并添加到结果列表
					baseFromLyData := models.BaseFromLyData{
						DataTime:          format,
						Value:             value,
						BaseFromLyIndexId: indexId,
						IndexCode:         indexCode,
					}
					result = append(result, baseFromLyData)
				} else {
					log.Printf("CanadaStatisticsBureauProcessor Process() : Column index out of range for row '%s'", columnDate)
				}
				break
			}
		}
	}

	return result, nil
}

// ImportExportAnalysisProcessor
// @Description: 进出口分析处理器
type ImportExportAnalysisProcessor struct{}

func (p *ImportExportAnalysisProcessor) Process(ctx context.Context, product string, reportContent string, keywords []string, classifyId int) ([]models.BaseFromLyData, error) {
	fmt.Println("Processing processing profit...")
	// 解析关键字
	if len(keywords) < 3 {
		return []models.BaseFromLyData{}, fmt.Errorf("ImportExportAnalysisProcessor Process() : keywords must contain at least 3 elements")
	}

	// 拿到 行关键字和列关键字
	var columnDates []string
	// 提取所有表格数据
	tableData := getNoHeadTableData(reportContent)

	// 提取日期信息
	dateText, err := getDateInfo(ctx)
	if err != nil {
		return []models.BaseFromLyData{}, err
	}

	// 时间格式转换
	format, err := utils.ConvertTimeFormat(dateText)
	if err != nil {
		return []models.BaseFromLyData{}, err
	}

	// 2025年1月可能才出2024年12月的数据，所以往前取一年
	columnDates, err = utils.GetCurrentYearAndLastYear(format)
	if err != nil {
		return nil, err
	}

	// 处理提取的表格数据
	var result []models.BaseFromLyData

	for _, data := range tableData {
		tableHeaders := data.Headers
		tableRows := data.Rows

		for _, columnDate := range columnDates {
			// 查找目标列
			columnIdx := -1
			for i, header := range tableHeaders {
				if strings.Contains(header, columnDate) {
					columnIdx = i
					break
				}
			}

			if columnIdx == -1 {
				log.Printf("ImportExportAnalysisProcessor Process() : Column '%s' not found in table", columnDate)
				continue
			}

			// 处理表格中的每一行
			for _, row := range tableRows {
				if len(row) >= len(tableHeaders) {
					if columnIdx < len(row) && isNumber(row[columnIdx]) && isNumber(row[0]) {
						// 指标名称
						indexName := strings.Join(keywords[:len(keywords)-2], "：")
						// 指标编码
						indexCode := utils.GenerateIndexCode(lySourceName, indexName)
						// 指标id获取
						indexId, err := getIndexId(indexCode, indexName, classifyId, lySourceName, keywords[len(keywords)-2], keywords[len(keywords)-1])
						if err != nil {
							logs.Error("ImportExportAnalysisProcessor Process() : Failed to get index id: %v", err)
							continue
						}
						atoi, err := strconv.Atoi(row[0])
						if err != nil {
							return nil, err
						}
						date := columnDate[:4] + "-" + fmt.Sprintf("%02d", atoi)
						lastDayOfMonth, err := utils.GetLastDayOfMonth(date)
						if err != nil {
							return nil, err
						}

						indexData, err := models.GetLyDataByIndexIdAndDataTime(indexId, lastDayOfMonth)
						if err != nil {
							logs.Error("ImportExportAnalysisProcessor Process() : Failed to get data by index id and date: %v", err)
							continue
						}
						if len(indexData) > 0 {
							logs.Info("ImportExportAnalysisProcessor Process() : Data already exists for index %d and date %s", indexId, dateText)
							continue
						}

						valueStr := row[columnIdx]
						value, err := strconv.ParseFloat(valueStr, 64)
						if err != nil {
							return []models.BaseFromLyData{}, fmt.Errorf("failed to parse value '%s': %v", valueStr, err)
						}
						// 创建并添加到结果列表
						baseFromLyData := models.BaseFromLyData{
							DataTime:          lastDayOfMonth,
							Value:             value,
							BaseFromLyIndexId: indexId,
							IndexCode:         indexCode,
						}
						result = append(result, baseFromLyData)
						continue
					} else {
						log.Printf("ImportExportAnalysisProcessor Process() : Column index out of range for row '%s'", columnDate)
					}
					break
				}
			}
		}
	}

	return result, nil
}

// ExtractValueInParentheses 从字符串中提取括号中的值
func extractValueInParentheses(input string) (string, error) {
	re := regexp.MustCompile(`（([^）]+)）`)
	matches := re.FindStringSubmatch(input)

	if len(matches) > 1 {
		return matches[1], nil
	}

	return "", fmt.Errorf("no value found in parentheses")
}

// 获取指标id，根据指标名称判断，没有插入指标生成返回
func getIndexId(indexCode string, indexName string, classifyId int, lySourceName string, frequency string, unit string) (int, error) {
	// 判断指标是否存在
	var indexId int
	indexInfo, err := models.GetLyIndexByCode(indexCode)
	if err != nil {
		// 新增指标
		index, err := addLyIndex(classifyId, indexCode, indexName, frequency, unit)
		if err != nil {
			return 0, err
		}
		indexId = index
	} else {
		indexId = indexInfo.BaseFromLyIndexId
	}
	return indexId, nil
}

// 获取页面时间信息
func getDateInfo(ctx context.Context) (string, error) {
	var dateText string
	err := chromedp.Run(ctx,
		chromedp.Evaluate(`document.querySelector('div.a_date span').innerText`, &dateText),
	)
	if err != nil {
		return "", fmt.Errorf("processing Process() : Failed to extract report date: %v", err)
	}

	logs.Info("Processing Process() : Report Extracted Date: %s", dateText)
	return dateText, nil
}

// 获取所有表格数据 获取表格中有thead标签的数据
func getAllTableData(reportContent string) []TableData {
	var tableData []TableData

	doc, err := goquery.NewDocumentFromReader(strings.NewReader(reportContent))
	if err != nil {
		log.Fatal(err)
	}

	// 选择 id 为 "a_content" 的 div
	doc.Find("#a_content").Each(func(index int, item *goquery.Selection) {
		item.Find("table").Each(func(index int, table *goquery.Selection) {
			var headers []string
			var rows [][]string

			// 提取表头
			table.Find("thead th").Each(func(index int, th *goquery.Selection) {
				headers = append(headers, th.Text())
			})

			// 提取表格行数据
			table.Find("tbody tr").Each(func(index int, row *goquery.Selection) {
				var rowData []string
				row.Find("td").Each(func(index int, td *goquery.Selection) {
					rowData = append(rowData, td.Text())
				})
				rows = append(rows, rowData)
			})

			// 仅在表头存在时添加到结果中
			if len(headers) > 0 {
				tableData = append(tableData, TableData{
					Headers: headers,
					Rows:    rows,
				})
			}
		})
	})
	return tableData
}

// 获取无头表格数据
func getNoHeadTableData(reportContent string) []TableData {
	var tableData []TableData

	doc, err := goquery.NewDocumentFromReader(strings.NewReader(reportContent))
	if err != nil {
		log.Fatal(err)
	}

	// Find the div with id "a_content"
	doc.Find("#a_content").Each(func(index int, div *goquery.Selection) {
		// Find all tables within the div
		div.Find("table").Each(func(index int, table *goquery.Selection) {
			var headers []string
			var rows [][]string

			// Extract table headers if any
			table.Find("tr").Each(func(index int, tr *goquery.Selection) {
				var rowData []string
				tr.Find("td, th").Each(func(index int, cell *goquery.Selection) {
					cellText := cell.Text()
					rowData = append(rowData, cellText)
				})

				if index == 0 && len(rowData) > 0 {
					// The first row is treated as the header row
					headers = rowData
				} else if len(rowData) > 0 {
					// Add the row data to the rows slice
					rows = append(rows, rowData)
				}
			})

			// Only add table data if headers are present
			if len(headers) > 0 {
				tableData = append(tableData, TableData{
					Headers: headers,
					Rows:    rows,
				})
			}
		})
	})

	return tableData
}

// 获取表格数据 获取id 为 a_content 的 div 中的第一个表格 左上角那个单元格会拼在第一个，会拼上列上的合并单元格
func getTableData(reportContent string, isFirst bool) TableData {
	doc, err := goquery.NewDocumentFromReader(strings.NewReader(reportContent))
	if err != nil {
		log.Fatal(err)
	}

	tableData := &TableData{}

	// 只提取 id 为 a_content 的 div 中的第一个表格
	var firstTable *goquery.Selection
	if isFirst {
		firstTable = doc.Find("div#a_content table").First()
	} else {
		firstTable = doc.Find("div#a_content table").Last()
	}

	var combinedHeaders []string

	// 提取表头
	firstTable.Find("tr").Each(func(i int, row *goquery.Selection) {
		if i == 0 {
			// 第一行处理合并单元格，保存到 combinedHeaders
			row.Find("td,th").Each(func(j int, cell *goquery.Selection) {
				if j == 0 {
					// 把左上角的“年度（10/9月)”放入 Headers 第一个元素
					tableData.Headers = append(tableData.Headers, strings.TrimSpace(cell.Text()))
				} else {
					// 处理其他单元格
					colspan, exists := cell.Attr("colspan")
					if exists {
						spanCount := 0
						fmt.Sscanf(colspan, "%d", &spanCount)
						for k := 0; k < spanCount; k++ {
							combinedHeaders = append(combinedHeaders, strings.TrimSpace(cell.Text()))
						}
					} else {
						combinedHeaders = append(combinedHeaders, strings.TrimSpace(cell.Text()))
					}
				}
			})
		} else if i == 1 {
			// 第二行处理具体标题，组合后保存到 Headers
			row.Find("td,th").Each(func(j int, cell *goquery.Selection) {
				if j < len(combinedHeaders) {
					fullHeader := combinedHeaders[j] + strings.TrimSpace(cell.Text())
					tableData.Headers = append(tableData.Headers, fullHeader)
				}
			})
		} else {
			// 处理数据行
			var rowData []string
			row.Find("td").Each(func(j int, cell *goquery.Selection) {
				rowData = append(rowData, strings.TrimSpace(cell.Text()))
			})
			if len(rowData) > 0 {
				tableData.Rows = append(tableData.Rows, rowData)
			}
		}
	})

	return *tableData
}

// 获取采购装船表格数据
func getPurchaseShippingTableData(reportContent string) TableData {
	doc, err := goquery.NewDocumentFromReader(strings.NewReader(reportContent))
	if err != nil {
		log.Fatal(err)
	}

	var tableData TableData

	// 只提取 id 为 a_content 的 div 中的第一个表格
	firstTable := doc.Find("div#a_content table").First()

	// 提取表头
	var headers []string
	var subHeaders []string

	firstTable.Find("thead tr").Each(func(i int, row *goquery.Selection) {
		row.Find("th").Each(func(j int, cell *goquery.Selection) {
			headerText := strings.TrimSpace(cell.Text())

			if i == 0 {
				// 处理第一行表头
				colspan, exists := cell.Attr("colspan")
				if exists {
					spanCount := 0
					fmt.Sscanf(colspan, "%d", &spanCount)
					for k := 0; k < spanCount; k++ {
						headers = append(headers, headerText)
					}
				} else {
					headers = append(headers, headerText)
				}
			} else if i == 1 {
				// 处理第二行表头
				subHeaders = append(subHeaders, headerText)
			}
		})
	})

	// 合并第一行和第二行表头信息
	if len(subHeaders) > 0 {
		for i := 0; i < len(subHeaders); i++ {
			// 从第四个单元格开始拼接
			headers[3+i] = headers[3+i] + subHeaders[i]
		}
	}

	tableData.Headers = headers

	// 处理数据行
	firstTable.Find("tbody tr").Each(func(i int, row *goquery.Selection) {
		var rowData []string
		row.Find("td").Each(func(j int, cell *goquery.Selection) {
			rowData = append(rowData, strings.TrimSpace(cell.Text()))
		})
		if len(rowData) > 0 {
			tableData.Rows = append(tableData.Rows, rowData)
		}
	})

	return tableData
}

// 判断字符串是否是数字
func isNumeric(value string) bool {
	// 正则表达式匹配整数和浮点数
	re := regexp.MustCompile(`^[+-]?(\d+(\.\d*)?|\.\d+)$`)
	return re.MatchString(value)
}

// 只保留汉字
func extractChinese(text string) string {
	re := regexp.MustCompile(`[^\p{Han}]`) // 匹配非汉字字符
	return re.ReplaceAllString(text, "")
}

// 去除括号中的内容 包含括号 ()
func removeParentheses(text string) string {
	re := regexp.MustCompile(`\([^)]*\)`)
	return re.ReplaceAllString(text, "")
}

// IsChinese 判断传入的是否是汉字
func IsChinese(str string) bool {
	for _, r := range str {
		if unicode.Is(unicode.Han, r) {
			return true
		}
	}
	return false
}

// 判断是否是数字
func isNumber(str string) bool {
	_, err := strconv.ParseFloat(str, 64)
	return err == nil
}