优化

services/ruizide/oil_supply_analysis_processor.go

@@ -0,0 +1,193 @@
+package ruizide
+
+import (
+	"encoding/json"
+	"eta/eta_data_analysis/models"
+	"eta/eta_data_analysis/utils"
+	"fmt"
+	"github.com/shopspring/decimal"
+	"github.com/xuri/excelize/v2"
+	"os"
+	"path/filepath"
+	"strings"
+)
+
+const (
+	OilSupplyAnalysis = "Oil Supply Analysis"
+	OsaBeginRow       = 1
+	OsaRows           = 3
+)
+
+var (
+	OilSupplyAnalysisConfigMap = map[string]OSAPConfig{
+		"Chart 1": {
+			Frequency:          "月度",
+			Unit:               "千桶每天",
+			IndexNameColSuffix: "Oil Classification Group",
+			IndexRow:           1,
+			DataRow:            2,
+			DateRow:            0,
+		},
+		"Chart 3": {
+			Frequency:          "月度",
+			Unit:               "千桶每天",
+			IndexNameColSuffix: "CapacityDetail",
+			IndexRow:           1,
+			DataRow:            2,
+			DateRow:            0,
+		},
+		"Chart 4": {
+			Frequency:          "月度",
+			Unit:               "千桶每天",
+			IndexNameColSuffix: "Region",
+			IndexRow:           2,
+			DataRow:            1,
+			DateRow:            0,
+		},
+		"Chart 5": {
+			Frequency:          "月度",
+			Unit:               "千桶每天",
+			IndexNameColSuffix: "OilAndGasCategory",
+			IndexRow:           1,
+			DataRow:            2,
+			DateRow:            0,
+		},
+	}
+)
+
+type OSAPConfig struct {
+	Frequency          string
+	Unit               string
+	IndexNameColSuffix string
+	IndexRow           int
+	DataRow            int
+	DateRow            int
+}
+
+type OilSupplyAnalysisProcessor struct {
+}
+
+func (p *OilSupplyAnalysisProcessor) Process(tableName string) (err error) {
+	var fileName string
+	// 解析表格
+	fileName = tableName + "_" + utils.GetCurrentYearMonth() + ".xlsx"
+	filePath := filepath.Join(excelDir, fileName)
+	if _, err = os.Stat(filePath); os.IsNotExist(err) {
+		utils.FileLog.Error("文件不存在: %v", err)
+		return
+	}
+	// 打开 Excel 文件
+	file, err := excelize.OpenFile(filePath)
+	if err != nil {
+		utils.FileLog.Error("无法打开 Excel 文件: %v", err)
+		return
+	}
+	sheetNames := file.GetSheetList()
+	classifyIds, err := dealClassify(CubeDashboards, []string{OilSupplyAnalysis})
+	// 获取所有工作表
+	if err != nil {
+		utils.FileLog.Error("获取分类Id失败: %v", err)
+		return
+	}
+	//按照sheet页来处理数据,一个sheet发送一次服务器
+	for _, sheetName := range sheetNames {
+		var indexMap = make(map[string]*models.IndexInfo)
+		// 获取工作表的最大行数
+		dataRows, excelErr := file.GetRows(sheetName) // 直接获取所有行数据
+		if excelErr != nil {
+			utils.FileLog.Error("获取工作表数据时出错: %v", excelErr)
+			continue
+		}
+		if sheetConfig, ok := OilSupplyAnalysisConfigMap[sheetName]; ok {
+			// 匹配非数字字符
+			classifyId := classifyIds[OilSupplyAnalysis]
+			indexMap, err = DataHandler(dataRows, sheetConfig, classifyId)
+			// 新增数据源指标数据
+			if len(indexMap) > 0 {
+				for _, index := range indexMap {
+					pars := chunkIndexData(index, rzdBatchSize)
+					for _, par := range pars {
+						// 转换成json
+						marshal, parseErr := json.Marshal(par)
+						if parseErr != nil {
+							utils.FileLog.Error("json.Marshal err: %v", parseErr)
+							return parseErr
+						}
+						// 发送 HTTP POST 请求
+						_, err = utils.HttpPostRequest(utils.EDB_LIB_URL+utils.ADD_BATCH_RZD_DATA, string(marshal), "application/json")
+						if err != nil {
+							utils.FileLog.Error("postEdbLib err: %v", err)
+							return err
+						}
+					}
+				}
+			}
+		}
+	}
+	return
+}
+func DataHandler(dataRows [][]string, sheetConfig OSAPConfig, classifyId int) (indexMap map[string]*models.IndexInfo, err error) {
+	indexMap = make(map[string]*models.IndexInfo)
+	for _, rowData := range dataRows[BeginRow : len(dataRows)-1] {
+		less := Rows - len(rowData)
+		if less > 0 {
+			for i := 0; i < less; i++ {
+				rowData = append(rowData, "")
+			}
+		}
+		// step_2: 指标
+		// 指标名称
+		indexName := OilSupplyAnalysis + "/" + sheetConfig.IndexNameColSuffix + "/" + rowData[sheetConfig.IndexRow]
+		// 生成指标编码
+		indexCode := getIndexId(fmt.Sprintf("%s %s", OilSupplyAnalysis, sheetConfig.IndexNameColSuffix), strings.ReplaceAll(strings.ToLower(rowData[sheetConfig.IndexRow]), " ", ""), "")
+		dataTime := rowData[sheetConfig.DateRow]
+		if dataTime == "0" {
+			continue
+		}
+
+		date, _, convertErr := utils.ConvertDateFormatForOSA(dataTime)
+		if convertErr != nil {
+			utils.FileLog.Error(fmt.Sprintf("转换时间数据失败,index_code:%s,time_value:%s err:%v", indexCode, dataTime, convertErr))
+			continue
+		}
+		if rowData[sheetConfig.DataRow] != "" {
+			value, parseErr := decimal.NewFromString(rowData[sheetConfig.DataRow])
+			if parseErr != nil {
+				utils.FileLog.Error(fmt.Sprintf("转换data数据失败,index_code:%s,data_value:%s err:%v", indexCode, rowData[sheetConfig.DataRow], err))
+				continue
+			} else {
+				if index, ok := indexMap[indexCode]; ok {
+					if index.StartDate.After(date) {
+						index.StartDate = date
+					}
+					if index.EndDate.Before(date) {
+						index.EndDate = date
+						index.LatestValue = value
+					}
+					index.DataList = append(index.DataList, models.IndexData{
+						DataTime: date,
+						Value:    value,
+					})
+				} else {
+					indexMap[indexCode] = &models.IndexInfo{
+						IndexName:   indexName,
+						IndexCode:   indexCode,
+						Frequency:   sheetConfig.Frequency,
+						Unit:        sheetConfig.Unit,
+						StartDate:   date,
+						EndDate:     date,
+						LatestValue: value,
+						ClassifyId:  classifyId,
+						DataList: []models.IndexData{
+							{
+								DataTime: date,
+								Value:    value,
+							},
+						},
+					}
+				}
+			}
+		}
+	}
+	return
+}

services/ruizide/processor_business_logic.go

@@ -11,275 +11,6 @@ import (
 	"unicode"
 )
 
-//
-//// OilSupplyAnalysisChartOneProcessor
-//// @Description: OilSupplyAnalysisChartOneProcessor处理器
-//type OilSupplyAnalysisChartOneProcessor struct{}
-//
-//func (p *OilSupplyAnalysisChartOneProcessor) Process(tableName string, sheetName string, rowIndex int, rowData []string) ([]models.BaseFromRzdData, error) {
-//	utils.FileLog.Info("Processing OilSupplyAnalysisChartOne...")
-//	if rowIndex < 1 {
-//		return nil, nil
-//	}
-//
-//	frequency := "月度"
-//	unit := "千桶每天"
-//	indexNameColSuffix := "Oil And Gas Category"
-//	//indexNameColPrefix := "CountryRevisionGroup"
-//
-//	// step_1: 分类
-//	classifyId, err := dealClassify("cube dashboards", "Oil Supply Analysis")
-//	if err != nil {
-//		return nil, err
-//	}
-//	utils.FileLog.Info("classifyId: %v", classifyId)
-//
-//	// step_2: 指标
-//	// 指标名称
-//	indexName := "Oil Supply Analysis" + "/" + indexNameColSuffix + "/" + rowData[len(rowData)-2]
-//
-//	// 生成指标编码
-//	indexCode, err := getIndexId("Oil Supply Analysis "+indexNameColSuffix, strings.ReplaceAll(strings.ToLower(rowData[len(rowData)-2]), " ", ""), "")
-//
-//	indexInfoMap := make(map[string]string)
-//	indexInfoMap[indexCode] = indexName
-//
-//	var indexInfoList []*models.IndexInfo
-//	value, err := strconv.ParseFloat(rowData[len(rowData)-1], 64)
-//	if err != nil {
-//		return nil, err
-//	}
-//
-//	dataTime := rowData[0]
-//	format, err := utils.ConvertDateFormat3(dataTime)
-//	if err != nil {
-//		return nil, err
-//	}
-//
-//	indexInfoList = append(indexInfoList, &models.IndexInfo{
-//		IndexName: indexName,
-//		IndexCode: indexCode,
-//		Value:     value,
-//		DataTime:  format,
-//	})
-//
-//	indexInfoList, err = dealIndex(indexInfoList, frequency, unit, classifyId)
-//	if err != nil {
-//		return nil, err
-//	}
-//	utils.FileLog.Info("OilSupplyAnalysisChartOneProcessor indexInfoList: %v", indexInfoList)
-//
-//	// step_3: 指标数据
-//	dataList, err := dealData(indexInfoList)
-//	if err != nil {
-//		return nil, err
-//	}
-//	utils.FileLog.Info("OilSupplyAnalysisChartOneProcessor dataList: %v", dataList)
-//
-//	return dataList, err
-//}
-//
-//// OilSupplyAnalysisChartTwoProcessor
-//// @Description: OilSupplyAnalysisChartTwoProcessor处理器
-//type OilSupplyAnalysisChartTwoProcessor struct{}
-//
-//func (p *OilSupplyAnalysisChartTwoProcessor) Process(tableName string, sheetName string, rowIndex int, rowData []string) ([]models.BaseFromRzdData, error) {
-//	utils.FileLog.Info("Processing OilSupplyAnalysisChartTwoProcessor...")
-//	if rowIndex < 1 {
-//		return nil, nil
-//	}
-//
-//	frequency := "月度"
-//	unit := "千桶每天"
-//	indexNameColSuffix := "Region"
-//	//indexNameColPrefix := "CountryRevisionGroup"
-//
-//	// step_1: 分类
-//	classifyId, err := dealClassify("cube dashboards", "Oil Supply Analysis")
-//	if err != nil {
-//		return nil, err
-//	}
-//	utils.FileLog.Info("classifyId: %v", classifyId)
-//
-//	// step_2: 指标
-//	// 指标名称
-//	indexName := "Oil Supply Analysis" + "/" + indexNameColSuffix + "/" + rowData[len(rowData)-1]
-//
-//	// 生成指标编码
-//	indexCode, err := getIndexId("Oil Supply Analysis "+indexNameColSuffix, strings.ReplaceAll(strings.ToLower(rowData[len(rowData)-1]), " ", ""), "")
-//
-//	indexInfoMap := make(map[string]string)
-//	indexInfoMap[indexCode] = indexName
-//
-//	var indexInfoList []*models.IndexInfo
-//	value, err := strconv.ParseFloat(rowData[len(rowData)-2], 64)
-//	if err != nil {
-//		return nil, err
-//	}
-//
-//	dataTime := rowData[0]
-//	format, err := utils.ConvertDateFormat3(dataTime)
-//	if err != nil {
-//		return nil, err
-//	}
-//
-//	indexInfoList = append(indexInfoList, &models.IndexInfo{
-//		IndexName: indexName,
-//		IndexCode: indexCode,
-//		Value:     value,
-//		DataTime:  format,
-//	})
-//
-//	indexInfoList, err = dealIndex(indexInfoList, frequency, unit, classifyId)
-//	if err != nil {
-//		return nil, err
-//	}
-//	utils.FileLog.Info("OilSupplyAnalysisChartTwoProcessor indexInfoList: %v", indexInfoList)
-//
-//	// step_3: 指标数据
-//	dataList, err := dealData(indexInfoList)
-//	if err != nil {
-//		return nil, err
-//	}
-//	utils.FileLog.Info("OilSupplyAnalysisChartTwoProcessor dataList: %v", dataList)
-//
-//	return dataList, err
-//}
-//
-//// OilSupplyAnalysisChartThreeProcessor
-//// @Description: OilSupplyAnalysisChartThreeProcessor处理器
-//type OilSupplyAnalysisChartThreeProcessor struct{}
-//
-//func (p *OilSupplyAnalysisChartThreeProcessor) Process(tableName string, sheetName string, rowIndex int, rowData []string) ([]models.BaseFromRzdData, error) {
-//	utils.FileLog.Info("Processing OilSupplyAnalysisChartThreeProcessor...")
-//	if rowIndex < 1 {
-//		return nil, nil
-//	}
-//
-//	frequency := "月度"
-//	unit := "千桶每天"
-//	indexNameColSuffix := "CapacityDetail"
-//	//indexNameColPrefix := "CountryRevisionGroup"
-//
-//	// step_1: 分类
-//	classifyId, err := dealClassify("cube dashboards", "Oil Supply Analysis")
-//	if err != nil {
-//		return nil, err
-//	}
-//	utils.FileLog.Info("classifyId: %v", classifyId)
-//
-//	// step_2: 指标
-//	// 指标名称
-//	indexName := "Oil Supply Analysis" + "/" + indexNameColSuffix + "/" + rowData[len(rowData)-2]
-//
-//	// 生成指标编码
-//	indexCode, err := getIndexId("Oil Supply Analysis "+indexNameColSuffix, strings.ReplaceAll(strings.ToLower(rowData[len(rowData)-2]), " ", ""), "")
-//
-//	indexInfoMap := make(map[string]string)
-//	indexInfoMap[indexCode] = indexName
-//
-//	var indexInfoList []*models.IndexInfo
-//	value, err := strconv.ParseFloat(rowData[len(rowData)-1], 64)
-//	if err != nil {
-//		return nil, err
-//	}
-//
-//	dataTime := rowData[0]
-//	format, err := utils.ConvertDateFormat3(dataTime)
-//	if err != nil {
-//		return nil, err
-//	}
-//
-//	indexInfoList = append(indexInfoList, &models.IndexInfo{
-//		IndexName: indexName,
-//		IndexCode: indexCode,
-//		Value:     value,
-//		DataTime:  format,
-//	})
-//
-//	indexInfoList, err = dealIndex(indexInfoList, frequency, unit, classifyId)
-//	if err != nil {
-//		return nil, err
-//	}
-//	utils.FileLog.Info("OilSupplyAnalysisChartThreeProcessor indexInfoList: %v", indexInfoList)
-//
-//	// step_3: 指标数据
-//	dataList, err := dealData(indexInfoList)
-//	if err != nil {
-//		return nil, err
-//	}
-//	utils.FileLog.Info("OilSupplyAnalysisChartThreeProcessor dataList: %v", dataList)
-//
-//	return dataList, err
-//}
-//
-//// OilSupplyAnalysisChartFourProcessor
-//// @Description: OilSupplyAnalysisChartFourProcessor处理器
-//type OilSupplyAnalysisChartFourProcessor struct{}
-//
-//func (p *OilSupplyAnalysisChartFourProcessor) Process(tableName string, sheetName string, rowIndex int, rowData []string) ([]models.BaseFromRzdData, error) {
-//	utils.FileLog.Info("Processing OilSupplyAnalysisChartFourProcessor...")
-//	if rowIndex < 1 {
-//		return nil, nil
-//	}
-//
-//	frequency := "月度"
-//	unit := "千桶每天"
-//	indexNameColSuffix := "Oil Classification Group"
-//	//indexNameColPrefix := "CountryRevisionGroup"
-//
-//	// step_1: 分类
-//	classifyId, err := dealClassify("cube dashboards", "Oil Supply Analysis")
-//	if err != nil {
-//		return nil, err
-//	}
-//	utils.FileLog.Info("classifyId: %v", classifyId)
-//
-//	// step_2: 指标
-//	// 指标名称
-//	indexName := "Oil Supply Analysis" + "/" + indexNameColSuffix + "/" + rowData[len(rowData)-2]
-//
-//	// 生成指标编码
-//	indexCode, err := getIndexId("Oil Supply Analysis "+indexNameColSuffix, strings.ReplaceAll(strings.ToLower(rowData[len(rowData)-2]), " ", ""), "")
-//
-//	indexInfoMap := make(map[string]string)
-//	indexInfoMap[indexCode] = indexName
-//
-//	var indexInfoList []*models.IndexInfo
-//	value, err := strconv.ParseFloat(rowData[len(rowData)-1], 64)
-//	if err != nil {
-//		return nil, err
-//	}
-//
-//	dataTime := rowData[0]
-//	format, err := utils.ConvertDateFormat3(dataTime)
-//	if err != nil {
-//		return nil, err
-//	}
-//
-//	indexInfoList = append(indexInfoList, &models.IndexInfo{
-//		IndexName: indexName,
-//		IndexCode: indexCode,
-//		Value:     value,
-//		DataTime:  format,
-//	})
-//
-//	indexInfoList, err = dealIndex(indexInfoList, frequency, unit, classifyId)
-//	if err != nil {
-//		return nil, err
-//	}
-//	utils.FileLog.Info("OilSupplyAnalysisChartFourProcessor indexInfoList: %v", indexInfoList)
-//
-//	// step_3: 指标数据
-//	dataList, err := dealData(indexInfoList)
-//	if err != nil {
-//		return nil, err
-//	}
-//	utils.FileLog.Info("OilSupplyAnalysisChartFourProcessor dataList: %v", dataList)
-//
-//	return dataList, err
-//}
-
 // // OilDemandAnalysisContinentProcessor
 // // @Description: OilDemandAnalysisContinentProcessor处理器
 // type OilDemandAnalysisContinentProcessor struct{}

services/ruizide/processor_factory.go

@@ -56,8 +56,8 @@ var (
 			ClassifyName: "Supply Revision Analysis",
 			Sort:         1,
 		},
-		"Oil_Supply_Analysis": {
-			ClassifyName: "Oil_Supply_Analysis",
+		"Oil Supply Analysis": {
+			ClassifyName: "Oil Supply Analysis",
 			Sort:         2,
 		},
 	}
@@ -72,20 +72,6 @@ type RzdDataProcessor interface {
 }
 
 func GetProcessor(tableName string, sheetName string) (ReportProcessor, error) {
-	//if tableName == "Oil_Supply_Analysis" {
-	//	switch sheetName {
-	//	case "Chart1":
-	//		return &OilSupplyAnalysisChartOneProcessor{}, nil
-	//	case "Chart2":
-	//		return &OilSupplyAnalysisChartTwoProcessor{}, nil
-	//	case "Chart3":
-	//		return &OilSupplyAnalysisChartThreeProcessor{}, nil
-	//	case "Chart4":
-	//		return &OilSupplyAnalysisChartFourProcessor{}, nil
-	//	default:
-	//		return nil, fmt.Errorf("unknown sheetName: %s", sheetName)
-	//	}
-	//}
 	//else if tableName == "Oil_Demand_Analysis_Continent" {
 	//	switch sheetName {
 	//	case "Chart1":
@@ -269,6 +255,7 @@ func GetProcessor(tableName string, sheetName string) (ReportProcessor, error) {
 var RzdProcessorMap = map[string]RzdDataProcessor{
 	"Oil_Demand_Signals_Weekly_Report": &OilDemandSignalsWeeklyReportProcessor{},
 	"Supply_Revision_Analysis":         &SupplyRevisionAnalysisProcessor{},
+	"Oil_Supply_Analysis":              &OilSupplyAnalysisProcessor{},
 }
 
 func GetRZDProcessor(tableName string) (RzdDataProcessor, error) {

utils/date_util.go

@@ -576,33 +576,32 @@ func ConvertDateFormatQuarter(dataText string) (date time.Time, dateStr string,
 }
 
 // ConvertDateFormat3 转换时间格式 Jan-23 --> 2023-01-31
-func ConvertDateFormat3(dataText string) (string, error) {
+func ConvertDateFormatForOSA(dataText string) (date time.Time, dateStr string, err error) {
 	// 拆分月份和年份
 	parts := strings.Split(dataText, "-")
 	if len(parts) != 2 {
-		return "", fmt.Errorf("日期格式不正确")
+		err = fmt.Errorf("日期格式不正确")
+		return
 	}
-
 	// 提取月份和年份
 	monthStr := parts[0]
 	yearStr := parts[1]
-
 	// 转换年份
 	year, err := strconv.Atoi(yearStr)
 	if err != nil {
-		return "", fmt.Errorf("无效的年份: %v", err)
+		err = fmt.Errorf("无效的年份: %v", err)
+		return
 	}
-
 	// 获取月份数字
 	month, err := time.Parse("Jan", monthStr)
 	if err != nil {
-		return "", fmt.Errorf("无效的月份: %v", err)
+		err = fmt.Errorf("无效的月份: %v", err)
+		return
 	}
-
 	// 计算最后一天
-	lastDay := time.Date(year+2000, month.Month()+1, 0, 0, 0, 0, 0, time.UTC) // 直接设置为下个月的第一天，再设为0日得到上个月最后一天
-
-	return lastDay.Format("2006-01-02"), nil
+	date = time.Date(year+2000, month.Month()+1, 0, 0, 0, 0, 0, time.UTC) // 直接设置为下个月的第一天，再设为0日得到上个月最后一天
+	dateStr = date.Format("2006-01-02")
+	return
 }
 
 // ConvertDateFormat4 转换时间格式 2022 --> 2022-12-31