package ruizide import ( "encoding/json" "eta/eta_data_analysis/models" "eta/eta_data_analysis/utils" "fmt" "github.com/shopspring/decimal" "github.com/xuri/excelize/v2" "os" "path/filepath" "strings" ) const ( OilSupplyAnalysis = "Oil Supply Analysis" OsaBeginRow = 1 OsaRows = 3 ) var ( OilSupplyAnalysisConfigMap = map[string]OSAPConfig{ "Chart 1": { Frequency: "月度", Unit: "千桶每天", IndexNameColSuffix: "Oil Classification Group", IndexRow: 1, DataRow: 2, DateRow: 0, }, "Chart 3": { Frequency: "月度", Unit: "千桶每天", IndexNameColSuffix: "CapacityDetail", IndexRow: 1, DataRow: 2, DateRow: 0, }, "Chart 4": { Frequency: "月度", Unit: "千桶每天", IndexNameColSuffix: "Region", IndexRow: 2, DataRow: 1, DateRow: 0, }, "Chart 5": { Frequency: "月度", Unit: "千桶每天", IndexNameColSuffix: "OilAndGasCategory", IndexRow: 1, DataRow: 2, DateRow: 0, }, } ) type OSAPConfig struct { Frequency string Unit string IndexNameColSuffix string IndexRow int DataRow int DateRow int } type OilSupplyAnalysisProcessor struct { } func (p *OilSupplyAnalysisProcessor) Process(tableName string) (err error) { var fileName string // 解析表格 fileName = tableName + "_" + utils.GetCurrentYearMonth() + ".xlsx" filePath := filepath.Join(excelDir, fileName) if _, err = os.Stat(filePath); os.IsNotExist(err) { utils.FileLog.Error("文件不存在: %v", err) return } // 打开 Excel 文件 file, err := excelize.OpenFile(filePath) if err != nil { utils.FileLog.Error("无法打开 Excel 文件: %v", err) return } sheetNames := file.GetSheetList() classifyIds, err := dealClassify(CubeDashboards, []string{OilSupplyAnalysis}) // 获取所有工作表 if err != nil { utils.FileLog.Error("获取分类Id失败: %v", err) return } //按照sheet页来处理数据,一个sheet发送一次服务器 for _, sheetName := range sheetNames { var indexMap = make(map[string]*models.IndexInfo) // 获取工作表的最大行数 dataRows, excelErr := file.GetRows(sheetName) // 直接获取所有行数据 if excelErr != nil { utils.FileLog.Error("获取工作表数据时出错: %v", excelErr) continue } if sheetConfig, ok := OilSupplyAnalysisConfigMap[sheetName]; ok { // 匹配非数字字符 classifyId := classifyIds[OilSupplyAnalysis] indexMap, err = DataHandler(dataRows, sheetConfig, classifyId) // 新增数据源指标数据 if len(indexMap) > 0 { for _, index := range indexMap { pars := chunkIndexData(index, rzdBatchSize) for _, par := range pars { // 转换成json marshal, parseErr := json.Marshal(par) if parseErr != nil { utils.FileLog.Error("json.Marshal err: %v", parseErr) return parseErr } // 发送 HTTP POST 请求 _, err = utils.HttpPostRequest(utils.EDB_LIB_URL+utils.ADD_BATCH_RZD_DATA, string(marshal), "application/json") if err != nil { utils.FileLog.Error("postEdbLib err: %v", err) return err } } } } } } return } func DataHandler(dataRows [][]string, sheetConfig OSAPConfig, classifyId int) (indexMap map[string]*models.IndexInfo, err error) { indexMap = make(map[string]*models.IndexInfo) for _, rowData := range dataRows[BeginRow : len(dataRows)-1] { less := Rows - len(rowData) if less > 0 { for i := 0; i < less; i++ { rowData = append(rowData, "") } } // step_2: 指标 // 指标名称 indexName := OilSupplyAnalysis + "/" + sheetConfig.IndexNameColSuffix + "/" + rowData[sheetConfig.IndexRow] // 生成指标编码 indexCode := getIndexId(fmt.Sprintf("%s %s", OilSupplyAnalysis, sheetConfig.IndexNameColSuffix), strings.ReplaceAll(strings.ToLower(rowData[sheetConfig.IndexRow]), " ", ""), "") dataTime := rowData[sheetConfig.DateRow] if dataTime == "0" { continue } date, _, convertErr := utils.ConvertDateFormatForOSA(dataTime) if convertErr != nil { utils.FileLog.Error(fmt.Sprintf("转换时间数据失败,index_code:%s,time_value:%s err:%v", indexCode, dataTime, convertErr)) continue } if rowData[sheetConfig.DataRow] != "" { value, parseErr := decimal.NewFromString(rowData[sheetConfig.DataRow]) if parseErr != nil { utils.FileLog.Error(fmt.Sprintf("转换data数据失败,index_code:%s,data_value:%s err:%v", indexCode, rowData[sheetConfig.DataRow], err)) continue } else { if index, ok := indexMap[indexCode]; ok { if index.StartDate.After(date) { index.StartDate = date } if index.EndDate.Before(date) { index.EndDate = date index.LatestValue = value } index.DataList = append(index.DataList, models.IndexData{ DataTime: date, Value: value, }) } else { indexMap[indexCode] = &models.IndexInfo{ IndexName: indexName, IndexCode: indexCode, Frequency: sheetConfig.Frequency, Unit: sheetConfig.Unit, StartDate: date, EndDate: date, LatestValue: value, ClassifyId: classifyId, DataList: []models.IndexData{ { DataTime: date, Value: value, }, }, } } } } } return }