|
|
@@ -0,0 +1,399 @@
|
|
|
+package ruizide
|
|
|
+
|
|
|
+import (
|
|
|
+ "encoding/json"
|
|
|
+ "eta/eta_data_analysis/models"
|
|
|
+ "eta/eta_data_analysis/utils"
|
|
|
+ "fmt"
|
|
|
+ "github.com/shopspring/decimal"
|
|
|
+ "github.com/xuri/excelize/v2"
|
|
|
+ "os"
|
|
|
+ "path/filepath"
|
|
|
+ "regexp"
|
|
|
+ "strings"
|
|
|
+ "time"
|
|
|
+)
|
|
|
+
|
|
|
+const (
|
|
|
+ CubeDashboards = "cube dashboards"
|
|
|
+ SupplyRevisionAnalysis = "Supply Revision Analysis"
|
|
|
+ BeginRow = 1
|
|
|
+ Rows = 3
|
|
|
+)
|
|
|
+
|
|
|
+var (
|
|
|
+ SupplyRevisionAnalysisConfigMap = map[string]SRAPConfig{
|
|
|
+ "Chart 1": {
|
|
|
+ Frequency: "季度",
|
|
|
+ Unit: "千桶每天",
|
|
|
+ IndexNameColSuffix: "Index",
|
|
|
+ IndexNameColPrefix: "Index 7DMA",
|
|
|
+ chartType: "one",
|
|
|
+ IndexRow: 2,
|
|
|
+ DateRow: 1,
|
|
|
+ DateType: "quarter",
|
|
|
+ },
|
|
|
+ "Chart 2": {
|
|
|
+ Frequency: "季度",
|
|
|
+ Unit: "千桶每天",
|
|
|
+ IndexNameColSuffix: "Index",
|
|
|
+ IndexNameColPrefix: "Index 7DMA",
|
|
|
+ chartType: "two",
|
|
|
+ NamePrevious: "",
|
|
|
+ NameCurrent: "",
|
|
|
+ CodePrevious: "",
|
|
|
+ CodeCurrent: "",
|
|
|
+ IndexRow: 2,
|
|
|
+ DataPreviousRow: 4,
|
|
|
+ DataCurrentRow: 5,
|
|
|
+ DateRow: 1,
|
|
|
+ DateType: "quarter",
|
|
|
+ },
|
|
|
+ "Chart 3": {
|
|
|
+ Frequency: "年度",
|
|
|
+ Unit: "千桶每天",
|
|
|
+ IndexNameColSuffix: "Index",
|
|
|
+ IndexNameColPrefix: "Index 7DMA",
|
|
|
+ chartType: "one",
|
|
|
+ IndexRow: 2,
|
|
|
+ DateRow: 1,
|
|
|
+ DateType: "year",
|
|
|
+ },
|
|
|
+ "Chart 4": {
|
|
|
+ Frequency: "月度",
|
|
|
+ Unit: "千桶每天",
|
|
|
+ IndexNameColSuffix: "Index",
|
|
|
+ IndexNameColPrefix: "Index 7DMA",
|
|
|
+ chartType: "two",
|
|
|
+ NamePrevious: "",
|
|
|
+ NameCurrent: "",
|
|
|
+ CodePrevious: "",
|
|
|
+ CodeCurrent: "",
|
|
|
+ IndexRow: 2,
|
|
|
+ DataPreviousRow: 4,
|
|
|
+ DataCurrentRow: 5,
|
|
|
+ DateRow: 1,
|
|
|
+ DateType: "year",
|
|
|
+ },
|
|
|
+ "Chart 5": {
|
|
|
+ Frequency: "月度",
|
|
|
+ Unit: "千桶每天",
|
|
|
+ IndexNameColSuffix: "Index",
|
|
|
+ IndexNameColPrefix: "Index 7DMA",
|
|
|
+ chartType: "two",
|
|
|
+ NamePrevious: "",
|
|
|
+ NameCurrent: "",
|
|
|
+ CodePrevious: "",
|
|
|
+ CodeCurrent: "",
|
|
|
+ IndexRow: 3,
|
|
|
+ DataPreviousRow: 4,
|
|
|
+ DataCurrentRow: 5,
|
|
|
+ DateRow: 3,
|
|
|
+ DateType: "month",
|
|
|
+ },
|
|
|
+ "Chart 6": {
|
|
|
+ Frequency: "日度",
|
|
|
+ Unit: "千桶每天",
|
|
|
+ IndexNameColSuffix: "Index",
|
|
|
+ IndexNameColPrefix: "Index 7DMA",
|
|
|
+ chartType: "one",
|
|
|
+ IndexRow: 3,
|
|
|
+ DateRow: 1,
|
|
|
+ DateType: "month",
|
|
|
+ },
|
|
|
+ }
|
|
|
+)
|
|
|
+
|
|
|
+type SRAPConfig struct {
|
|
|
+ Frequency string
|
|
|
+ Unit string
|
|
|
+ IndexNameColSuffix string
|
|
|
+ IndexNameColPrefix string
|
|
|
+ IndexRow int
|
|
|
+ DataPreviousRow int
|
|
|
+ DataCurrentRow int
|
|
|
+ DateRow int
|
|
|
+ DateType string
|
|
|
+ chartType string
|
|
|
+ NamePrevious string
|
|
|
+ NameCurrent string
|
|
|
+ CodePrevious string
|
|
|
+ CodeCurrent string
|
|
|
+}
|
|
|
+
|
|
|
+type SupplyRevisionAnalysisProcessor struct {
|
|
|
+}
|
|
|
+
|
|
|
+func (p *SupplyRevisionAnalysisProcessor) Process(tableName string) (err error) {
|
|
|
+ var fileName string
|
|
|
+ // 解析表格
|
|
|
+ fileName = tableName + "_" + utils.GetCurrentYearMonth() + ".xlsx"
|
|
|
+ filePath := filepath.Join(excelDir, fileName)
|
|
|
+ if _, err = os.Stat(filePath); os.IsNotExist(err) {
|
|
|
+ utils.FileLog.Error("文件不存在: %v", err)
|
|
|
+ return
|
|
|
+ }
|
|
|
+ // 打开 Excel 文件
|
|
|
+ file, err := excelize.OpenFile(filePath)
|
|
|
+ if err != nil {
|
|
|
+ utils.FileLog.Error("无法打开 Excel 文件: %v", err)
|
|
|
+ return
|
|
|
+ }
|
|
|
+ sheetNames := file.GetSheetList()
|
|
|
+ classifyIds, err := dealClassify(CubeDashboards, []string{SupplyRevisionAnalysis})
|
|
|
+ // 获取所有工作表
|
|
|
+ if err != nil {
|
|
|
+ utils.FileLog.Error("获取分类Id失败: %v", err)
|
|
|
+ return
|
|
|
+ }
|
|
|
+ //按照sheet页来处理数据,一个sheet发送一次服务器
|
|
|
+ for _, sheetName := range sheetNames {
|
|
|
+ var indexMap = make(map[string]*models.IndexInfo)
|
|
|
+ utils.FileLog.Info("读取工作表: %s\n", sheetName)
|
|
|
+ // 获取工作表的最大行数
|
|
|
+ dataRows, excelErr := file.GetRows(sheetName) // 直接获取所有行数据
|
|
|
+ if excelErr != nil {
|
|
|
+ utils.FileLog.Error("获取工作表数据时出错: %v", excelErr)
|
|
|
+ continue
|
|
|
+ }
|
|
|
+ fmt.Printf("sheetName:%s,sheetConfig:%d", sheetName, len(dataRows))
|
|
|
+ sheetConfig := SupplyRevisionAnalysisConfigMap[sheetName]
|
|
|
+ // 匹配非数字字符
|
|
|
+ classifyId := classifyIds[SupplyRevisionAnalysis]
|
|
|
+ switch sheetConfig.chartType {
|
|
|
+ case "one":
|
|
|
+ indexMap, err = OneDataHandler(dataRows, sheetConfig, classifyId)
|
|
|
+ case "two":
|
|
|
+ indexMap, err = TwoDataHandler(dataRows, sheetConfig, classifyId)
|
|
|
+ }
|
|
|
+ utils.FileLog.Info("读取工作表完成:", sheetName)
|
|
|
+ // 新增数据源指标数据
|
|
|
+ if len(indexMap) > 0 {
|
|
|
+ for _, index := range indexMap {
|
|
|
+ pars := chunkIndexData(index, rzdBatchSize)
|
|
|
+ for _, par := range pars {
|
|
|
+ // 转换成json
|
|
|
+ marshal, parseErr := json.Marshal(par)
|
|
|
+ if parseErr != nil {
|
|
|
+ utils.FileLog.Error("json.Marshal err: %v", parseErr)
|
|
|
+ return parseErr
|
|
|
+ }
|
|
|
+ // 发送 HTTP POST 请求
|
|
|
+ _, err = utils.HttpPostRequest(utils.EDB_LIB_URL+utils.ADD_BATCH_RZD_DATA, string(marshal), "application/json")
|
|
|
+ if err != nil {
|
|
|
+ utils.FileLog.Error("postEdbLib err: %v", err)
|
|
|
+ return err
|
|
|
+ }
|
|
|
+ }
|
|
|
+ }
|
|
|
+ }
|
|
|
+ }
|
|
|
+ return
|
|
|
+}
|
|
|
+func OneDataHandler(dataRows [][]string, sheetConfig SRAPConfig, classifyId int) (indexMap map[string]*models.IndexInfo, err error) {
|
|
|
+ for _, rowData := range dataRows[BeginRow:] {
|
|
|
+ // step_2: 指标
|
|
|
+ // 指标名称
|
|
|
+ indexNameOne := sheetName + "/" + rowData[sheetConfig.IndexRow] + "/" + sheetConfig.IndexNameColOne
|
|
|
+ indexNameTwo := sheetName + "/" + rowData[sheetConfig.IndexRow] + "/" + sheetConfig.IndexNameColTwo
|
|
|
+ // 生成指标编码
|
|
|
+ indexCodeOne := getIndexId(sheetName, strings.ReplaceAll(strings.ToLower(rowData[len(rowData)-3]), " ", ""), sheetConfig.IndexNameColOne)
|
|
|
+ indexCodeTwo := getIndexId(sheetName, strings.ReplaceAll(strings.ToLower(rowData[len(rowData)-3]), " ", ""), sheetConfig.IndexNameColTwo)
|
|
|
+ dataTime := rowData[sheetConfig.DateRow]
|
|
|
+ date, _, convertErr := utils.ConvertDateFormat(dataTime)
|
|
|
+ if convertErr != nil {
|
|
|
+ utils.FileLog.Error(fmt.Sprintf("转换时间数据失败,index_code:%s,time_value:%s err:%v", indexCodeOne, dataTime, convertErr))
|
|
|
+ continue
|
|
|
+ }
|
|
|
+ if rowData[sheetConfig.DataOneRow] != "" && re.MatchString(strings.TrimSpace(rowData[sheetConfig.DataOneRow])) {
|
|
|
+ valueOne, parseErr := decimal.NewFromString(strings.ReplaceAll(rowData[sheetConfig.DataOneRow], ",", ""))
|
|
|
+ if parseErr != nil {
|
|
|
+ utils.FileLog.Error(fmt.Sprintf("转换data数据失败,index_code:%s,data_value:%s err:%v", indexCodeOne, rowData[sheetConfig.DataOneRow], err))
|
|
|
+ continue
|
|
|
+ } else {
|
|
|
+ if index, ok := indexMap[indexCodeOne]; ok {
|
|
|
+ if index.StartDate.After(date) {
|
|
|
+ index.StartDate = date
|
|
|
+ }
|
|
|
+ if index.EndDate.Before(date) {
|
|
|
+ index.EndDate = date
|
|
|
+ index.LatestValue = valueOne
|
|
|
+ }
|
|
|
+ index.DataList = append(index.DataList, models.IndexData{
|
|
|
+ DataTime: date,
|
|
|
+ Value: valueOne,
|
|
|
+ })
|
|
|
+ } else {
|
|
|
+ indexMap[indexCodeOne] = &models.IndexInfo{
|
|
|
+ IndexName: indexNameOne,
|
|
|
+ IndexCode: indexCodeOne,
|
|
|
+ Frequency: sheetConfig.Frequency,
|
|
|
+ Unit: sheetConfig.Unit,
|
|
|
+ StartDate: date,
|
|
|
+ EndDate: date,
|
|
|
+ LatestValue: valueOne,
|
|
|
+ ClassifyId: classifyId,
|
|
|
+ DataList: []models.IndexData{
|
|
|
+ {
|
|
|
+ DataTime: date,
|
|
|
+ Value: valueOne,
|
|
|
+ },
|
|
|
+ },
|
|
|
+ }
|
|
|
+ }
|
|
|
+ }
|
|
|
+ }
|
|
|
+ if rowData[sheetConfig.DataTwoRow] != "" && re.MatchString(strings.TrimSpace(rowData[sheetConfig.DataTwoRow])) {
|
|
|
+ valueTwo, parseErr := decimal.NewFromString(strings.ReplaceAll(rowData[sheetConfig.DataTwoRow], ",", ""))
|
|
|
+ if parseErr != nil {
|
|
|
+ utils.FileLog.Error(fmt.Sprintf("转换data数据失败,index_code:%s,data_value:%s err:%v", indexCodeTwo, rowData[sheetConfig.DataTwoRow], err))
|
|
|
+ continue
|
|
|
+ } else {
|
|
|
+ if index, ok := indexMap[indexCodeTwo]; ok {
|
|
|
+ if index.StartDate.After(date) {
|
|
|
+ index.StartDate = date
|
|
|
+ }
|
|
|
+ if index.EndDate.Before(date) {
|
|
|
+ index.EndDate = date
|
|
|
+ index.LatestValue = valueTwo
|
|
|
+ }
|
|
|
+ index.DataList = append(index.DataList, models.IndexData{
|
|
|
+ DataTime: date,
|
|
|
+ Value: valueTwo,
|
|
|
+ })
|
|
|
+ } else {
|
|
|
+ indexMap[indexCodeTwo] = &models.IndexInfo{
|
|
|
+ IndexName: indexNameTwo,
|
|
|
+ IndexCode: indexCodeTwo,
|
|
|
+ Frequency: sheetConfig.Frequency,
|
|
|
+ Unit: sheetConfig.Unit,
|
|
|
+ StartDate: date,
|
|
|
+ EndDate: date,
|
|
|
+ LatestValue: valueTwo,
|
|
|
+ ClassifyId: classifyId,
|
|
|
+ DataList: []models.IndexData{
|
|
|
+ {
|
|
|
+ DataTime: date,
|
|
|
+ Value: valueTwo,
|
|
|
+ },
|
|
|
+ },
|
|
|
+ }
|
|
|
+ }
|
|
|
+ }
|
|
|
+ }
|
|
|
+ }
|
|
|
+}
|
|
|
+func TwoDataHandler(dataRows [][]string, sheetConfig SRAPConfig, classifyId int) (indexMap map[string]*models.IndexInfo, err error) {
|
|
|
+ for _, rowData := range dataRows[BeginRow:] {
|
|
|
+ // 匹配非数字字符
|
|
|
+ re := regexp.MustCompile(`^\d*\.?\d*\s*$`)
|
|
|
+ //处理数据不全的情况
|
|
|
+ less := Rows - len(rowData)
|
|
|
+ if less > 0 {
|
|
|
+ for i := 0; i < less; i++ {
|
|
|
+ rowData = append(rowData, "")
|
|
|
+ }
|
|
|
+ }
|
|
|
+ // step_2: 指标
|
|
|
+ // 指标名称
|
|
|
+ indexNameCurrent := SupplyRevisionAnalysis + "/" + sheetConfig.IndexNameColPrefix + "/" + sheetConfig.NameCurrent
|
|
|
+ indexNamePrevious := SupplyRevisionAnalysis + "/" + sheetConfig.IndexNameColPrefix + "/" + sheetConfig.NamePrevious
|
|
|
+
|
|
|
+ // 生成指标编码
|
|
|
+ indexCodeCurrent := getIndexId(fmt.Sprintf("%s %s", SupplyRevisionAnalysis, sheetConfig.IndexNameColPrefix), sheetConfig.CodeCurrent, "")
|
|
|
+ indexCodePrevious := getIndexId(fmt.Sprintf("%s %s", SupplyRevisionAnalysis, sheetConfig.IndexNameColPrefix), sheetConfig.CodePrevious, "")
|
|
|
+
|
|
|
+ dataTime := rowData[sheetConfig.DateRow]
|
|
|
+ var date time.Time
|
|
|
+ var convertErr error
|
|
|
+ switch sheetConfig.DateType {
|
|
|
+ case "quarter":
|
|
|
+ date, _, convertErr = utils.ConvertDateFormatQuarter(dataTime)
|
|
|
+ case "month":
|
|
|
+ date, _, convertErr = utils.ConvertDateFormat(dataTime)
|
|
|
+ case "year":
|
|
|
+ date, _, convertErr = utils.ConvertDateFormat(dataTime)
|
|
|
+ }
|
|
|
+ if convertErr != nil {
|
|
|
+ utils.FileLog.Error(fmt.Sprintf("转换时间数据失败,index_code:%s,time_value:%s err:%v", indexNameCurrent, dataTime, convertErr))
|
|
|
+ continue
|
|
|
+ }
|
|
|
+
|
|
|
+ if rowData[sheetConfig.DataCurrentRow] != "" && re.MatchString(strings.TrimSpace(rowData[sheetConfig.DataCurrentRow])) {
|
|
|
+ valueCurrent, parseErr := decimal.NewFromString(rowData[sheetConfig.DataCurrentRow])
|
|
|
+ if parseErr != nil {
|
|
|
+ utils.FileLog.Error(fmt.Sprintf("转换data数据失败,index_code:%s,data_value:%s err:%v", indexCodeCurrent, rowData[sheetConfig.DataCurrentRow], err))
|
|
|
+ continue
|
|
|
+ } else {
|
|
|
+ if index, ok := indexMap[indexCodeCurrent]; ok {
|
|
|
+ if index.StartDate.After(date) {
|
|
|
+ index.StartDate = date
|
|
|
+ }
|
|
|
+ if index.EndDate.Before(date) {
|
|
|
+ index.EndDate = date
|
|
|
+ index.LatestValue = valueCurrent
|
|
|
+ }
|
|
|
+ index.DataList = append(index.DataList, models.IndexData{
|
|
|
+ DataTime: date,
|
|
|
+ Value: valueCurrent,
|
|
|
+ })
|
|
|
+ } else {
|
|
|
+ indexMap[indexCodeCurrent] = &models.IndexInfo{
|
|
|
+ IndexName: indexNameCurrent,
|
|
|
+ IndexCode: indexCodeCurrent,
|
|
|
+ Frequency: sheetConfig.Frequency,
|
|
|
+ Unit: sheetConfig.Unit,
|
|
|
+ StartDate: date,
|
|
|
+ EndDate: date,
|
|
|
+ LatestValue: valueCurrent,
|
|
|
+ ClassifyId: classifyId,
|
|
|
+ DataList: []models.IndexData{
|
|
|
+ {
|
|
|
+ DataTime: date,
|
|
|
+ Value: valueCurrent,
|
|
|
+ },
|
|
|
+ },
|
|
|
+ }
|
|
|
+ }
|
|
|
+ }
|
|
|
+ }
|
|
|
+ if rowData[sheetConfig.DataPreviousRow] != "" && re.MatchString(strings.TrimSpace(rowData[sheetConfig.DataPreviousRow])) {
|
|
|
+ valuePrevious, parseErr := decimal.NewFromString(rowData[sheetConfig.DataPreviousRow])
|
|
|
+ if parseErr != nil {
|
|
|
+ utils.FileLog.Error(fmt.Sprintf("转换data数据失败,index_code:%s,data_value:%s err:%v", indexCodePrevious, rowData[sheetConfig.DataPreviousRow], err))
|
|
|
+ continue
|
|
|
+ } else {
|
|
|
+ if index, ok := indexMap[indexCodePrevious]; ok {
|
|
|
+ if index.StartDate.After(date) {
|
|
|
+ index.StartDate = date
|
|
|
+ }
|
|
|
+ if index.EndDate.Before(date) {
|
|
|
+ index.EndDate = date
|
|
|
+ index.LatestValue = valuePrevious
|
|
|
+ }
|
|
|
+ index.DataList = append(index.DataList, models.IndexData{
|
|
|
+ DataTime: date,
|
|
|
+ Value: valuePrevious,
|
|
|
+ })
|
|
|
+ } else {
|
|
|
+ indexMap[indexCodePrevious] = &models.IndexInfo{
|
|
|
+ IndexName: indexNamePrevious,
|
|
|
+ IndexCode: indexCodePrevious,
|
|
|
+ Frequency: sheetConfig.Frequency,
|
|
|
+ Unit: sheetConfig.Unit,
|
|
|
+ StartDate: date,
|
|
|
+ EndDate: date,
|
|
|
+ LatestValue: valuePrevious,
|
|
|
+ ClassifyId: classifyId,
|
|
|
+ DataList: []models.IndexData{
|
|
|
+ {
|
|
|
+ DataTime: date,
|
|
|
+ Value: valuePrevious,
|
|
|
+ },
|
|
|
+ },
|
|
|
+ }
|
|
|
+ }
|
|
|
+ }
|
|
|
+ }
|
|
|
+ }
|
|
|
+}
|
kobe6258