优化睿咨得数据

controllers/rzd_controller.go

@@ -25,7 +25,7 @@ func (this *RzdController) DealData() {
 		this.ServeJSON()
 	}()
 	var err error
-	err = ruizide.FileResolver()
+	err = ruizide.FileResolverV2()
 	if err != nil {
 		utils.FileLog.Error("睿咨得数据处理失败 err:%v", err)
 		br.Msg = "操作异常"

models/base.go

@@ -28,7 +28,9 @@ type BaseResponseResult struct {
 
 type RequestResponse[T any] struct {
 	Ret     int    `json:"Ret"`
+	ErrMsg  string `json:"ErrMsg"`
 	Success bool   `json:"Success"`
+	ErrCode string `json:"ErrCode"`
 	Data    T      `json:"Data"` // 这里是你要获取的关键字段
 	Msg     string `json:"Msg"`
 }

models/base_from_rzd_classify.go

@@ -2,11 +2,23 @@
 package models
 
 type BaseFromRzdClassify struct {
-	BaseFromRzdClassifyId int    `orm:"column(base_from_rzd_classify_id);pk"`
-	CreateTime            string `orm:"column(create_time)"`
-	ModifyTime            string `orm:"column(modify_time)"`
-	ClassifyName          string `orm:"column(classify_name)"`
-	ParentId              int    `orm:"column(parent_id)"`
-	Sort                  int    `orm:"column(sort)"`
-	ClassifyNameEn        string `orm:"column(classify_name_en)"`
+	BaseFromRzdClassifyId int
+	CreateTime            string
+	ModifyTime            string
+	ClassifyName          string
+	ParentId              int
+	Sort                  int
+	ClassifyNameEn        string
+}
+type AddRzdClassify struct {
+	ClassifyName   string
+	ParentId       int
+	Sort           int
+	ClassifyNameEn string
+	Children       []AddRzdClassify
+}
+
+type RzdClassifyMapStruct struct {
+	ClassifyName string
+	Sort         int
 }

services/ruizide/data_processor.go

@@ -237,7 +237,7 @@ func ResolverNet(_ context.Context) (err error) {
 	}
 	utils.FileLog.Info("睿咨得数据下载完成")
 	// 解析表格 读取数据
-	FileResolver()
+	FileResolverV2()
 	return
 }
 
@@ -449,11 +449,22 @@ func httpRequestFill(data interface{}, urlMethod string) (postEdbLib []byte, err
 
 // 解析本地文件
 func FileResolverV2() error {
+	//获取rzd数据库分类数据
+	rzdClassifyList, err := getRzdClassifyList()
+	if err != nil {
+		fmt.Printf("获取睿咨得分类数据失败: %v", err)
+		utils.FileLog.Error("获取睿咨得分类数据失败:", err.Error())
+		return err
+	}
+	//更新睿咨得分类Map
+	for _, classify := range rzdClassifyList {
+		RzdClassifyMap[classify.ClassifyName] = &classify
+	}
 	for _, tableName := range tableNameList {
-		rzdProcessor, err := GetRZDProcessor(tableName)
-		if err != nil {
-			fmt.Printf("获取睿咨得数据处理器: %v", err)
-			utils.FileLog.Error("获取睿咨得数据处理器:", err.Error())
+		rzdProcessor, processorErr := GetRZDProcessor(tableName)
+		if processorErr != nil {
+			fmt.Printf("获取睿咨得数据处理器: %v", processorErr)
+			utils.FileLog.Error("获取睿咨得数据处理器:", processorErr.Error())
 			continue
 		}
 		_, err = rzdProcessor.Process(tableName)

services/ruizide/oil_demand_signals_weekly_report_processor.go

@@ -144,37 +144,98 @@ func (p *OilDemandSignalsWeeklyReportProcessor) Process(tableName string) (list
 		utils.FileLog.Error("无法打开 Excel 文件: %v", err)
 		return
 	}
-	classifyIdList, httpErr := dealClassify(tableName, sheetName)
-	// 获取所有工作表
 	sheetNames := file.GetSheetList()
+	classifyIds, err := dealClassify(tableName, sheetNames)
+	// 获取所有工作表
+	if err != nil {
+		utils.FileLog.Error("获取分类Id失败: %v", err)
+		return
+	}
 	for _, sheetName := range sheetNames {
-		// step_1: 分类
-
-		if httpErr != nil {
-			utils.FileLog.Error("获取分类Id失败: %v", err)
-			continue
-		}
-		utils.FileLog.Info("classifyId: %v", classifyId)
 		if sheetName == "Content" {
 			utils.FileLog.Info("跳过工作表:", sheetName)
 			continue
 		}
 		utils.FileLog.Info("读取工作表: %s\n", sheetName)
 		// 获取工作表的最大行数
-		maxRow, err := file.GetRows(sheetName) // 直接获取所有行数据
-		if err != nil {
-			utils.FileLog.Error("获取工作表数据时出错: %v", err)
+		dataRows, excelErr := file.GetRows(sheetName) // 直接获取所有行数据
+		if excelErr != nil {
+			utils.FileLog.Error("获取工作表数据时出错: %v", excelErr)
 			continue
 		}
 		var indexData []models.BaseFromRzdData
-		for rowIndex, rowData := range maxRow {
-			// 因为excel文件中的sheet表格不固定 对于 Supply_Revision_Analysis, Oil_Supply_Analysis 文件 手动调整sheet表格顺序
-			baseFromRzdDataList, err := processor.Process(tableName, sheetName, rowIndex, rowData)
+		for rowIndex, rowData := range dataRows[:] {
+			if rowIndex < 4 {
+				return nil, nil
+			}
+
+			frequency := "日度"
+			unit := "辆"
+			indexNameColOne := "Active Fleet"
+			indexNameColTwo := "Active Fleet 7DMA"
+
+			// step_1: 分类
+			classifyId, err := dealClassify(tableName, sheetName)
+			if err != nil {
+				return nil, err
+			}
+			utils.FileLog.Info("classifyId: %v", classifyId)
+
+			// step_2: 指标
+			// 指标名称
+			indexNameOne := sheetName + "/" + rowData[len(rowData)-3] + "/" + indexNameColOne
+			indexNameTwo := sheetName + "/" + rowData[len(rowData)-3] + "/" + indexNameColTwo
+
+			// 生成指标编码
+			indexCodeOne, err := getIndexId(sheetName, strings.ReplaceAll(strings.ToLower(rowData[len(rowData)-3]), " ", ""), indexNameColOne)
+			indexCodeTwo, err := getIndexId(sheetName, strings.ReplaceAll(strings.ToLower(rowData[len(rowData)-3]), " ", ""), indexNameColTwo)
+
+			var indexInfoList []*models.IndexInfo
+			valueOne, err := strconv.ParseFloat(strings.ReplaceAll(rowData[len(rowData)-2], ",", ""), 64)
+			if err != nil {
+				utils.FileLog.Error(fmt.Sprintf("转换data数据失败,index_code:%s,data_value:%s err:%v", indexCodeOne, rowData[len(rowData)-2], err))
+				return nil, err
+			}
+
+			dataTimeOne := rowData[1]
+			formatOne, err := utils.ConvertDateFormat(dataTimeOne)
+			if err != nil {
+				utils.FileLog.Error(fmt.Sprintf("转换时间数据失败,index_code:%s,time_value:%s err:%v", indexCodeOne, dataTimeOne, err))
+				return nil, err
+			}
+
+			indexInfoList = append(indexInfoList, &models.IndexInfo{
+				IndexName: indexNameOne,
+				IndexCode: indexCodeOne,
+				Value:     valueOne,
+				DataTime:  formatOne,
+			})
+			valueTwo, err := strconv.ParseFloat(strings.ReplaceAll(rowData[len(rowData)-1], ",", ""), 64)
 			if err != nil {
-				utils.FileLog.Error("处理数据失败: %v", err)
-				return err
+				utils.FileLog.Error(fmt.Sprintf("转换data数据失败,index_code:%s,data_value:%s err:%v", indexCodeTwo, rowData[len(rowData)-1], err))
+				return nil, err
 			}
-			fmt.Printf("处理数据成功: %v\n", baseFromRzdDataList)
+			indexInfoList = append(indexInfoList, &models.IndexInfo{
+				IndexName: indexNameTwo,
+				IndexCode: indexCodeTwo,
+				Value:     valueTwo,
+				DataTime:  formatOne,
+			})
+
+			indexInfoList, err = dealIndex(indexInfoList, frequency, unit, classifyId)
+			if err != nil {
+				return nil, err
+			}
+			utils.FileLog.Info("AviationActiveFleetProcessor indexInfoList: %v", indexInfoList)
+
+			// step_3: 指标数据
+			dataList, err := dealData(indexInfoList)
+			if err != nil {
+				return nil, err
+			}
+			utils.FileLog.Info("AviationActiveFleetProcessor dataList: %v", dataList)
+
+			return dataList, err
 			indexData = append(indexData, baseFromRzdDataList...)
 		}
 		fmt.Printf("读取工作表完成: %s\n", sheetName)
@@ -287,4 +348,5 @@ func (p *OilDemandSignalsWeeklyReportProcessor) Process(tableName string) (list
 		utils.FileLog.Info("dataList: %v", dataList)
 		return dataList, err
 	}
+	return nil, nil
 }

services/ruizide/processor_business_logic.go

@@ -13,20 +13,6 @@ import (
 	"unicode"
 )
 
-var classifyMap = map[string]string{
-	"Road Index":                       "analytics library",
-	"Road Active Fleet":                "analytics library",
-	"Aviation Index":                   "analytics library",
-	"Aviation Active Fleet":            "analytics library",
-	"Demand-Gasoline":                  "analytics library",
-	"Demand - Diesel":                  "analytics library",
-	"Demand - Jet Fuel":                "analytics library",
-	"Demand - Maritime Bunker":         "analytics library",
-	"Oil_Demand_Signals_Weekly_Report": "analytics library",
-	"cube dashboards":                  "cube dashboards",
-	"Oil Market Cube":                  "Oil Market Cube",
-}
-
 // RoadIndexProcessor
 // @Description: AnalyticsLibrary处理器
 type RoadIndexProcessor struct{}
@@ -3414,74 +3400,89 @@ func getIndexId(prefix string, area string, suffix string) (string, error) {
 }
 
 // 处理分类
-func dealClassify(tableName, sheetName string) (int, error) {
-	// 查询一级分类是否存在
-	paramsLib := make(map[string]interface{})
-	paramsLib["classifyName"] = classifyMap[tableName]
-	postEdbLib, err := httpRequestFill(paramsLib, utils.GET_RZD_CLASSIFY_BY_NAME)
-	if err != nil {
-		return 0, fmt.Errorf("AnalyticsLibraryProcessor Process() : failed to get classify: %v", err)
-	}
-	var requestResponse models.RequestResponse[models.BaseFromRzdClassify]
-	err = json.Unmarshal(postEdbLib, &requestResponse)
-	if err != nil {
-		return 0, err
-	}
-
-	// 处理一级分类
+func dealClassify(tableName string, sheetNames []string) (classifyList map[string]int, err error) {
+	var rzdClassifyList []models.AddRzdClassify
 	var parentId int
-	if requestResponse.Data.BaseFromRzdClassifyId == 0 {
-		// 一级分类不存在，新增一级分类
-		paramsLib = make(map[string]interface{})
-		paramsLib["parentId"] = 0
-		paramsLib["classifyName"] = classifyMap[tableName]
-		postEdbLib, err = httpRequestFill(paramsLib, utils.ADD_RZD_CLASSIFY)
-		if err != nil {
-			return 0, fmt.Errorf("AnalyticsLibraryProcessor Process() : failed to add classify: %v", err)
+	classifyList = make(map[string]int)
+	// 查询一级分类是否存在
+	if classify, ok := RzdClassifyMap[tableName]; !ok {
+		rzdMapStruct := classifyMap[tableName]
+		parentClassify := models.AddRzdClassify{
+			ClassifyName: rzdMapStruct.ClassifyName,
+			Sort:         rzdMapStruct.Sort,
+			ParentId:     0,
 		}
-		var requestResponse models.RequestResponse[int]
-		err = json.Unmarshal(postEdbLib, &requestResponse)
-		if err != nil {
-			return 0, err
+		rzdClassifyList = append(rzdClassifyList, parentClassify)
+		resp, httpErr := httpRequestFill(rzdClassifyList, utils.BATCH_ADD_RZD_CLASSYFY)
+		if httpErr != nil {
+			err = fmt.Errorf("添加睿咨得分类失败: %v", err)
+			return
+		}
+		var rzdResp models.RequestResponse[[]models.BaseFromRzdClassify]
+		jsonErr := json.Unmarshal(resp, &rzdResp)
+		if jsonErr != nil {
+			err = fmt.Errorf("解析添加睿咨得分类失败: %v", err)
+			return
+		}
+		if rzdResp.Ret != 200 {
+			err = fmt.Errorf("添加睿咨得分类失败: %v", err)
+			return
 		}
-		parentId = requestResponse.Data
+		parentId = rzdResp.Data[0].BaseFromRzdClassifyId
+		classifyList[tableName] = rzdResp.Data[0].BaseFromRzdClassifyId
 	} else {
-		// 一级分类已存在，使用其 ID
-		parentId = requestResponse.Data.BaseFromRzdClassifyId
+		parentId = classify.BaseFromRzdClassifyId
+		classifyList[tableName] = classify.BaseFromRzdClassifyId
+	}
+	//处理二级分类
+	for _, sheetName := range sheetNames {
+		if _, ok := RzdClassifyMap[sheetName]; !ok {
+			rzdMapStruct := classifyMap[sheetName]
+			rzdClassifyList = append(rzdClassifyList, models.AddRzdClassify{
+				ClassifyName: rzdMapStruct.ClassifyName,
+				Sort:         rzdMapStruct.Sort,
+				ParentId:     parentId,
+			})
+		}
 	}
-	// 查询二级分类是否存在
-	paramsSubLib := make(map[string]interface{})
-	paramsSubLib["classifyName"] = sheetName // 这里替换成实际的二级分类名称
-	postSubEdbLib, err := httpRequestFill(paramsSubLib, utils.GET_RZD_CLASSIFY_BY_NAME)
-	if err != nil {
-		return 0, fmt.Errorf("AnalyticsLibraryProcessor Process() : failed to get sub classify: %v", err)
+	resp, httpErr := httpRequestFill(rzdClassifyList, utils.BATCH_ADD_RZD_CLASSYFY)
+	if httpErr != nil {
+		err = fmt.Errorf("添加睿咨得分类失败: %v", err)
+		return
 	}
-
-	var subRequestResponse models.RequestResponse[models.BaseFromRzdClassify]
-	err = json.Unmarshal(postSubEdbLib, &subRequestResponse)
-	if err != nil {
-		return 0, err
+	var rzdResp models.RequestResponse[[]models.BaseFromRzdClassify]
+	jsonErr := json.Unmarshal(resp, &rzdResp)
+	if jsonErr != nil {
+		err = fmt.Errorf("解析添加睿咨得分类失败: %v", err)
+		return
 	}
-
-	// 新增二级分类
-	var classifyId int
-	if subRequestResponse.Data.BaseFromRzdClassifyId == 0 {
-		paramsLib = make(map[string]interface{})
-		paramsLib["parentId"] = parentId
-		paramsLib["classifyName"] = sheetName
-		postEdbLib, err = httpRequestFill(paramsLib, utils.ADD_RZD_CLASSIFY)
-		if err != nil {
-			return 0, fmt.Errorf("AnalyticsLibraryProcessor Process() : failed to add classify: %v", err)
-		}
-		var requestResponse models.RequestResponse[int]
-		err = json.Unmarshal(postEdbLib, &requestResponse)
-		if err != nil {
-			return 0, err
-		}
-		classifyId = requestResponse.Data
-	} else {
-		classifyId = subRequestResponse.Data.BaseFromRzdClassifyId
+	if rzdResp.Ret != 200 {
+		err = fmt.Errorf("添加睿咨得分类失败: %v", err)
+		return
 	}
+	for _, classify := range rzdResp.Data {
+		classifyList[classify.ClassifyName] = classify.BaseFromRzdClassifyId
+	}
+	return
+}
 
-	return classifyId, nil
+// 处理分类
+func getRzdClassifyList() (list []models.BaseFromRzdClassify, err error) {
+	//查询睿咨得分类
+	postEdbLib, httpErr := httpRequestFill(nil, utils.GET_RZD_CLASSIFY)
+	if httpErr != nil {
+		err = fmt.Errorf("获取睿咨得分类数据失败: %v", httpErr)
+		return
+	}
+	var requestResponse models.RequestResponse[[]models.BaseFromRzdClassify]
+	jsonErr := json.Unmarshal(postEdbLib, &requestResponse)
+	if jsonErr != nil {
+		err = fmt.Errorf("获取睿咨得分类数据失败,解析JSON数据失败: %v", jsonErr)
+		return
+	}
+	if requestResponse.Ret != 200 {
+		err = fmt.Errorf("获取睿咨得分类数据失败,请求应答异常: %v", requestResponse.ErrMsg)
+		return
+	}
+	return requestResponse.Data, nil
 }

services/ruizide/processor_factory.go

@@ -5,6 +5,56 @@ import (
 	"fmt"
 )
 
+var (
+	RzdClassifyMap map[string]*models.BaseFromRzdClassify
+	classifyMap    = map[string]models.RzdClassifyMapStruct{
+		"Road Index": {
+			ClassifyName: "Road Index",
+			Sort:         1,
+		},
+		"Road Active Fleet": {
+			ClassifyName: "Road Active Fleet",
+			Sort:         2,
+		},
+		"Aviation Index": {
+			ClassifyName: "Aviation Index",
+			Sort:         3,
+		},
+		"Aviation Active Fleet": {
+			ClassifyName: "Aviation Active Fleet",
+			Sort:         4,
+		},
+		"Demand-Gasoline": {
+			ClassifyName: "Demand-Gasoline",
+			Sort:         5,
+		},
+		"Demand - Diesel": {
+			ClassifyName: "Demand - Diesel",
+			Sort:         6,
+		},
+		"Demand - Jet Fuel": {
+			ClassifyName: "Demand - Jet Fuel",
+			Sort:         7,
+		},
+		"Demand - Maritime Bunker": {
+			ClassifyName: "Demand - Maritime Bunker",
+			Sort:         8,
+		},
+		"Oil_Demand_Signals_Weekly_Report": {
+			ClassifyName: "analytics library",
+			Sort:         1,
+		},
+		"cube dashboards": {
+			ClassifyName: "cube dashboards",
+			Sort:         2,
+		},
+		"Oil Market Cube": {
+			ClassifyName: "Oil Market Cube",
+			Sort:         3,
+		},
+	}
+)
+
 type ReportProcessor interface {
 	Process(string, string, int, []string) ([]models.BaseFromRzdData, error)
 }

utils/constants.go

@@ -273,6 +273,8 @@ const (
 	GET_EDB_RZD_DATA_BY_CODE_AND_TIME                 = "/rzd/get/edb/rzd/data/by/code/and/time"                // 根据指标编码和日期获取指标数据
 	GET_RZD_INDEX_BY_CODE                             = "/rzd/get/rzd/index/by/code"                            // 根据指标编码获取指标
 	GET_RZD_CLASSIFY_BY_NAME                          = "/rzd/get/rzd/classify/by/name"                         // 根据分类名称查询分类
+	GET_RZD_CLASSIFY                                  = "/rzd/get/rzd/classify"                                 // 睿咨得查询分类
+	BATCH_ADD_RZD_CLASSYFY                            = "/add/rzd/batch_add_classify"                           //批量增加睿咨得分类
 	ADD_RZD_CLASSIFY                                  = "/rzd/add/rzd/classify"                                 // 新增睿咨得分类
 	ADD_BATCH_RZD_EDB_DATA                            = "/rzd/add/batch/rzd/edb/data"                           // 批量增加指标库指标数据
 	GET_RZD_EDB_INFO_BY_INDEX_CODE                    = "/rzd/get/rzd/edb/info/by/code"                         // 根据指标code获取指标信息