睿咨得数据爬取

models/base_from_rzd_index.go

@@ -12,3 +12,11 @@ type BaseFromRzdIndex struct {
 	Frequency            string `orm:"column(frequency)"`
 	Unit                 string `orm:"column(unit)"`
 }
+
+type IndexInfo struct {
+	IndexName   string
+	IndexCode   string
+	Value       float64
+	IndexInfoId int
+	DataTime    string
+}

services/ruizide/processor_business_logic.go

@@ -49,53 +49,55 @@ func (p *RoadIndexProcessor) Process(tableName string, sheetName string, rowInde
 	logs.Info("classifyId: %v", classifyId)
 
 	// step_2: 指标
-	indexOneId, indexTwoId, indexCodeOne, indexCodeTwo, err := dealIndex(sheetName, rowData, indexNameColOne, indexNameColTwo, frequency, unit, classifyId)
+	// 指标名称
+	indexNameOne := sheetName + "/" + rowData[len(rowData)-3] + "/" + indexNameColOne
+	indexNameTwo := sheetName + "/" + rowData[len(rowData)-3] + "/" + indexNameColTwo
+
+	// 生成指标编码
+	indexCodeOne, err := getIndexId(sheetName, strings.ToLower(rowData[len(rowData)-3]), indexNameColOne)
+	indexCodeTwo, err := getIndexId(sheetName, strings.ToLower(rowData[len(rowData)-3]), indexNameColTwo)
+
+	var indexInfoMap map[string]string
+	indexInfoMap[indexCodeOne] = indexNameOne
+	indexInfoMap[indexCodeTwo] = indexNameTwo
+
+	var indexInfoList []*models.IndexInfo
+	valueOne, err := strconv.ParseFloat(rowData[len(rowData)-2], 64)
 	if err != nil {
 		return nil, err
 	}
-	logs.Info("indexOneId: %v, indexTwoId: %v, indexCodeOne: %v, indexCodeTwo: %v", indexOneId, indexTwoId, indexCodeOne, indexCodeTwo)
 
-	// step_3: 指标数据
-	dataList, err := dealData(indexOneId, indexTwoId, indexCodeOne, indexCodeTwo, rowData)
+	dataTimeOne := rowData[1]
+	formatOne, err := utils.ConvertDateFormat(dataTimeOne)
 	if err != nil {
 		return nil, err
 	}
-	logs.Info("dataList: %v", dataList)
-
-	return dataList, err
-}
-
-// ReDashboardExportOneProcessor
-// @Description: ReDashboardExportOneProcessor处理器
-type ReDashboardExportOneProcessor struct{}
-
-func (p *ReDashboardExportOneProcessor) Process(tableName string, sheetName string, rowIndex int, rowData []string) ([]models.BaseFromRzdData, error) {
-	logs.Info("Processing ReDashboardExportOne...")
-	if rowIndex < 4 {
-		return nil, nil
-	}
 
-	frequency := "季度"
-	unit := "千桶每天"
-	indexNameColOne := "Index"
-	indexNameColTwo := "Index 7DMA"
-
-	// step_1: 分类
-	classifyId, err := dealClassify("cube dashboards", "Supply Revision Analysis")
+	indexInfoList = append(indexInfoList, &models.IndexInfo{
+		IndexName: indexNameOne,
+		IndexCode: indexCodeOne,
+		Value:     valueOne,
+		DataTime:  formatOne,
+	})
+	valueTwo, err := strconv.ParseFloat(rowData[len(rowData)-1], 64)
 	if err != nil {
 		return nil, err
 	}
-	logs.Info("classifyId: %v", classifyId)
+	indexInfoList = append(indexInfoList, &models.IndexInfo{
+		IndexName: indexNameTwo,
+		IndexCode: indexCodeTwo,
+		Value:     valueTwo,
+		DataTime:  formatOne,
+	})
 
-	// step_2: 指标
-	indexOneId, indexTwoId, indexCodeOne, indexCodeTwo, err := dealIndex(sheetName, rowData, indexNameColOne, indexNameColTwo, frequency, unit, classifyId)
+	indexInfoList, err = dealIndex(indexInfoList, frequency, unit, classifyId)
 	if err != nil {
 		return nil, err
 	}
-	logs.Info("indexOneId: %v, indexTwoId: %v, indexCodeOne: %v, indexCodeTwo: %v", indexOneId, indexTwoId, indexCodeOne, indexCodeTwo)
+	logs.Info("indexInfoList: %v", indexInfoList)
 
 	// step_3: 指标数据
-	dataList, err := dealData(indexOneId, indexTwoId, indexCodeOne, indexCodeTwo, rowData)
+	dataList, err := dealData(indexInfoList)
 	if err != nil {
 		return nil, err
 	}
@@ -104,105 +106,131 @@ func (p *ReDashboardExportOneProcessor) Process(tableName string, sheetName stri
 	return dataList, err
 }
 
-func dealData(indexOneId, indexTwoId int, indexCodeOne, indexCodeTwo string, rowData []string) ([]models.BaseFromRzdData, error) {
-	var dataList []models.BaseFromRzdData
+// ReDashboardExportOneProcessor
+// @Description: ReDashboardExportOneProcessor处理器
+type ReDashboardExportOneProcessor struct{}
 
-	valueOne, err := strconv.ParseFloat(rowData[len(rowData)-2], 64)
-	if err != nil {
-		return nil, err
-	}
-	valueOne = math.Round(valueOne*10000) / 10000
+/*
+	func (p *ReDashboardExportOneProcessor) Process(tableName string, sheetName string, rowIndex int, rowData []string) ([]models.BaseFromRzdData, error) {
+		logs.Info("Processing ReDashboardExportOne...")
+		if rowIndex < 4 {
+			return nil, nil
+		}
 
-	dataTimeOne := rowData[1]
-	formatOne, err := utils.ConvertDateFormat(dataTimeOne)
-	if err != nil {
-		return nil, err
-	}
-	dataTimeOne = formatOne
+		frequency := "季度"
+		unit := "千桶每天"
+		indexNameColOne := "Index"
+		indexNameColTwo := "Index 7DMA"
 
-	paramsLib := make(map[string]interface{})
-	paramsLib["IndexCode"] = indexCodeOne
-	paramsLib["DataTime"] = dataTimeOne
-	postEdbLib, err := httpRequestFill(paramsLib, utils.GET_RZD_INDEX_DATA_BY_CODE_AND_TIME)
-	if err != nil {
-		return nil, err
-	}
-	var requestResponse models.RequestResponse[models.BaseFromRzdData]
-	err = json.Unmarshal(postEdbLib, &requestResponse)
-	if err != nil {
-		return nil, err
-	}
-	if requestResponse.Data.BaseFromRzdIndexId == 0 {
-		dataOne := models.BaseFromRzdData{
-			BaseFromRzdIndexId: indexOneId,
-			CreateTime:         utils.GetCurrentTime(),
-			DataTime:           dataTimeOne,
-			IndexCode:          indexCodeOne,
-			ModifyTime:         utils.GetCurrentTime(),
-			Value:              valueOne,
+		// step_1: 分类
+		classifyId, err := dealClassify("cube dashboards", "Supply Revision Analysis")
+		if err != nil {
+			return nil, err
 		}
-		dataList = append(dataList, dataOne)
-	}
+		logs.Info("classifyId: %v", classifyId)
 
-	valueTwo, err := strconv.ParseFloat(rowData[len(rowData)-2], 64)
-	if err != nil {
-		return nil, err
-	}
-	valueTwo = math.Round(valueTwo*10000) / 10000
+		// step_2: 指标
+		indexOneId, indexTwoId, indexCodeOne, indexCodeTwo, err := dealIndex(sheetName, rowData, indexNameColOne, indexNameColTwo, frequency, unit, classifyId)
+		if err != nil {
+			return nil, err
+		}
+		logs.Info("indexOneId: %v, indexTwoId: %v, indexCodeOne: %v, indexCodeTwo: %v", indexOneId, indexTwoId, indexCodeOne, indexCodeTwo)
 
-	dataTimeTwo := rowData[1]
-	formatTwo, err := utils.ConvertDateFormat(dataTimeTwo)
-	if err != nil {
-		return nil, err
-	}
-	dataTimeTwo = formatTwo
+		// step_3: 指标数据
+		dataList, err := dealData(indexOneId, indexTwoId, indexCodeOne, indexCodeTwo, rowData)
+		if err != nil {
+			return nil, err
+		}
+		logs.Info("dataList: %v", dataList)
 
-	paramsLib = make(map[string]interface{})
-	paramsLib["indexCode"] = indexCodeTwo
-	paramsLib["dataTime"] = dataTimeTwo
-	postEdbLib, err = httpRequestFill(paramsLib, utils.GET_RZD_INDEX_DATA_BY_CODE_AND_TIME)
-	if err != nil {
-		return nil, err
-	}
-	var requestResponseTwo models.RequestResponse[models.BaseFromRzdData]
-	err = json.Unmarshal(postEdbLib, &requestResponseTwo)
-	if err != nil {
-		return nil, err
+		return dataList, err
 	}
-	if requestResponseTwo.Data.BaseFromRzdIndexId == 0 {
-		dataTwo := models.BaseFromRzdData{
-			BaseFromRzdIndexId: indexTwoId,
-			CreateTime:         utils.GetCurrentTime(),
-			DataTime:           dataTimeTwo,
-			IndexCode:          indexCodeTwo,
-			ModifyTime:         utils.GetCurrentTime(),
-			Value:              valueTwo,
+*/
+func dealData(indexInfoList []*models.IndexInfo) ([]models.BaseFromRzdData, error) {
+	var dataList []models.BaseFromRzdData
+	for _, indexInfo := range indexInfoList {
+		paramsLib := make(map[string]interface{})
+		paramsLib["IndexCode"] = indexInfo.IndexCode
+		paramsLib["DataTime"] = indexInfo.DataTime
+		postEdbLib, err := httpRequestFill(paramsLib, utils.GET_RZD_INDEX_DATA_BY_CODE_AND_TIME)
+		if err != nil {
+			return nil, err
+		}
+		var requestResponse models.RequestResponse[models.BaseFromRzdData]
+		err = json.Unmarshal(postEdbLib, &requestResponse)
+		if err != nil {
+			return nil, err
+		}
+		if requestResponse.Data.BaseFromRzdIndexId == 0 {
+			dataOne := models.BaseFromRzdData{
+				BaseFromRzdIndexId: indexInfo.IndexInfoId,
+				CreateTime:         utils.GetCurrentTime(),
+				DataTime:           indexInfo.DataTime,
+				IndexCode:          indexInfo.IndexCode,
+				ModifyTime:         utils.GetCurrentTime(),
+				Value:              math.Round(indexInfo.Value*10000) / 10000,
+			}
+			dataList = append(dataList, dataOne)
 		}
-		dataList = append(dataList, dataTwo)
 	}
 
 	return dataList, nil
 }
 
-func dealIndex(sheetName string, rowData []string, indexNameColOne string, indexNameColTwo string, frequency string, unit string, classifyId int) (indexOneId, indexTwoId int, indexCodeOne, indexCodeTwo string, err error) {
-	// 指标名称
-	indexNameOne := sheetName + "/" + rowData[len(rowData)-3] + "/" + indexNameColOne
-	indexNameTwo := sheetName + "/" + rowData[len(rowData)-3] + "/" + indexNameColTwo
-	// 生成指标编码
-	indexCodeOne, err = getIndexId(sheetName, strings.ToLower(rowData[len(rowData)-3]), indexNameColOne)
-	indexCodeTwo, err = getIndexId(sheetName, strings.ToLower(rowData[len(rowData)-3]), indexNameColTwo)
-
-	// 处理第一个指标
+func dealIndex(indexInfoList []*models.IndexInfo, frequency string, unit string, classifyId int) ([]*models.IndexInfo, error) {
+	for _, index := range indexInfoList {
+		// 处理第一个指标
+		paramsLib := make(map[string]interface{})
+		paramsLib["indexCode"] = index.IndexCode
+		postEdbLib, err := httpRequestFill(paramsLib, utils.GET_RZD_INDEX_BY_CODE)
+		if err != nil {
+			return nil, fmt.Errorf("getIndexId() : Failed to get rzd index by code: %v", err)
+		}
+		var requestResponse models.RequestResponse[models.BaseFromRzdIndex]
+		err = json.Unmarshal(postEdbLib, &requestResponse)
+		if err != nil {
+			return nil, err
+		}
+		if requestResponse.Data.BaseFromRzdIndexId == 0 {
+			indexOne := models.BaseFromRzdIndex{
+				CreateTime:           utils.GetCurrentTime(),
+				ModifyTime:           utils.GetCurrentTime(),
+				BaseFromLyClassifyId: classifyId,
+				IndexCode:            index.IndexCode,
+				IndexName:            index.IndexName,
+				Frequency:            frequency,
+				Unit:                 unit,
+			}
+			// 这里避免服务器宕机 出现唯一索引异常，进行分开保存
+			postEdbLib, err = httpRequestFill(indexOne, utils.ADD_RZD_INDEX)
+			if err != nil {
+				return nil, fmt.Errorf("getIndexId() : Failed to get rzd index by code: %v", err)
+			}
+			var requestResponse models.RequestResponse[int]
+			err = json.Unmarshal(postEdbLib, &requestResponse)
+			if err != nil {
+				return nil, err
+			}
+			logs.Info("indexOneId: %v", requestResponse.Data)
+
+			index.IndexInfoId = requestResponse.Data
+		} else {
+			logs.Info("indexOneId: %v", requestResponse.Data.BaseFromRzdIndexId)
+			index.IndexInfoId = requestResponse.Data.BaseFromRzdIndexId
+		}
+	}
+	return indexInfoList, nil
+	/*// 处理第一个指标
 	paramsLib := make(map[string]interface{})
 	paramsLib["indexCode"] = indexCodeOne
 	postEdbLib, err := httpRequestFill(paramsLib, utils.GET_RZD_INDEX_BY_CODE)
 	if err != nil {
-		return 0, 0, "0", "0", fmt.Errorf("getIndexId() : Failed to get rzd index by code: %v", err)
+		return 0, 0, fmt.Errorf("getIndexId() : Failed to get rzd index by code: %v", err)
 	}
 	var requestResponse models.RequestResponse[models.BaseFromRzdIndex]
 	err = json.Unmarshal(postEdbLib, &requestResponse)
 	if err != nil {
-		return 0, 0, "0", "0", err
+		return 0, 0, err
 	}
 	if requestResponse.Data.BaseFromRzdIndexId == 0 {
 		indexOne := models.BaseFromRzdIndex{
@@ -217,12 +245,12 @@ func dealIndex(sheetName string, rowData []string, indexNameColOne string, index
 		// 这里避免服务器宕机 出现唯一索引异常，进行分开保存
 		postEdbLib, err = httpRequestFill(indexOne, utils.ADD_RZD_INDEX)
 		if err != nil {
-			return 0, 0, "0", "0", fmt.Errorf("getIndexId() : Failed to get rzd index by code: %v", err)
+			return 0, 0, fmt.Errorf("getIndexId() : Failed to get rzd index by code: %v", err)
 		}
 		var requestResponse models.RequestResponse[int]
 		err = json.Unmarshal(postEdbLib, &requestResponse)
 		if err != nil {
-			return 0, 0, "0", "0", err
+			return 0, 0, err
 		}
 		indexOneId = requestResponse.Data
 		logs.Info("indexOneId: %v", indexOneId)
@@ -236,12 +264,12 @@ func dealIndex(sheetName string, rowData []string, indexNameColOne string, index
 	paramsLib["indexCode"] = indexCodeTwo
 	postEdbLib, err = httpRequestFill(paramsLib, utils.GET_RZD_INDEX_BY_CODE)
 	if err != nil {
-		return 0, 0, "0", "0", fmt.Errorf("getIndexId() : Failed to get rzd index by code: %v", err)
+		return 0, 0, fmt.Errorf("getIndexId() : Failed to get rzd index by code: %v", err)
 	}
 	var requestResponseTwo models.RequestResponse[models.BaseFromRzdIndex]
 	err = json.Unmarshal(postEdbLib, &requestResponseTwo)
 	if err != nil {
-		return 0, 0, "0", "0", err
+		return 0, 0, err
 	}
 	if requestResponseTwo.Data.BaseFromRzdIndexId == 0 {
 		indexTwo := models.BaseFromRzdIndex{
@@ -257,19 +285,19 @@ func dealIndex(sheetName string, rowData []string, indexNameColOne string, index
 		var requestResponse models.RequestResponse[int]
 		postEdbLib, err = httpRequestFill(indexTwo, utils.ADD_RZD_INDEX)
 		if err != nil {
-			return 0, 0, "0", "0", fmt.Errorf("getIndexId() : Failed to get rzd index by code: %v", err)
+			return 0, 0, fmt.Errorf("getIndexId() : Failed to get rzd index by code: %v", err)
 		}
 		err = json.Unmarshal(postEdbLib, &requestResponse)
 		if err != nil {
-			return 0, 0, "0", "0", err
+			return 0, 0, err
 		}
 		indexTwoId = requestResponse.Data
 		logs.Info("indexTwoId: %v", indexTwoId)
 	} else {
 		indexTwoId = requestResponseTwo.Data.BaseFromRzdIndexId
-	}
+	}*/
 
-	return indexOneId, indexTwoId, indexCodeOne, indexCodeTwo, nil
+	//return indexOneId, indexTwoId, nil
 }
 
 func getIndexId(prefix string, area string, suffix string) (string, error) {