睿姿得数据对接-初始化

services/ruizide/data_processor.go

@@ -1,55 +1,147 @@
-package liangyou
+package main
 
 import (
 	"context"
-	"encoding/json"
-	"eta/eta_data_analysis/models"
 	"eta/eta_data_analysis/utils"
 	"fmt"
-	"log"
 	"os"
+	"path/filepath"
 	"time"
 
 	"github.com/chromedp/chromedp"
 )
 
+const downloadDir = "./downloads"
+
+// 定义选择器
 var (
 	rzdLoginPath = "https://clients.rystadenergy.com/clients/"
+
+	clientSearchLink               = `a[href="/clients/search/"]`
+	supplyRevisionAnalysisSelector = `div.ais-Hits li[contains(., 'Supply Revision Analysis')]`
+	oilDemandAnalysisSelector      = `div.ais-Hits li[contains(., 'Oil Demand Analysis')]`
+	oilSupplyAnalysisSelector      = `div.ais-Hits li[contains(., 'Oil Supply Analysis')]`
+	dateSlicerInputSelector        = `div.visual.customPadding.allow-deferred-rendering.visual-slicer input.date-slicer-input.enable-hover:first-of-type`
+	downloadButtonSelector         = `div.btn.btn-link.btn-sm.dashboard-action.dashboard-action--download-data`
 )
 
-func RzdDataDeal(cont context.Context) (err error) {
+// 函数用于设置查询时间范围
+func setQueryTime(ctx context.Context, year string) error {
+	return chromedp.Run(ctx,
+		chromedp.WaitVisible(dateSlicerInputSelector, chromedp.ByQuery),
+		chromedp.SetValue(dateSlicerInputSelector, year, chromedp.ByQuery),
+		chromedp.SendKeys(dateSlicerInputSelector, "\n"), // 回车查询
+	)
+}
 
-	// 读取 JSON 文件
-	configFile, err := os.ReadFile(utils.RZD_JSON_PATH)
-	if err != nil {
-		fmt.Printf("读取配置文件错误: %v\n", err)
-		return
+// 函数用于点击下载按钮
+func clickDownload(ctx context.Context) error {
+	return chromedp.Run(ctx, chromedp.Click(downloadButtonSelector, chromedp.ByQuery))
+}
+
+// 处理数据下载的步骤
+func downloadData(ctx context.Context) error {
+	// Analytics Library
+	if err := chromedp.Run(ctx,
+		chromedp.WaitVisible(clientSearchLink, chromedp.ByQuery),
+		chromedp.Click(clientSearchLink, chromedp.ByQuery),
+		chromedp.SetValue(`input[class="ais-SearchBox-input rounded border py-2 px-3 shadow-sm font-size-14 w-100"]`, "oil demand signals weekly report", chromedp.ByQuery),
+		chromedp.Click(`div.ais-InfiniteHits li:first-child img[src="/Static/img/icons/xls.png"]`, chromedp.ByQuery),
+	); err != nil {
+		return err
 	}
 
-	// 定义通用的 map 结构体来解析 JSON
-	var data map[string]map[string][]string
+	// Cube Dashboards: Supply Revision Analysis
+	if err := chromedp.Run(ctx,
+		chromedp.WaitVisible(`a[href="/clients/subscription/"]`, chromedp.ByQuery),
+		chromedp.Click(`a[href="/clients/subscription/"]`, chromedp.ByQuery),
+		chromedp.Click(supplyRevisionAnalysisSelector, chromedp.ByQuery),
+	); err != nil {
+		return err
+	}
+	if err := setQueryTime(ctx, "2020"); err != nil {
+		return err
+	}
+	if err := clickDownload(ctx); err != nil {
+		return err
+	}
+	if err := waitAndRenameDownloadedFile("Supply_Revision_Analysis_2020.xlsx"); err != nil {
+		return err
+	}
 
-	// 解析 JSON 文件内容
-	err = json.Unmarshal(configFile, &data)
+	// Oil Demand Analysis
+	if err := chromedp.Run(ctx,
+		chromedp.Click(`a[href="/clients/subscription/"]`, chromedp.ByQuery),
+		chromedp.Click(oilDemandAnalysisSelector, chromedp.ByQuery),
+	); err != nil {
+		return err
+	}
+	if err := setQueryTime(ctx, "2015"); err != nil {
+		return err
+	}
+	if err := clickDownload(ctx); err != nil {
+		return err
+	}
+	if err := waitAndRenameDownloadedFile("Oil_Demand_Analysis_2015.xlsx"); err != nil {
+		return err
+	}
+
+	// Oil Supply Analysis
+	if err := chromedp.Run(ctx,
+		chromedp.Click(`a[href="/clients/subscription/"]`, chromedp.ByQuery),
+		chromedp.Click(oilSupplyAnalysisSelector, chromedp.ByQuery),
+	); err != nil {
+		return err
+	}
+	if err := setQueryTime(ctx, "2010"); err != nil {
+		return err
+	}
+	if err := clickDownload(ctx); err != nil {
+		return err
+	}
+	if err := waitAndRenameDownloadedFile("Oil_Supply_Analysis_2010.xlsx"); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+// 等待下载文件并重命名
+func waitAndRenameDownloadedFile(newFileName string) error {
+	// 等待一段时间以确保文件下载完成
+	time.Sleep(5 * time.Second) // 可能需要根据实际情况调整
+
+	// 查找下载目录中的文件
+	files, err := filepath.Glob(filepath.Join(downloadDir, "*.xlsx"))
 	if err != nil {
-		fmt.Printf("解析配置文件错误: %v\n", err)
-		return
+		return fmt.Errorf("查找文件时出错: %v", err)
+	}
+
+	// 重命名最新的文件
+	for _, file := range files {
+		if err := os.Rename(file, filepath.Join(downloadDir, newFileName)); err != nil {
+			return fmt.Errorf("重命名文件时出错: %v", err)
+		}
+		// 打印重命名后的文件名
+		fmt.Printf("文件重命名为: %s\n", newFileName)
+		break // 只重命名第一个找到的文件
 	}
 
-	// 打印解析后的数据以验证
-	fmt.Printf("%+v\n", data)
+	return nil
+}
 
-	// 创建 chromedp 执行上下文
-	// 下载目录
-	downloadDir := "./downloads"
-	os.MkdirAll(downloadDir, os.ModePerm) // 创建下载目录
+func main() {
+	// 创建下载目录
+	if err := os.MkdirAll(downloadDir, os.ModePerm); err != nil {
+		fmt.Printf("创建下载目录时出错: %v\n", err)
+		return
+	}
 
+	// 创建 chromedp 执行上下文
 	options := []chromedp.ExecAllocatorOption{
 		chromedp.Flag("headless", false),
 		chromedp.Flag("disable-blink-features", "AutomationControlled"),
 		chromedp.UserAgent(`Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/98.0.4758.80 Safari/537.36`),
-
-		// 创建下载路径
 		chromedp.Flag("download.default_directory", downloadDir),
 		chromedp.Flag("download.prompt_for_download", false), // 不弹出下载对话框
 		chromedp.Flag("safebrowsing.enabled", true),          // 启用安全浏览
@@ -60,98 +152,25 @@ func RzdDataDeal(cont context.Context) (err error) {
 	ctx, cancel := chromedp.NewContext(allocCtx)
 	defer cancel()
 
-	// 登录操作
-	err = login(ctx)
-	if err != nil {
-		fmt.Printf("登录错误: %v\n", err)
+	// 启动 Chrome 实例
+	if err := chromedp.Run(ctx); err != nil {
+		fmt.Printf("启动 Chrome 实例时出错: %v\n", err)
 		return
 	}
 
-	// step_1：Analytics Library
-	err = chromedp.Run(ctx,
-		chromedp.WaitVisible(`a[href="/clients/search/"]`, chromedp.ByQuery),
-		chromedp.Click(`a[href="/clients/search/"]`, chromedp.ByQuery), // 点击链接
-		chromedp.Sleep(5*time.Second),
-
-		// 输入搜索内容
-		chromedp.SetValue(`input[class="ais-SearchBox-input rounded border py-2 px-3 shadow-sm font-size-14 w-100"]`, "oil demand signals weekly report", chromedp.ByQuery),
-		chromedp.Sleep(3*time.Second),
-
-		// 点击第一个 li 标签下的第一个 Excel 图标
-		chromedp.Click(`div.ais-InfiniteHits li:first-child img[src="/Static/img/icons/xls.png"]`, chromedp.ByQuery),
-		chromedp.Sleep(5*time.Second), // 等待下载完成
-	)
-	if err != nil {
-		return err
+	// 登录操作
+	if err := login(ctx); err != nil {
+		fmt.Printf("登录错误: %v\n", err)
+		return
 	}
 
-	// step_2：Cube Dashboards
-	err = chromedp.Run(ctx,
-		// Supply Revision Analysis 数据读取
-		chromedp.WaitVisible(`a[href="/clients/subscription/"]`, chromedp.ByQuery),
-		chromedp.Click(`a[href="/clients/subscription/"]`, chromedp.ByQuery), // 点击链接
-		chromedp.Sleep(5*time.Second),
-
-		// 点击包含 Supply Revision Analysis 的 li 元素
-		chromedp.Click(`div.ais-Hits li[contains(., 'Supply Revision Analysis')]`, chromedp.ByQuery),
-		chromedp.Sleep(10*time.Second),
-
-		// 设置查询时间范围
-		chromedp.WaitVisible(`div.visual.customPadding.allow-deferred-rendering.visual-slicer`, chromedp.ByQuery),                                                         // 等待 div 可见
-		chromedp.SetValue(`div.visual.customPadding.allow-deferred-rendering.visual-slicer input.date-slicer-input.enable-hover:first-of-type`, "2020", chromedp.ByQuery), // 设置第一个 input 的值为 2020
-		chromedp.Sleep(2*time.Second),
-		// 回车查询
-		chromedp.SendKeys(`div.visual.customPadding.allow-deferred-rendering.visual-slicer input.date-slicer-input.enable-hover:first-of-type`, "\n"),
-
-		// 点击下载按钮 下载表格
-		chromedp.Click(`div.[ btn btn-link btn-sm ] [ dashboard-action dashboard-action--download-data ]]`, chromedp.ByQuery),
-		chromedp.Sleep(5*time.Second), // 等待下载完成
-
-		// Oil Demand Analysis 数据读取
-		chromedp.WaitVisible(`a[href="/clients/subscription/"]`, chromedp.ByQuery),
-		chromedp.Click(`a[href="/clients/subscription/"]`, chromedp.ByQuery), // 点击链接
-		chromedp.Sleep(5*time.Second),
-
-		// 点击包含 Supply Revision Analysis 的 li 元素
-		chromedp.Click(`div.ais-Hits li[contains(., 'Oil Demand Analysis')]`, chromedp.ByQuery),
-		chromedp.Sleep(10*time.Second),
-
-		// 设置查询时间范围
-		chromedp.WaitVisible(`div.visual.customPadding.allow-deferred-rendering.visual-slicer`, chromedp.ByQuery), // 等待 div 可见
-		chromedp.SetValue(`div.visual.customPadding.allow-deferred-rendering.visual-slicer input.date-slicer-input.enable-hover:first-of-type`, "2015", chromedp.ByQuery),
-		chromedp.Sleep(2*time.Second),
-		// 回车查询
-		chromedp.SendKeys(`div.visual.customPadding.allow-deferred-rendering.visual-slicer input.date-slicer-input.enable-hover:first-of-type`, "\n"),
-
-		// 点击下载按钮 下载表格
-		chromedp.Click(`div.[ btn btn-link btn-sm ] [ dashboard-action dashboard-action--download-data ]]`, chromedp.ByQuery),
-		chromedp.Sleep(5*time.Second), // 等待下载完成
-
-		// Oil Supply Analysis 数据读取
-		chromedp.WaitVisible(`a[href="/clients/subscription/"]`, chromedp.ByQuery),
-		chromedp.Click(`a[href="/clients/subscription/"]`, chromedp.ByQuery), // 点击链接
-		chromedp.Sleep(5*time.Second),
-
-		// 点击包含 Supply Revision Analysis 的 li 元素
-		chromedp.Click(`div.ais-Hits li[contains(., 'Oil Demand Analysis')]`, chromedp.ByQuery),
-		chromedp.Sleep(10*time.Second),
-
-		// 设置查询时间范围
-		chromedp.WaitVisible(`div.visual.customPadding.allow-deferred-rendering.visual-slicer`, chromedp.ByQuery), // 等待 div 可见
-		chromedp.SetValue(`div.visual.customPadding.allow-deferred-rendering.visual-slicer input.date-slicer-input.enable-hover:first-of-type`, "2010", chromedp.ByQuery),
-		chromedp.Sleep(2*time.Second),
-		// 回车查询
-		chromedp.SendKeys(`div.visual.customPadding.allow-deferred-rendering.visual-slicer input.date-slicer-input.enable-hover:first-of-type`, "\n"),
-
-		// 点击下载按钮 下载表格
-		chromedp.Click(`div.[ btn btn-link btn-sm ] [ dashboard-action dashboard-action--download-data ]]`, chromedp.ByQuery),
-		chromedp.Sleep(5*time.Second), // 等待下载完成
-	)
-	if err != nil {
-		return err
+	// 下载数据
+	if err := downloadData(ctx); err != nil {
+		fmt.Printf("数据下载错误: %v\n", err)
+		return
 	}
 
-	return nil
+	fmt.Println("数据下载完成")
 }
 
 func login(ctx context.Context) error {
@@ -170,49 +189,3 @@ func login(ctx context.Context) error {
 		chromedp.Sleep(5*time.Second),                                 // 等待页面加载完成
 	)
 }
-
-func addLyIndex(classifyId int, indexCode string, indexName string, unit string, frequency string) (int, error) {
-	// 添加指标
-	index := &models.BaseFromLyIndex{
-		CreateTime:           utils.GetCurrentTime(),
-		ModifyTime:           utils.GetCurrentTime(),
-		BaseFromLyClassifyId: classifyId,
-		IndexCode:            indexCode,
-		IndexName:            indexName,
-		Frequency:            frequency,
-		Unit:                 unit,
-		EdbExist:             0,
-	}
-	postEdbLib, err := httpRequestFill(index, utils.ADD_LY_INDEX)
-	if err != nil {
-		// 有错误就不继续执行
-		log.Printf("postEdbLib err: %v", err)
-		return 0, err
-	}
-	var requestResponse models.RequestResponse[int64]
-	err = json.Unmarshal(postEdbLib, &requestResponse)
-	indexId := requestResponse.Data
-	return int(indexId), nil
-}
-
-func httpRequestFill(data interface{}, urlMethod string) (postEdbLib []byte, err error) {
-	// 转换成json
-	marshal, err := json.Marshal(data)
-	if err != nil {
-		return nil, err
-	}
-	// json 转 interface
-	var result map[string]interface{}
-	err = json.Unmarshal(marshal, &result)
-	if err != nil {
-		return nil, err
-	}
-
-	postEdbLib, err = utils.PostEdbLibRequest(result, urlMethod)
-	if err != nil {
-		// 有错误就不继续执行
-		log.Printf("postEdbLib err: %v", err)
-		return nil, err
-	}
-	return postEdbLib, nil
-}

services/ruizide/processor_business_logic.go

@@ -1,5 +1,5 @@
 // @Author gmy 2024/8/6 10:50:00
-package liangyou
+package main
 
 import (
 	"context"

services/ruizide/processor_factory.go

@@ -1,5 +1,5 @@
 // @Author gmy 2024/8/6 10:48:00
-package liangyou
+package main
 
 import (
 	"context"