|
@@ -1,55 +1,147 @@
|
|
|
-package liangyou
|
|
|
+package main
|
|
|
|
|
|
import (
|
|
|
"context"
|
|
|
- "encoding/json"
|
|
|
- "eta/eta_data_analysis/models"
|
|
|
"eta/eta_data_analysis/utils"
|
|
|
"fmt"
|
|
|
- "log"
|
|
|
"os"
|
|
|
+ "path/filepath"
|
|
|
"time"
|
|
|
|
|
|
"github.com/chromedp/chromedp"
|
|
|
)
|
|
|
|
|
|
+const downloadDir = "./downloads"
|
|
|
+
|
|
|
+// 定义选择器
|
|
|
var (
|
|
|
rzdLoginPath = "https://clients.rystadenergy.com/clients/"
|
|
|
+
|
|
|
+ clientSearchLink = `a[href="/clients/search/"]`
|
|
|
+ supplyRevisionAnalysisSelector = `div.ais-Hits li[contains(., 'Supply Revision Analysis')]`
|
|
|
+ oilDemandAnalysisSelector = `div.ais-Hits li[contains(., 'Oil Demand Analysis')]`
|
|
|
+ oilSupplyAnalysisSelector = `div.ais-Hits li[contains(., 'Oil Supply Analysis')]`
|
|
|
+ dateSlicerInputSelector = `div.visual.customPadding.allow-deferred-rendering.visual-slicer input.date-slicer-input.enable-hover:first-of-type`
|
|
|
+ downloadButtonSelector = `div.btn.btn-link.btn-sm.dashboard-action.dashboard-action--download-data`
|
|
|
)
|
|
|
|
|
|
-func RzdDataDeal(cont context.Context) (err error) {
|
|
|
+// 函数用于设置查询时间范围
|
|
|
+func setQueryTime(ctx context.Context, year string) error {
|
|
|
+ return chromedp.Run(ctx,
|
|
|
+ chromedp.WaitVisible(dateSlicerInputSelector, chromedp.ByQuery),
|
|
|
+ chromedp.SetValue(dateSlicerInputSelector, year, chromedp.ByQuery),
|
|
|
+ chromedp.SendKeys(dateSlicerInputSelector, "\n"), // 回车查询
|
|
|
+ )
|
|
|
+}
|
|
|
|
|
|
- // 读取 JSON 文件
|
|
|
- configFile, err := os.ReadFile(utils.RZD_JSON_PATH)
|
|
|
- if err != nil {
|
|
|
- fmt.Printf("读取配置文件错误: %v\n", err)
|
|
|
- return
|
|
|
+// 函数用于点击下载按钮
|
|
|
+func clickDownload(ctx context.Context) error {
|
|
|
+ return chromedp.Run(ctx, chromedp.Click(downloadButtonSelector, chromedp.ByQuery))
|
|
|
+}
|
|
|
+
|
|
|
+// 处理数据下载的步骤
|
|
|
+func downloadData(ctx context.Context) error {
|
|
|
+ // Analytics Library
|
|
|
+ if err := chromedp.Run(ctx,
|
|
|
+ chromedp.WaitVisible(clientSearchLink, chromedp.ByQuery),
|
|
|
+ chromedp.Click(clientSearchLink, chromedp.ByQuery),
|
|
|
+ chromedp.SetValue(`input[class="ais-SearchBox-input rounded border py-2 px-3 shadow-sm font-size-14 w-100"]`, "oil demand signals weekly report", chromedp.ByQuery),
|
|
|
+ chromedp.Click(`div.ais-InfiniteHits li:first-child img[src="/Static/img/icons/xls.png"]`, chromedp.ByQuery),
|
|
|
+ ); err != nil {
|
|
|
+ return err
|
|
|
}
|
|
|
|
|
|
- // 定义通用的 map 结构体来解析 JSON
|
|
|
- var data map[string]map[string][]string
|
|
|
+ // Cube Dashboards: Supply Revision Analysis
|
|
|
+ if err := chromedp.Run(ctx,
|
|
|
+ chromedp.WaitVisible(`a[href="/clients/subscription/"]`, chromedp.ByQuery),
|
|
|
+ chromedp.Click(`a[href="/clients/subscription/"]`, chromedp.ByQuery),
|
|
|
+ chromedp.Click(supplyRevisionAnalysisSelector, chromedp.ByQuery),
|
|
|
+ ); err != nil {
|
|
|
+ return err
|
|
|
+ }
|
|
|
+ if err := setQueryTime(ctx, "2020"); err != nil {
|
|
|
+ return err
|
|
|
+ }
|
|
|
+ if err := clickDownload(ctx); err != nil {
|
|
|
+ return err
|
|
|
+ }
|
|
|
+ if err := waitAndRenameDownloadedFile("Supply_Revision_Analysis_2020.xlsx"); err != nil {
|
|
|
+ return err
|
|
|
+ }
|
|
|
|
|
|
- // 解析 JSON 文件内容
|
|
|
- err = json.Unmarshal(configFile, &data)
|
|
|
+ // Oil Demand Analysis
|
|
|
+ if err := chromedp.Run(ctx,
|
|
|
+ chromedp.Click(`a[href="/clients/subscription/"]`, chromedp.ByQuery),
|
|
|
+ chromedp.Click(oilDemandAnalysisSelector, chromedp.ByQuery),
|
|
|
+ ); err != nil {
|
|
|
+ return err
|
|
|
+ }
|
|
|
+ if err := setQueryTime(ctx, "2015"); err != nil {
|
|
|
+ return err
|
|
|
+ }
|
|
|
+ if err := clickDownload(ctx); err != nil {
|
|
|
+ return err
|
|
|
+ }
|
|
|
+ if err := waitAndRenameDownloadedFile("Oil_Demand_Analysis_2015.xlsx"); err != nil {
|
|
|
+ return err
|
|
|
+ }
|
|
|
+
|
|
|
+ // Oil Supply Analysis
|
|
|
+ if err := chromedp.Run(ctx,
|
|
|
+ chromedp.Click(`a[href="/clients/subscription/"]`, chromedp.ByQuery),
|
|
|
+ chromedp.Click(oilSupplyAnalysisSelector, chromedp.ByQuery),
|
|
|
+ ); err != nil {
|
|
|
+ return err
|
|
|
+ }
|
|
|
+ if err := setQueryTime(ctx, "2010"); err != nil {
|
|
|
+ return err
|
|
|
+ }
|
|
|
+ if err := clickDownload(ctx); err != nil {
|
|
|
+ return err
|
|
|
+ }
|
|
|
+ if err := waitAndRenameDownloadedFile("Oil_Supply_Analysis_2010.xlsx"); err != nil {
|
|
|
+ return err
|
|
|
+ }
|
|
|
+
|
|
|
+ return nil
|
|
|
+}
|
|
|
+
|
|
|
+// 等待下载文件并重命名
|
|
|
+func waitAndRenameDownloadedFile(newFileName string) error {
|
|
|
+ // 等待一段时间以确保文件下载完成
|
|
|
+ time.Sleep(5 * time.Second) // 可能需要根据实际情况调整
|
|
|
+
|
|
|
+ // 查找下载目录中的文件
|
|
|
+ files, err := filepath.Glob(filepath.Join(downloadDir, "*.xlsx"))
|
|
|
if err != nil {
|
|
|
- fmt.Printf("解析配置文件错误: %v\n", err)
|
|
|
- return
|
|
|
+ return fmt.Errorf("查找文件时出错: %v", err)
|
|
|
+ }
|
|
|
+
|
|
|
+ // 重命名最新的文件
|
|
|
+ for _, file := range files {
|
|
|
+ if err := os.Rename(file, filepath.Join(downloadDir, newFileName)); err != nil {
|
|
|
+ return fmt.Errorf("重命名文件时出错: %v", err)
|
|
|
+ }
|
|
|
+ // 打印重命名后的文件名
|
|
|
+ fmt.Printf("文件重命名为: %s\n", newFileName)
|
|
|
+ break // 只重命名第一个找到的文件
|
|
|
}
|
|
|
|
|
|
- // 打印解析后的数据以验证
|
|
|
- fmt.Printf("%+v\n", data)
|
|
|
+ return nil
|
|
|
+}
|
|
|
|
|
|
- // 创建 chromedp 执行上下文
|
|
|
- // 下载目录
|
|
|
- downloadDir := "./downloads"
|
|
|
- os.MkdirAll(downloadDir, os.ModePerm) // 创建下载目录
|
|
|
+func main() {
|
|
|
+ // 创建下载目录
|
|
|
+ if err := os.MkdirAll(downloadDir, os.ModePerm); err != nil {
|
|
|
+ fmt.Printf("创建下载目录时出错: %v\n", err)
|
|
|
+ return
|
|
|
+ }
|
|
|
|
|
|
+ // 创建 chromedp 执行上下文
|
|
|
options := []chromedp.ExecAllocatorOption{
|
|
|
chromedp.Flag("headless", false),
|
|
|
chromedp.Flag("disable-blink-features", "AutomationControlled"),
|
|
|
chromedp.UserAgent(`Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/98.0.4758.80 Safari/537.36`),
|
|
|
-
|
|
|
- // 创建下载路径
|
|
|
chromedp.Flag("download.default_directory", downloadDir),
|
|
|
chromedp.Flag("download.prompt_for_download", false), // 不弹出下载对话框
|
|
|
chromedp.Flag("safebrowsing.enabled", true), // 启用安全浏览
|
|
@@ -60,98 +152,25 @@ func RzdDataDeal(cont context.Context) (err error) {
|
|
|
ctx, cancel := chromedp.NewContext(allocCtx)
|
|
|
defer cancel()
|
|
|
|
|
|
- // 登录操作
|
|
|
- err = login(ctx)
|
|
|
- if err != nil {
|
|
|
- fmt.Printf("登录错误: %v\n", err)
|
|
|
+ // 启动 Chrome 实例
|
|
|
+ if err := chromedp.Run(ctx); err != nil {
|
|
|
+ fmt.Printf("启动 Chrome 实例时出错: %v\n", err)
|
|
|
return
|
|
|
}
|
|
|
|
|
|
- // step_1:Analytics Library
|
|
|
- err = chromedp.Run(ctx,
|
|
|
- chromedp.WaitVisible(`a[href="/clients/search/"]`, chromedp.ByQuery),
|
|
|
- chromedp.Click(`a[href="/clients/search/"]`, chromedp.ByQuery), // 点击链接
|
|
|
- chromedp.Sleep(5*time.Second),
|
|
|
-
|
|
|
- // 输入搜索内容
|
|
|
- chromedp.SetValue(`input[class="ais-SearchBox-input rounded border py-2 px-3 shadow-sm font-size-14 w-100"]`, "oil demand signals weekly report", chromedp.ByQuery),
|
|
|
- chromedp.Sleep(3*time.Second),
|
|
|
-
|
|
|
- // 点击第一个 li 标签下的第一个 Excel 图标
|
|
|
- chromedp.Click(`div.ais-InfiniteHits li:first-child img[src="/Static/img/icons/xls.png"]`, chromedp.ByQuery),
|
|
|
- chromedp.Sleep(5*time.Second), // 等待下载完成
|
|
|
- )
|
|
|
- if err != nil {
|
|
|
- return err
|
|
|
+ // 登录操作
|
|
|
+ if err := login(ctx); err != nil {
|
|
|
+ fmt.Printf("登录错误: %v\n", err)
|
|
|
+ return
|
|
|
}
|
|
|
|
|
|
- // step_2:Cube Dashboards
|
|
|
- err = chromedp.Run(ctx,
|
|
|
- // Supply Revision Analysis 数据读取
|
|
|
- chromedp.WaitVisible(`a[href="/clients/subscription/"]`, chromedp.ByQuery),
|
|
|
- chromedp.Click(`a[href="/clients/subscription/"]`, chromedp.ByQuery), // 点击链接
|
|
|
- chromedp.Sleep(5*time.Second),
|
|
|
-
|
|
|
- // 点击包含 Supply Revision Analysis 的 li 元素
|
|
|
- chromedp.Click(`div.ais-Hits li[contains(., 'Supply Revision Analysis')]`, chromedp.ByQuery),
|
|
|
- chromedp.Sleep(10*time.Second),
|
|
|
-
|
|
|
- // 设置查询时间范围
|
|
|
- chromedp.WaitVisible(`div.visual.customPadding.allow-deferred-rendering.visual-slicer`, chromedp.ByQuery), // 等待 div 可见
|
|
|
- chromedp.SetValue(`div.visual.customPadding.allow-deferred-rendering.visual-slicer input.date-slicer-input.enable-hover:first-of-type`, "2020", chromedp.ByQuery), // 设置第一个 input 的值为 2020
|
|
|
- chromedp.Sleep(2*time.Second),
|
|
|
- // 回车查询
|
|
|
- chromedp.SendKeys(`div.visual.customPadding.allow-deferred-rendering.visual-slicer input.date-slicer-input.enable-hover:first-of-type`, "\n"),
|
|
|
-
|
|
|
- // 点击下载按钮 下载表格
|
|
|
- chromedp.Click(`div.[ btn btn-link btn-sm ] [ dashboard-action dashboard-action--download-data ]]`, chromedp.ByQuery),
|
|
|
- chromedp.Sleep(5*time.Second), // 等待下载完成
|
|
|
-
|
|
|
- // Oil Demand Analysis 数据读取
|
|
|
- chromedp.WaitVisible(`a[href="/clients/subscription/"]`, chromedp.ByQuery),
|
|
|
- chromedp.Click(`a[href="/clients/subscription/"]`, chromedp.ByQuery), // 点击链接
|
|
|
- chromedp.Sleep(5*time.Second),
|
|
|
-
|
|
|
- // 点击包含 Supply Revision Analysis 的 li 元素
|
|
|
- chromedp.Click(`div.ais-Hits li[contains(., 'Oil Demand Analysis')]`, chromedp.ByQuery),
|
|
|
- chromedp.Sleep(10*time.Second),
|
|
|
-
|
|
|
- // 设置查询时间范围
|
|
|
- chromedp.WaitVisible(`div.visual.customPadding.allow-deferred-rendering.visual-slicer`, chromedp.ByQuery), // 等待 div 可见
|
|
|
- chromedp.SetValue(`div.visual.customPadding.allow-deferred-rendering.visual-slicer input.date-slicer-input.enable-hover:first-of-type`, "2015", chromedp.ByQuery),
|
|
|
- chromedp.Sleep(2*time.Second),
|
|
|
- // 回车查询
|
|
|
- chromedp.SendKeys(`div.visual.customPadding.allow-deferred-rendering.visual-slicer input.date-slicer-input.enable-hover:first-of-type`, "\n"),
|
|
|
-
|
|
|
- // 点击下载按钮 下载表格
|
|
|
- chromedp.Click(`div.[ btn btn-link btn-sm ] [ dashboard-action dashboard-action--download-data ]]`, chromedp.ByQuery),
|
|
|
- chromedp.Sleep(5*time.Second), // 等待下载完成
|
|
|
-
|
|
|
- // Oil Supply Analysis 数据读取
|
|
|
- chromedp.WaitVisible(`a[href="/clients/subscription/"]`, chromedp.ByQuery),
|
|
|
- chromedp.Click(`a[href="/clients/subscription/"]`, chromedp.ByQuery), // 点击链接
|
|
|
- chromedp.Sleep(5*time.Second),
|
|
|
-
|
|
|
- // 点击包含 Supply Revision Analysis 的 li 元素
|
|
|
- chromedp.Click(`div.ais-Hits li[contains(., 'Oil Demand Analysis')]`, chromedp.ByQuery),
|
|
|
- chromedp.Sleep(10*time.Second),
|
|
|
-
|
|
|
- // 设置查询时间范围
|
|
|
- chromedp.WaitVisible(`div.visual.customPadding.allow-deferred-rendering.visual-slicer`, chromedp.ByQuery), // 等待 div 可见
|
|
|
- chromedp.SetValue(`div.visual.customPadding.allow-deferred-rendering.visual-slicer input.date-slicer-input.enable-hover:first-of-type`, "2010", chromedp.ByQuery),
|
|
|
- chromedp.Sleep(2*time.Second),
|
|
|
- // 回车查询
|
|
|
- chromedp.SendKeys(`div.visual.customPadding.allow-deferred-rendering.visual-slicer input.date-slicer-input.enable-hover:first-of-type`, "\n"),
|
|
|
-
|
|
|
- // 点击下载按钮 下载表格
|
|
|
- chromedp.Click(`div.[ btn btn-link btn-sm ] [ dashboard-action dashboard-action--download-data ]]`, chromedp.ByQuery),
|
|
|
- chromedp.Sleep(5*time.Second), // 等待下载完成
|
|
|
- )
|
|
|
- if err != nil {
|
|
|
- return err
|
|
|
+ // 下载数据
|
|
|
+ if err := downloadData(ctx); err != nil {
|
|
|
+ fmt.Printf("数据下载错误: %v\n", err)
|
|
|
+ return
|
|
|
}
|
|
|
|
|
|
- return nil
|
|
|
+ fmt.Println("数据下载完成")
|
|
|
}
|
|
|
|
|
|
func login(ctx context.Context) error {
|
|
@@ -170,49 +189,3 @@ func login(ctx context.Context) error {
|
|
|
chromedp.Sleep(5*time.Second), // 等待页面加载完成
|
|
|
)
|
|
|
}
|
|
|
-
|
|
|
-func addLyIndex(classifyId int, indexCode string, indexName string, unit string, frequency string) (int, error) {
|
|
|
- // 添加指标
|
|
|
- index := &models.BaseFromLyIndex{
|
|
|
- CreateTime: utils.GetCurrentTime(),
|
|
|
- ModifyTime: utils.GetCurrentTime(),
|
|
|
- BaseFromLyClassifyId: classifyId,
|
|
|
- IndexCode: indexCode,
|
|
|
- IndexName: indexName,
|
|
|
- Frequency: frequency,
|
|
|
- Unit: unit,
|
|
|
- EdbExist: 0,
|
|
|
- }
|
|
|
- postEdbLib, err := httpRequestFill(index, utils.ADD_LY_INDEX)
|
|
|
- if err != nil {
|
|
|
- // 有错误就不继续执行
|
|
|
- log.Printf("postEdbLib err: %v", err)
|
|
|
- return 0, err
|
|
|
- }
|
|
|
- var requestResponse models.RequestResponse[int64]
|
|
|
- err = json.Unmarshal(postEdbLib, &requestResponse)
|
|
|
- indexId := requestResponse.Data
|
|
|
- return int(indexId), nil
|
|
|
-}
|
|
|
-
|
|
|
-func httpRequestFill(data interface{}, urlMethod string) (postEdbLib []byte, err error) {
|
|
|
- // 转换成json
|
|
|
- marshal, err := json.Marshal(data)
|
|
|
- if err != nil {
|
|
|
- return nil, err
|
|
|
- }
|
|
|
- // json 转 interface
|
|
|
- var result map[string]interface{}
|
|
|
- err = json.Unmarshal(marshal, &result)
|
|
|
- if err != nil {
|
|
|
- return nil, err
|
|
|
- }
|
|
|
-
|
|
|
- postEdbLib, err = utils.PostEdbLibRequest(result, urlMethod)
|
|
|
- if err != nil {
|
|
|
- // 有错误就不继续执行
|
|
|
- log.Printf("postEdbLib err: %v", err)
|
|
|
- return nil, err
|
|
|
- }
|
|
|
- return postEdbLib, nil
|
|
|
-}
|