|
@@ -0,0 +1,98 @@
|
|
|
+package services
|
|
|
+
|
|
|
+import (
|
|
|
+ "context"
|
|
|
+ "fmt"
|
|
|
+ "github.com/chromedp/cdproto/browser"
|
|
|
+ "github.com/chromedp/chromedp"
|
|
|
+ "log"
|
|
|
+ "os"
|
|
|
+ "path/filepath"
|
|
|
+ "time"
|
|
|
+)
|
|
|
+
|
|
|
+func GetSourceChangesVisitorsCovid(dirPath string) (filePathStr string, err error) {
|
|
|
+ options := []chromedp.ExecAllocatorOption{
|
|
|
+ chromedp.WindowSize(1920,1080),
|
|
|
+ chromedp.UserAgent(`Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/98.0.4758.80 Safari/537.36`),
|
|
|
+ }
|
|
|
+ options = append(chromedp.DefaultExecAllocatorOptions[:], options...)
|
|
|
+
|
|
|
+ allocCtx, cancel := chromedp.NewExecAllocator(context.Background(), options...)
|
|
|
+
|
|
|
+
|
|
|
+ ctx, cancel := chromedp.NewContext(
|
|
|
+ allocCtx,
|
|
|
+ chromedp.WithLogf(log.Printf),
|
|
|
+ )
|
|
|
+ defer cancel()
|
|
|
+
|
|
|
+
|
|
|
+ ctx, cancel = context.WithTimeout(ctx, 200*time.Second)
|
|
|
+ defer cancel()
|
|
|
+
|
|
|
+
|
|
|
+
|
|
|
+
|
|
|
+ done := make(chan string, 1)
|
|
|
+
|
|
|
+
|
|
|
+
|
|
|
+
|
|
|
+ chromedp.ListenTarget(ctx, func(v interface{}) {
|
|
|
+ if ev, ok := v.(*browser.EventDownloadProgress); ok {
|
|
|
+ completed := "(unknown)"
|
|
|
+ if ev.TotalBytes != 0 {
|
|
|
+ completed = fmt.Sprintf("%0.2f%%", ev.ReceivedBytes/ev.TotalBytes*100.0)
|
|
|
+ }
|
|
|
+ log.Printf("state: %s, completed: %s\n", ev.State.String(), completed)
|
|
|
+ if ev.State == browser.DownloadProgressStateCompleted {
|
|
|
+ done <- ev.GUID
|
|
|
+ close(done)
|
|
|
+ log.Print("download finished")
|
|
|
+ }
|
|
|
+ }
|
|
|
+ })
|
|
|
+
|
|
|
+ if dirPath == "" {
|
|
|
+ dirPath, err = os.Getwd()
|
|
|
+ if err != nil {
|
|
|
+ log.Fatal(err)
|
|
|
+ }
|
|
|
+ }
|
|
|
+
|
|
|
+
|
|
|
+
|
|
|
+
|
|
|
+
|
|
|
+ err = chromedp.Run(ctx,
|
|
|
+ chromedp.Navigate(`https://ourworldindata.org/grapher/changes-visitors-covid`),
|
|
|
+
|
|
|
+ chromedp.WaitVisible(`.GrapherComponent`),
|
|
|
+ chromedp.ScrollIntoView(`//figure[contains(@data-grapher-src,"changes-visitors-covid")]//ul//li//a[contains(@data-track-note, "chart-click-download")]`),
|
|
|
+
|
|
|
+ chromedp.Click(`//figure[contains(@data-grapher-src,"changes-visitors-covid")]//ul//li//a[contains(@data-track-note, "chart-click-download")]`, chromedp.NodeReady),
|
|
|
+
|
|
|
+
|
|
|
+
|
|
|
+
|
|
|
+
|
|
|
+ browser.SetDownloadBehavior(browser.SetDownloadBehaviorBehaviorAllowAndName).
|
|
|
+ WithDownloadPath(dirPath).
|
|
|
+ WithEventsEnabled(true),
|
|
|
+ chromedp.Click(`//div//button[contains(@data-track-note, "chart-download-csv")]`, chromedp.NodeVisible),
|
|
|
+ )
|
|
|
+ if err != nil {
|
|
|
+ log.Fatal(err)
|
|
|
+ }
|
|
|
+
|
|
|
+
|
|
|
+
|
|
|
+ guid := <-done
|
|
|
+
|
|
|
+
|
|
|
+
|
|
|
+ filePathStr = filepath.Join(dirPath, guid)
|
|
|
+ log.Printf("wrote %s", filePathStr)
|
|
|
+ return
|
|
|
+}
|