eta_crawler_local/services/fgenergy.go

package services

import (
	"context"
	"eta/eta_crawler_local/utils"
	"fmt"
	"io"
	"log"
	"net/http"
	"strings"
	"time"

	"github.com/chromedp/chromedp"
)

func autoLogin() {
	fmt.Println("task start")
	loginUrl := `https://www.fgenergy.com`
	userName := "lizhan.edri@sinopec.com"
	pwd := "Lzhan@@123"

	opts := append(
		chromedp.DefaultExecAllocatorOptions[:],
		chromedp.Flag("headless", false),
	)

	allocCtx, _ := chromedp.NewExecAllocator(context.Background(), opts...)
	//defer cancel()

	// 创建chrome实例
	ctx, _ := chromedp.NewContext(
		allocCtx,
		chromedp.WithLogf(log.Printf),
	)

	err := chromedp.Run(ctx,
		chromedp.Navigate(loginUrl),
		chromedp.Sleep(5*time.Second),
		chromedp.Click(`a[id="lnkClientLogin"]`, chromedp.ByQuery),
		chromedp.Sleep(2*time.Second),
		chromedp.SetValue(`input[id="ContentPlaceHolderDefault_MemberLogin_NEW_2_m_login_UserName"]`, userName, chromedp.ByQuery),
		chromedp.SetValue(`input[id="ContentPlaceHolderDefault_MemberLogin_NEW_2_m_login_Password"]`, pwd, chromedp.ByQuery),
		chromedp.Sleep(2*time.Second),
		chromedp.Click(`input[id="ContentPlaceHolderDefault_MemberLogin_NEW_2_m_login_LoginButton"]`, chromedp.ByQuery),
		chromedp.Sleep(5*time.Second),
	)

	fmt.Println(err)

	time.Sleep(3 * time.Second)
	//自动登录之后，拿到cookie
	autoDownload()
	fmt.Println("task end")
}

func autoDownload() {
	url := "https://www.fgenergy.com/download/51635/FGE+-+Fuel+Oil+Alert+-+18+December+2024.pdf"
	method := "GET"

	payload := strings.NewReader(``)

	client := &http.Client{}
	req, err := http.NewRequest(method, url, payload)

	if err != nil {
		fmt.Println(err)
		return
	}
	req.Header.Add("cookie", "ASP.NET_SessionId=10zktz2ng0zjt31y1b2jmrzg; _gid=GA1.2.832819655.1735175796; _lfa=LF1.1.cc68a12187105a86.1735175796600; hubspotutk=420d6b24dbf885d44e7b7a39127fc66e; __hssrc=1; cebs=1; _ce.clock_data=-1242%2C58.152.60.223%2C1%2Cf51bb482c660d0eeadd1f058058a2b35%2CChrome%2CHK; yourAuthCookie=7EC2C20F11960F813492FFE8E028D9A61E2FE22A2D8B9E2BA941EE56F404F1B4C5E134E26DBCA84BACEF0CF5930E1AE3E71DB4C0DC42F56687B0C3A4658A66073AA74AA5F85C3ECF0DF8F651276916338263F76BB409AF6EBAB7A763E344F1682CE37CDBEAA0A07C5ABC36250B6B44A848B9E4F24630CBEF1EB8D7C6E88E7287; _ga=GA1.2.809660828.1735175796; __hstc=229974915.420d6b24dbf885d44e7b7a39127fc66e.1735175797800.1735183452658.1735190316139.4; __hssc=229974915.1.1735190316139; cebsp_=13; _ga_YJLTHWFMBM=GS1.1.1735190312.4.1.1735190427.60.0.0; _ce.s=v~f971c41dae7a80ad9ab608fea61febb8d9592e02~lcw~1735191678959~vir~new~lva~1735175800122~vpv~0~v11.fhb~1735175801735~v11.lhb~1735191679319~v11.cs~416135~v11.s~dc300190-c348-11ef-a2bc-31da31de46a1~v11.sla~1735191678959~v11.send~1735190346263~lcw~1735191679319")

	res, err := client.Do(req)
	if err != nil {
		fmt.Println(err)
		return
	}
	defer res.Body.Close()

	body, err := io.ReadAll(res.Body)
	if err != nil {
		fmt.Println(err)
		return
	}
	fmt.Println(string(body))

	utils.SaveToFile(string(body), "1.pdf")
}