eta_crawler/services/source_trade_zhengzhou.go

package services

import (
	"context"
	"github.com/chromedp/chromedp"
	"log"
	"time"
)

func getSourceZhengZhou(url string) (string, error) {
	//增加选项，允许chrome窗口显示出来
	options := []chromedp.ExecAllocatorOption{
		//chromedp.Flag("headless", false),
		chromedp.Flag("disable-blink-features","AutomationControlled"),
		chromedp.UserAgent(`Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/98.0.4758.80 Safari/537.36`),
	}
	options = append(chromedp.DefaultExecAllocatorOptions[:], options...)
	//创建chrome窗口
	allocCtx, cancel := chromedp.NewExecAllocator(context.Background(), options...)
	defer cancel()
	newctx, cancel := chromedp.NewContext(allocCtx)
	defer cancel()

	// 给每个页面的爬取设置超时时间
	ctx, cancel := context.WithTimeout(newctx, 200 * time.Second)
	defer cancel()


	// run task list
	var res string

	err := chromedp.Run(ctx,
		chromedp.Tasks{
			/*chromedp.ActionFunc(func(cxt context.Context) error {
				_, err := page.AddScriptToEvaluateOnNewDocument("Object.defineProperty(navigator, 'webdriver', {get: () => undefined})").Do(cxt)
				if err != nil {
					return err
				}
				return nil
			}),*/
			chromedp.Navigate(url),
			chromedp.Sleep(5*time.Second),
			chromedp.InnerHTML("body", &res),
			//chromedp.EvaluateAsDevTools(`document.querySelector("#left > iframe").contentWindow.document.body.outerHTML;`, &res),
		},
	)
	if err != nil {
		log.Print(err)
		return "", err
	}
	return res, nil
}