eta_crawler/services/task.go

package services

import (
	"context"
	"fmt"
	"github.com/beego/beego/v2/task"
)

func Task() {
	//sync()
	//sync2()
	//sync3()
	//syncAllComTradeMapping()
	//syncAllComTradeMapping2()
	//return
	fmt.Println("start crawler")
	refreshData := task.NewTask("refreshData", "0 0,30 16-20 * * *", RefreshData)
	refreshEic := task.NewTask("refreshData", "0 0 2,6 * * *", RefreshEic)
	refreshCoal := task.NewTask("refreshData", "0 0,30 17-23 * * *", RefreshCoal)
	refreshVisitors := task.NewTask("RefreshChangesVisitorsCovid", "0 30 2-22/10 * * *", RefreshChangesVisitorsCovid)
	syncEiaSteoData := task.NewTask("SyncEiaSteoData", "0 0 22 * * *", SyncEiaSteoData)

	task.AddTask("数据爬取", refreshData)
	task.AddTask("欧洲天然气爬取", refreshEic)
	task.AddTask("中国煤炭网爬取", refreshCoal)
	task.AddTask("谷歌出行指数爬取", refreshVisitors)
	task.AddTask("eia steo报告", syncEiaSteoData) //每天22点爬一次
	task.StartTask()
	//FileCoalJsm()
	//FileCoalFirm()
	FileCoalCoastal()
	FileCoalInland()
	SyncRankingFromDalianSearch(0)
	fmt.Println("end crawler")
}

func RefreshData(cont context.Context) (err error) {
	//大连
	SyncRankingFromDalianSearch(0)
	SyncRankingFromZhengzhou()
	SyncRankingFromShangHai()
	SyncRankingFromIne()
	SyncRankingFromCffex()
	//SyncStorageFromEic()

	return
}

func RefreshEic(cont context.Context) (err error) {
	SyncStorageFromEicV2()
	return
}

func RefreshCoal(cont context.Context) (err error) {
	FileCoalJsm()
	FileCoalFirm()
	FileCoalCoastal()
	FileCoalInland()
	return
}

func RefreshChangesVisitorsCovid(cont context.Context) (err error) {
	err = AddSourceChangesVisitorsCovid()
	return
}