package services import ( "context" "fmt" "github.com/beego/beego/v2/task" ) func Task() { fmt.Println("start crawler") SyncRankingFromShangHai() refreshData := task.NewTask("refreshData", "0 0,30 16-20 * * *", RefreshData) refreshEic := task.NewTask("refreshData", "0 0 2,6 * * *", RefreshEic) refreshCoal := task.NewTask("refreshData", "0 0,30 17-23 * * *", RefreshCoal) refreshVisitors := task.NewTask("RefreshChangesVisitorsCovid", "0 30 2-22/10 * * *", RefreshChangesVisitorsCovid) syncEiaSteoData := task.NewTask("SyncEiaSteoData", "0 0 22 * * *", SyncEiaSteoData) // 统计局-分月季年爬 //refreshNationalMonthA := task.NewTask("RefreshNationalMonthDbA", "0 15 2 10 * *", national_data.RefreshNationalMonthDbA) //refreshNationalMonthB := task.NewTask("RefreshNationalMonthDbB", "0 15 2 16 * *", national_data.RefreshNationalMonthDbB) //refreshNationalQuarter := task.NewTask("RefreshNationalQuarterDb", "0 25 1 15 * *", national_data.RefreshNationalQuarterDb) //refreshNationalYearA := task.NewTask("RefreshNationalYearDbA", "0 45 1 20 * *", national_data.RefreshNationalYearDbA) //refreshNationalYearB := task.NewTask("RefreshNationalYearDbB", "0 45 1 25 * *", national_data.RefreshNationalYearDbB) task.AddTask("数据爬取", refreshData) task.AddTask("欧洲天然气爬取", refreshEic) task.AddTask("中国煤炭网爬取", refreshCoal) task.AddTask("谷歌出行指数爬取", refreshVisitors) task.AddTask("eia steo报告", syncEiaSteoData) //每天22点爬一次 //task.AddTask("统计局数据爬取-月度A", refreshNationalMonthA) // 每月10号2:15执行 //task.AddTask("统计局数据爬取-月度B", refreshNationalMonthB) // 每月16号2:15执行 //task.AddTask("统计局数据爬取-季度", refreshNationalQuarter) // 每月15号1:25执行 //task.AddTask("统计局数据爬取-年度A", refreshNationalYearA) // 每月20日1:45执行 //task.AddTask("统计局数据爬取-年度B", refreshNationalYearB) // 每月25日1:45执行 task.StartTask() //FileCoalJsm() //FileCoalFirm() //FileCoalCoastal() //FileCoalInland() //SyncRankingFromDalianSearch(0) fmt.Println("end crawler") } func RefreshData(cont context.Context) (err error) { //大连 SyncRankingFromDalianSearch(0) SyncRankingFromZhengzhou() SyncRankingFromShangHai() SyncRankingFromIne() SyncRankingFromCffex() //SyncStorageFromEic() return } func RefreshEic(cont context.Context) (err error) { SyncStorageFromEicV2() return } func RefreshCoal(cont context.Context) (err error) { FileCoalJsm() FileCoalFirm() FileCoalCoastal() FileCoalInland() return } func RefreshChangesVisitorsCovid(cont context.Context) (err error) { err = AddSourceChangesVisitorsCovid() return } //func Task2() { // fmt.Println("start") // // var cont context.Context // _ = national_data.RefreshNationalYearDbA(cont) // // //_ = national_data.RefreshNationalYearDbB(cont) // // //_ = national_data.ApiTest() // fmt.Println("end") //}