eta_crawler/services/task.go

package services

import (
	"context"
	"fmt"
	"github.com/beego/beego/v2/task"
)

func Task() {
	//sync3()
	//syncAllComTradeMapping()
	//syncAllComTradeMapping2()
	//syncAllComTradeMapping3()
	//return
	fmt.Println("start crawler")
	//SyncRankingFromShangHai()
	//SyncRankingFromIne()
	FileCoalCoastal()
	FileCoalInland()
	refreshData := task.NewTask("refreshData", "0 0,30 16-18 * * *", RefreshData)
	refreshEic := task.NewTask("RefreshEic", "0 0 2,6 * * *", RefreshEic)
	refreshCoal := task.NewTask("RefreshCoal", "0 0,30 16-23 * * *", RefreshCoal)
	refreshVisitors := task.NewTask("RefreshChangesVisitorsCovid", "0 30 2-22/10 * * *", RefreshChangesVisitorsCovid)
	syncEiaSteoData := task.NewTask("SyncEiaSteoData", "0 0 22 * * *", SyncEiaSteoData)
	syncYearComTrade := task.NewTask("SyncYearComTrade", "0 0 3 1 1 *", SyncYearComTrade)                // 每年一月一号同步
	syncYearMonthComTrade := task.NewTask("SyncYearMonthComTrade", "0 0 3 1 * *", SyncYearMonthComTrade) // 每月1号同步

	// 统计局-分月季年爬
	//refreshNationalMonthA := task.NewTask("RefreshNationalMonthDbA", "0 15 2 10 * *", national_data.RefreshNationalMonthDbA)
	//refreshNationalMonthB := task.NewTask("RefreshNationalMonthDbB", "0 15 2 16 * *", national_data.RefreshNationalMonthDbB)
	//refreshNationalQuarter := task.NewTask("RefreshNationalQuarterDb", "0 25 1 15 * *", national_data.RefreshNationalQuarterDb)
	//refreshNationalYearA := task.NewTask("RefreshNationalYearDbA", "0 45 1 20 * *", national_data.RefreshNationalYearDbA)
	//refreshNationalYearB := task.NewTask("RefreshNationalYearDbB", "0 45 1 25 * *", national_data.RefreshNationalYearDbB)

	task.AddTask("数据爬取", refreshData)
	task.AddTask("欧洲天然气爬取", refreshEic)
	task.AddTask("中国煤炭网爬取", refreshCoal)
	task.AddTask("谷歌出行指数爬取", refreshVisitors)
	task.AddTask("eia steo报告", syncEiaSteoData)   //每天22点爬一次
	task.AddTask("UN年度数据", syncYearComTrade)      //每年一月一号的3点同步
	task.AddTask("UN月度数据", syncYearMonthComTrade) //每月1号的3点同步

	//task.AddTask("统计局数据爬取-月度A", refreshNationalMonthA) // 每月10号2:15执行
	//task.AddTask("统计局数据爬取-月度B", refreshNationalMonthB) // 每月16号2:15执行
	//task.AddTask("统计局数据爬取-季度", refreshNationalQuarter) // 每月15号1:25执行
	//task.AddTask("统计局数据爬取-年度A", refreshNationalYearA)  // 每月20日1:45执行
	//task.AddTask("统计局数据爬取-年度B", refreshNationalYearB)  // 每月25日1:45执行

	task.StartTask()
	//FileCoalJsm()
	//FileCoalFirm()
	//FileCoalCoastal()
	//FileCoalInland()
	//SyncRankingFromDalianSearch(0)
	fmt.Println("end crawler")
}

func RefreshData(cont context.Context) (err error) {
	//大连
	SyncRankingFromDalianSearch(0)
	SyncRankingFromZhengzhou()
	//SyncRankingFromShangHai()
	//SyncRankingFromIne()
	SyncRankingFromCffex()
	//SyncStorageFromEic()

	return
}

func RefreshEic(cont context.Context) (err error) {
	SyncStorageFromEicV2()
	return
}

func RefreshCoal(cont context.Context) (err error) {
	FileCoalJsm()
	FileCoalFirm()
	FileCoalCoastal()
	FileCoalInland()
	return
}

func RefreshChangesVisitorsCovid(cont context.Context) (err error) {
	err = AddSourceChangesVisitorsCovid()
	return
}

//func Task2() {
//	fmt.Println("start")
//
//	var cont context.Context
//	_ = national_data.RefreshNationalYearDbA(cont)
//
//	//_ = national_data.RefreshNationalYearDbB(cont)
//
//	//_ = national_data.ApiTest()
//	fmt.Println("end")
//}