eta_crawler/services/task.go

package services

import (
	"context"
	"eta/eta_crawler/services/liangyou"
	"eta/eta_crawler/services/sci99"
	"eta/eta_crawler/utils"
	"fmt"
	"strconv"
	"time"

	"github.com/beego/beego/v2/task"
)

func Task() {
	//sync3()
	//syncAllComTradeMapping()
	//syncAllComTradeMapping2()
	//syncAllComTradeMapping3()
	//return
	fmt.Println("start crawler")
	//SyncRankingFromShangHai()
	//SyncRankingFromIne()
	//FileCoalCoastal()
	//FileCoalInland()
	if utils.BusinessCode == utils.BusinessCodeRelease {
		refreshUsdaPsd := task.NewTask("RefreshUsdaPsd", "0 0 16,19 * * 1,5", DownloadUsdaPsdDataTask) //每周一五下午16点，19点
		refreshUsdaFms := task.NewTask("RefreshUsdaFms", "0 0 16,19 * * 1,5", DownloadUsdaFmsDataTask) //每周一五下午16点，19点
		refreshData := task.NewTask("refreshData", "0 0,30 16-18 * * *", RefreshData)
		refreshEic := task.NewTask("RefreshEic", "0 0 2,6 * * *", RefreshEic)
		refreshCoal := task.NewTask("RefreshCoal", "0 0,30 16-23 * * *", RefreshCoal)
		refreshMeeting := task.NewTask("RefreshMeeting", "0 15 7-19/2 * * *", RefreshMeetingProbabilities)
		//	refreshVisitors := task.NewTask("RefreshChangesVisitorsCovid", "0 30 2-22/10 * * *", RefreshChangesVisitorsCovid)
		syncEiaSteoData := task.NewTask("SyncEiaSteoData", "0 0 22 * * *", SyncEiaSteoData)
		syncYearComTrade := task.NewTask("SyncYearComTrade", "0 0 3 1 1 *", SyncYearComTrade)                // 每年一月一号同步
		syncYearMonthComTrade := task.NewTask("SyncYearMonthComTrade", "0 0 3 1 * *", SyncYearMonthComTrade) // 每月1号同步

		crawlerIcpi := task.NewTask("refreshData", "0 0,30 16-23 * * *", CrawlerIcpi) //居民消费价格指数

		// 统计局-分月季年爬
		//refreshNationalMonthA := task.NewTask("RefreshNationalMonthDbA", "0 15 2 10 * *", national_data.RefreshNationalMonthDbA)
		//refreshNationalMonthB := task.NewTask("RefreshNationalMonthDbB", "0 15 2 16 * *", national_data.RefreshNationalMonthDbB)
		//refreshNationalQuarter := task.NewTask("RefreshNationalQuarterDb", "0 25 1 15 * *", national_data.RefreshNationalQuarterDb)
		//refreshNationalYearA := task.NewTask("RefreshNationalYearDbA", "0 45 1 20 * *", national_data.RefreshNationalYearDbA)
		//refreshNationalYearB := task.NewTask("RefreshNationalYearDbB", "0 45 1 25 * *", national_data.RefreshNationalYearDbB)
		task.AddTask("美国农业部月度供需数据爬取", refreshUsdaPsd)
		task.AddTask("美国农业部出口销售数据爬取", refreshUsdaFms)
		task.AddTask("数据爬取", refreshData)
		task.AddTask("欧洲天然气爬取", refreshEic)
		task.AddTask("中国煤炭网爬取", refreshCoal)
		task.AddTask("美联储加息概率爬取", refreshMeeting)
		//task.AddTask("谷歌出行指数爬取", refreshVisitors)
		task.AddTask("eia steo报告", syncEiaSteoData)   //每天22点爬一次
		task.AddTask("UN年度数据", syncYearComTrade)      //每年一月一号的3点同步
		task.AddTask("UN月度数据", syncYearMonthComTrade) //每月1号的3点同步

		task.AddTask("居民消费价格指数", crawlerIcpi) //每月1号的3点同步

	}

	if utils.LY_OPEN == "1" {
		lyData := task.NewTask("refreshData", "0 0 12 * * *", liangyou.LyDataDeal) // 粮油商务网

		task.AddTask("粮油商务网", lyData)
	}

	if utils.BusinessCode == utils.BusinessCodeFuBang {
		// 仅富邦
		sci99.Scii99(nil)
		sci99 := task.NewTask("refreshSci99", "0 0,30 16-20 * * *", sci99.Scii99) //卓创资讯
		task.AddTask("卓创资讯", sci99)
	}

	if utils.BusinessCode == utils.BusinessCodeJinRui {
		crawlerSmm := task.NewTask("refreshShangHaiSmmData", "0 0 11-20 * * *", SyncShangHaiSmm)       // 上海有色网爬虫
		task.AddTask("上海有色网指标爬取", crawlerSmm)                                                          //每天10-20点, 每小时爬一次
		crawlerSmmExcel := task.NewTask("refreshShangHaiSmmDataExcel", "0/30 * * * * *", SyncOldExcel) // 上海有色网爬虫
		task.AddTask("shanghaiOldExcel", crawlerSmmExcel)                                              // 爬取成功后删除该任务

	}

	//task.AddTask("统计局数据爬取-月度A", refreshNationalMonthA) // 每月10号2:15执行
	//task.AddTask("统计局数据爬取-月度B", refreshNationalMonthB) // 每月16号2:15执行
	//task.AddTask("统计局数据爬取-季度", refreshNationalQuarter) // 每月15号1:25执行
	//task.AddTask("统计局数据爬取-年度A", refreshNationalYearA)  // 每月20日1:45执行
	//task.AddTask("统计局数据爬取-年度B", refreshNationalYearB)  // 每月25日1:45执行

	task.StartTask()
	//FileCoalJsm()
	//FileCoalFirm()
	//FileCoalCoastal()
	//FileCoalInland()
	//SyncRankingFromDalianSearch(0)
	fmt.Println("end crawler")
}

func RefreshData(cont context.Context) (err error) {
	//大连
	SyncRankingFromDalianSearch(0)
	SyncRankingFromZhengzhou()
	//SyncRankingFromShangHai()
	//SyncRankingFromIne()
	SyncRankingFromCffex()
	//SyncStorageFromEic()
	//广州期货交易所
	{
		n := 2
		//广州交易所期货-日行情
		for i := n; i >= 0; i-- {
			date := time.Now().AddDate(0, 0, -i)
			if date.Weekday() == time.Saturday || date.Weekday() == time.Sunday {
				continue
			}
			CrawlerFromGuangzhouDayQuotes(date)
			rn := utils.RangeRand(10, 60)
			fmt.Println("i:"+strconv.Itoa(i)+" date:", date.Format(utils.FormatDate)+";sleep:", strconv.Itoa(int(rn)))
			time.Sleep(time.Duration(rn) * time.Second)
		}
		//合约
		for i := n; i >= 0; i-- {
			date := time.Now().AddDate(0, 0, -i)
			if date.Weekday() == time.Saturday || date.Weekday() == time.Sunday {
				continue
			}
			fmt.Println("date:", date.Format(utils.FormatDate))
			CrawlerFromGuangzhouContract(date)
			rn := utils.RangeRand(3, 15)
			fmt.Println("i:"+strconv.Itoa(i)+" date:", date.Format(utils.FormatDate)+";sleep:", strconv.Itoa(int(rn)))
			time.Sleep(time.Duration(rn) * time.Second)
		}

		//日成交持仓排名
		for i := n; i >= 0; i-- {
			date := time.Now().AddDate(0, 0, -i)
			if date.Weekday() == time.Saturday || date.Weekday() == time.Sunday {
				continue
			}
			fmt.Println("date:", date.Format(utils.FormatDate))
			CrawlerFromGuangzhouDealPosiQuotes(date)
			rn := utils.RangeRand(3, 15)
			fmt.Println("i:"+strconv.Itoa(i)+" date:", date.Format(utils.FormatDate)+";sleep:", strconv.Itoa(int(rn)))
			time.Sleep(time.Duration(rn) * time.Second)
		}
		//广州交易所期货-仓单日报->开始日期:2023-06-20
		for i := n; i >= 0; i-- {
			date := time.Now().AddDate(0, 0, -i)
			if date.Weekday() == time.Saturday || date.Weekday() == time.Sunday {
				continue
			}
			fmt.Println("date:", date.Format(utils.FormatDate))
			CrawlerFromGuangzhouWbillWeeklyQuotes(date)
			rn := utils.RangeRand(3, 15)
			fmt.Println("i:"+strconv.Itoa(i)+" date:", date.Format(utils.FormatDate)+";sleep:", strconv.Itoa(int(rn)))
			time.Sleep(time.Duration(rn) * time.Second)
		}

		//一次性交割卖方仓单查询
		for i := n; i >= 0; i-- {
			date := time.Now().AddDate(0, -i, 0)
			fmt.Println("date:", date.Format(utils.FormatDate))
			CrawlerFromGuangzhouCongregateDeliveryQuotes(date)
			rn := utils.RangeRand(3, 15)
			fmt.Println("i:"+strconv.Itoa(i)+" date:", date.Format(utils.FormatDate)+";sleep:", strconv.Itoa(int(rn)))
			time.Sleep(time.Duration(rn) * time.Second)
		}
		//更新指标日期
		CrawlerFromGuangzhouIndexMinAndMaxDate()
	}
	return
}

func RefreshEic(cont context.Context) (err error) {
	SyncStorageFromEicV2()
	return
}

func RefreshCoal(cont context.Context) (err error) {
	FileCoalJsm()
	FileCoalFirm()
	FileCoalCoastal()
	FileCoalInland()
	return
}

func RefreshChangesVisitorsCovid(cont context.Context) (err error) {
	err = AddSourceChangesVisitorsCovid()
	return
}

//func Task2() {
//	fmt.Println("start")
//
//	var cont context.Context
//	_ = national_data.RefreshNationalYearDbA(cont)
//
//	//_ = national_data.RefreshNationalYearDbB(cont)
//
//	//_ = national_data.ApiTest()
//	fmt.Println("end")
//}

func Task123() {
	fmt.Println("start")
	ctx := new(context.Context)
	CrawlerIcpi(*ctx)
	fmt.Println("end")
}