package services import ( "context" "eta/eta_crawler/services/liangyou" "eta/eta_crawler/services/sci99" "eta/eta_crawler/utils" "fmt" "strconv" "time" "github.com/beego/beego/v2/task" ) func Task() { //sync3() //syncAllComTradeMapping() //syncAllComTradeMapping2() //syncAllComTradeMapping3() //return fmt.Println("start crawler") //SyncRankingFromShangHai() //SyncRankingFromIne() //FileCoalCoastal() //FileCoalInland() if utils.BusinessCode == utils.BusinessCodeRelease { refreshUsdaPsd := task.NewTask("RefreshUsdaPsd", "0 0 16,19 * * 1,5", DownloadUsdaPsdDataTask) //每周一五下午16点,19点 refreshUsdaFms := task.NewTask("RefreshUsdaFms", "0 0 16,19 * * 1,5", DownloadUsdaFmsDataTask) //每周一五下午16点,19点 refreshData := task.NewTask("refreshData", "0 0,30 16-18 * * *", RefreshData) refreshEic := task.NewTask("RefreshEic", "0 0 2,6 * * *", RefreshEic) refreshCoal := task.NewTask("RefreshCoal", "0 0,30 16-23 * * *", RefreshCoal) refreshMeeting := task.NewTask("RefreshMeeting", "0 15 7-19/2 * * *", RefreshMeetingProbabilities) // refreshVisitors := task.NewTask("RefreshChangesVisitorsCovid", "0 30 2-22/10 * * *", RefreshChangesVisitorsCovid) syncEiaSteoData := task.NewTask("SyncEiaSteoData", "0 0 22 * * *", SyncEiaSteoData) syncYearComTrade := task.NewTask("SyncYearComTrade", "0 0 3 1 1 *", SyncYearComTrade) // 每年一月一号同步 syncYearMonthComTrade := task.NewTask("SyncYearMonthComTrade", "0 0 3 1 * *", SyncYearMonthComTrade) // 每月1号同步 crawlerIcpi := task.NewTask("refreshData", "0 0,30 16-23 * * *", CrawlerIcpi) //居民消费价格指数 // 统计局-分月季年爬 //refreshNationalMonthA := task.NewTask("RefreshNationalMonthDbA", "0 15 2 10 * *", national_data.RefreshNationalMonthDbA) //refreshNationalMonthB := task.NewTask("RefreshNationalMonthDbB", "0 15 2 16 * *", national_data.RefreshNationalMonthDbB) //refreshNationalQuarter := task.NewTask("RefreshNationalQuarterDb", "0 25 1 15 * *", national_data.RefreshNationalQuarterDb) //refreshNationalYearA := task.NewTask("RefreshNationalYearDbA", "0 45 1 20 * *", national_data.RefreshNationalYearDbA) //refreshNationalYearB := task.NewTask("RefreshNationalYearDbB", "0 45 1 25 * *", national_data.RefreshNationalYearDbB) task.AddTask("美国农业部月度供需数据爬取", refreshUsdaPsd) task.AddTask("美国农业部出口销售数据爬取", refreshUsdaFms) task.AddTask("数据爬取", refreshData) task.AddTask("欧洲天然气爬取", refreshEic) task.AddTask("中国煤炭网爬取", refreshCoal) task.AddTask("美联储加息概率爬取", refreshMeeting) //task.AddTask("谷歌出行指数爬取", refreshVisitors) task.AddTask("eia steo报告", syncEiaSteoData) //每天22点爬一次 task.AddTask("UN年度数据", syncYearComTrade) //每年一月一号的3点同步 task.AddTask("UN月度数据", syncYearMonthComTrade) //每月1号的3点同步 task.AddTask("居民消费价格指数", crawlerIcpi) //每月1号的3点同步 } if utils.LY_OPEN == "1" { lyData := task.NewTask("refreshData", "0 0 12 * * *", liangyou.LyDataDeal) // 粮油商务网 task.AddTask("粮油商务网", lyData) } if utils.BusinessCode == utils.BusinessCodeFuBang { // 仅富邦 sci99.Scii99(nil) sci99 := task.NewTask("refreshSci99", "0 0,30 16-20 * * *", sci99.Scii99) //卓创资讯 task.AddTask("卓创资讯", sci99) } if utils.BusinessCode == utils.BusinessCodeJinRui { crawlerSmm := task.NewTask("refreshShangHaiSmmData", "0 0 11-20 * * *", SyncShangHaiSmm) // 上海有色网爬虫 task.AddTask("上海有色网指标爬取", crawlerSmm) //每天10-20点, 每小时爬一次 crawlerSmmExcel := task.NewTask("refreshShangHaiSmmDataExcel", "0/30 * * * * *", SyncOldExcel) // 上海有色网爬虫 task.AddTask("shanghaiOldExcel", crawlerSmmExcel) // 爬取成功后删除该任务 } //task.AddTask("统计局数据爬取-月度A", refreshNationalMonthA) // 每月10号2:15执行 //task.AddTask("统计局数据爬取-月度B", refreshNationalMonthB) // 每月16号2:15执行 //task.AddTask("统计局数据爬取-季度", refreshNationalQuarter) // 每月15号1:25执行 //task.AddTask("统计局数据爬取-年度A", refreshNationalYearA) // 每月20日1:45执行 //task.AddTask("统计局数据爬取-年度B", refreshNationalYearB) // 每月25日1:45执行 task.StartTask() //FileCoalJsm() //FileCoalFirm() //FileCoalCoastal() //FileCoalInland() //SyncRankingFromDalianSearch(0) fmt.Println("end crawler") } func RefreshData(cont context.Context) (err error) { //大连 SyncRankingFromDalianSearch(0) SyncRankingFromZhengzhou() //SyncRankingFromShangHai() //SyncRankingFromIne() SyncRankingFromCffex() //SyncStorageFromEic() //广州期货交易所 { n := 2 //广州交易所期货-日行情 for i := n; i >= 0; i-- { date := time.Now().AddDate(0, 0, -i) if date.Weekday() == time.Saturday || date.Weekday() == time.Sunday { continue } CrawlerFromGuangzhouDayQuotes(date) rn := utils.RangeRand(10, 60) fmt.Println("i:"+strconv.Itoa(i)+" date:", date.Format(utils.FormatDate)+";sleep:", strconv.Itoa(int(rn))) time.Sleep(time.Duration(rn) * time.Second) } //合约 for i := n; i >= 0; i-- { date := time.Now().AddDate(0, 0, -i) if date.Weekday() == time.Saturday || date.Weekday() == time.Sunday { continue } fmt.Println("date:", date.Format(utils.FormatDate)) CrawlerFromGuangzhouContract(date) rn := utils.RangeRand(3, 15) fmt.Println("i:"+strconv.Itoa(i)+" date:", date.Format(utils.FormatDate)+";sleep:", strconv.Itoa(int(rn))) time.Sleep(time.Duration(rn) * time.Second) } //日成交持仓排名 for i := n; i >= 0; i-- { date := time.Now().AddDate(0, 0, -i) if date.Weekday() == time.Saturday || date.Weekday() == time.Sunday { continue } fmt.Println("date:", date.Format(utils.FormatDate)) CrawlerFromGuangzhouDealPosiQuotes(date) rn := utils.RangeRand(3, 15) fmt.Println("i:"+strconv.Itoa(i)+" date:", date.Format(utils.FormatDate)+";sleep:", strconv.Itoa(int(rn))) time.Sleep(time.Duration(rn) * time.Second) } //广州交易所期货-仓单日报->开始日期:2023-06-20 for i := n; i >= 0; i-- { date := time.Now().AddDate(0, 0, -i) if date.Weekday() == time.Saturday || date.Weekday() == time.Sunday { continue } fmt.Println("date:", date.Format(utils.FormatDate)) CrawlerFromGuangzhouWbillWeeklyQuotes(date) rn := utils.RangeRand(3, 15) fmt.Println("i:"+strconv.Itoa(i)+" date:", date.Format(utils.FormatDate)+";sleep:", strconv.Itoa(int(rn))) time.Sleep(time.Duration(rn) * time.Second) } //一次性交割卖方仓单查询 for i := n; i >= 0; i-- { date := time.Now().AddDate(0, -i, 0) fmt.Println("date:", date.Format(utils.FormatDate)) CrawlerFromGuangzhouCongregateDeliveryQuotes(date) rn := utils.RangeRand(3, 15) fmt.Println("i:"+strconv.Itoa(i)+" date:", date.Format(utils.FormatDate)+";sleep:", strconv.Itoa(int(rn))) time.Sleep(time.Duration(rn) * time.Second) } //更新指标日期 CrawlerFromGuangzhouIndexMinAndMaxDate() } return } func RefreshEic(cont context.Context) (err error) { SyncStorageFromEicV2() return } func RefreshCoal(cont context.Context) (err error) { FileCoalJsm() FileCoalFirm() FileCoalCoastal() FileCoalInland() return } func RefreshChangesVisitorsCovid(cont context.Context) (err error) { err = AddSourceChangesVisitorsCovid() return } //func Task2() { // fmt.Println("start") // // var cont context.Context // _ = national_data.RefreshNationalYearDbA(cont) // // //_ = national_data.RefreshNationalYearDbB(cont) // // //_ = national_data.ApiTest() // fmt.Println("end") //} func Task123() { fmt.Println("start") ctx := new(context.Context) CrawlerIcpi(*ctx) fmt.Println("end") }