feat：un数据定期爬取

services/com_trade.go

@@ -26,40 +26,88 @@ func SyncComTradeData(cont context.Context) (err error) {
 const ComTradeUrl = "https://comtradeapi.un.org/public/v1/preview/C/A/HS?"
 const ComTradeMonthUrl = "https://comtradeapi.un.org/public/v1/preview/C/M/HS?"
 
-func sync() {
-	yearList := []int{2009, 2008, 2007, 2006, 2005, 2004, 2003, 2002, 2001, 2000}
+//func sync() {
+//	//yearList := []int{2009, 2008, 2007, 2006, 2005, 2004, 2003, 2002, 2001, 2000}
+//	yearList := []int{2022}
+//	flowCodeList := []string{"X", "M"}
+//
+//	for _, year := range yearList {
+//		for _, flow := range flowCodeList {
+//			fmt.Println("开始", year, "的", flow)
+//			syncAllComTradeData2(year, 0, flow, "year")
+//			fmt.Println("结束", year, "的", flow)
+//		}
+//	}
+//}
+//
+//func sync2() {
+//	defer func() {
+//		//go utils.SendEmail(utils.APPNAME+"【"+utils.RunMode+"】"+"失败提醒", "RefreshDataFromCoalCoastal ErrMsg:"+err.Error(), utils.EmailSendToUsers)
+//		msg := "UN数据同步完成"
+//		go alarm_msg.SendAlarmMsg(msg, 3)
+//	}()
+//	//yearList := []int{2009, 2008, 2007, 2006, 2005, 2004, 2003, 2002, 2001, 2000}
+//	//yearList := []int{2022, 2021, 2020, 2019, 2018, 2017, 2016, 2015, 2014, 2013, 2012, 2011, 2010}
+//	yearList := []int{2023}
+//	//monthList := []int{12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1}
+//	monthList := []int{1, 2, 3, 4, 5, 6}
+//	flowCodeList := []string{"X", "M"}
+//
+//	for _, year := range yearList {
+//		for _, month := range monthList {
+//			for _, flow := range flowCodeList {
+//				fmt.Println("开始", year, month, "的", flow)
+//				syncAllComTradeData2(year, month, flow, "month")
+//				fmt.Println("结束", year, "的", flow)
+//			}
+//		}
+//	}
+//}
+
+// SyncYearComTrade 同步年度数据
+func SyncYearComTrade(cont context.Context) (err error) {
+	errMsg := make([]string, 0)
+	defer func() {
+		if len(errMsg) > 0 {
+			go alarm_msg.SendAlarmMsg("UN年度度数据同步失败："+strings.Join(errMsg, "\n"), 3)
+		}
+	}()
 	flowCodeList := []string{"X", "M"}
-
-	for _, year := range yearList {
-		for _, flow := range flowCodeList {
-			fmt.Println("开始", year, "的", flow)
-			syncAllComTradeData2(year, 0, flow, "year")
-			fmt.Println("结束", year, "的", flow)
+	year := time.Now().Year()
+	for _, flow := range flowCodeList {
+		fmt.Println("开始", year, "的", flow)
+		err = syncAllComTradeData2(year, 0, flow, "year")
+		if err != nil {
+			errMsg = append(errMsg, fmt.Sprint(year, "的", flow, "数据爬取失败：", err.Error()))
 		}
+		fmt.Println("结束", year, "的", flow)
 	}
+
+	return
 }
 
-func sync2() {
+// SyncYearMonthComTrade 同步月度数据
+func SyncYearMonthComTrade(cont context.Context) (err error) {
+	errMsg := make([]string, 0)
 	defer func() {
-		//go utils.SendEmail(utils.APPNAME+"【"+utils.RunMode+"】"+"失败提醒", "RefreshDataFromCoalCoastal ErrMsg:"+err.Error(), utils.EmailSendToUsers)
-		msg := "UN数据同步完成"
-		go alarm_msg.SendAlarmMsg(msg, 3)
+		if len(errMsg) > 0 {
+			go alarm_msg.SendAlarmMsg("UN月度数据同步失败："+strings.Join(errMsg, "\n"), 3)
+		}
 	}()
-	//yearList := []int{2009, 2008, 2007, 2006, 2005, 2004, 2003, 2002, 2001, 2000}
-	//yearList := []int{2022, 2021, 2020, 2019, 2018, 2017, 2016, 2015, 2014, 2013, 2012, 2011, 2010}
-	yearList := []int{2022}
-	monthList := []int{12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1}
 	flowCodeList := []string{"X", "M"}
 
-	for _, year := range yearList {
-		for _, month := range monthList {
-			for _, flow := range flowCodeList {
-				fmt.Println("开始", year, month, "的", flow)
-				syncAllComTradeData2(year, month, flow, "month")
-				fmt.Println("结束", year, "的", flow)
-			}
+	year := time.Now().Year()
+	month := int(time.Now().Month())
+	for _, flow := range flowCodeList {
+		fmt.Println("开始", year, month, "的", flow)
+		err := syncAllComTradeData2(year, month, flow, "month")
+		if err != nil {
+			errMsg = append(errMsg, fmt.Sprint(year, month, "的", flow, "数据爬取失败：", err.Error()))
 		}
+		fmt.Println("结束", year, "的", flow)
 	}
+
+	return
 }
 
 type TmpStruct struct {
@@ -558,7 +606,7 @@ func syncComTradeData(comTradeId int, indexCode, indexNameCn, flowCode, partnerC
 				if partnerCnName == "世界" {
 					//若对手国和第二对手国是world时，则翻译为总量；
 					finalpartnerName = "总量"
-				}else {
+				} else {
 					//若对手国或第二对手国是相同的，只需翻译一个；
 					finalpartnerName = partnerCnName
 				}
@@ -602,18 +650,18 @@ func syncComTradeData(comTradeId int, indexCode, indexNameCn, flowCode, partnerC
 					codeMappingListMap[tmpData.IndexTradeCode] = valueName
 				}
 			} else {
-				if name != valueName{
+				if name != valueName {
 					tmpComTradeCodeMapping := &models.ComTradeCodeMapping{
-						Code:       tmpData.IndexTradeCode,
-						Name:       valueName,
+						Code: tmpData.IndexTradeCode,
+						Name: valueName,
 					}
 					e := models.UpdateComTradeCodeMapping(tmpComTradeCodeMapping, []string{"Name"})
 					if e != nil {
 						err = e
 						return
 					}
-					utils.FileLog.Info("更新指标名称","旧："+name, "新：" + valueName,"code:" +tmpData.IndexTradeCode)
-					fmt.Println("更新指标名称","旧："+name, "新：" + valueName)
+					utils.FileLog.Info("更新指标名称", "旧："+name, "新："+valueName, "code:"+tmpData.IndexTradeCode)
+					fmt.Println("更新指标名称", "旧："+name, "新："+valueName)
 				}
 			}
 
@@ -636,18 +684,18 @@ func syncComTradeData(comTradeId int, indexCode, indexNameCn, flowCode, partnerC
 					codeMappingListMap[tmpData.IndexNetWeightCode] = weightName
 				}
 			} else {
-				if name != weightName{
+				if name != weightName {
 					tmpComTradeCodeMapping := &models.ComTradeCodeMapping{
-						Code:       tmpData.IndexTradeCode,
-						Name:       valueName,
+						Code: tmpData.IndexTradeCode,
+						Name: valueName,
 					}
 					e := models.UpdateComTradeCodeMapping(tmpComTradeCodeMapping, []string{"Name"})
 					if e != nil {
 						err = e
 						return
 					}
-					utils.FileLog.Info("更新指标名称","旧："+name, "新：" + valueName ,"code:" +tmpData.IndexNetWeightCode)
-					fmt.Println("更新指标名称","旧："+name, "新：" + valueName)
+					utils.FileLog.Info("更新指标名称", "旧："+name, "新："+valueName, "code:"+tmpData.IndexNetWeightCode)
+					fmt.Println("更新指标名称", "旧："+name, "新："+valueName)
 				}
 			}
 		}
@@ -921,7 +969,7 @@ func syncAllComTradeMapping2() (err error) {
 	return
 }
 
-func syncAllComTradeMapping3()  {
+func syncAllComTradeMapping3() {
 	codeMappingListMap := make(map[string]*models.ComTradeCodeMapping)
 	codeMappingList, err := models.GetAllComTradeCodeMappingList()
 	if err != nil {
@@ -991,7 +1039,7 @@ func syncAllComTradeMapping3()  {
 			if partnerCnName == "世界" {
 				//若对手国和第二对手国是world时，则翻译为总量；
 				finalpartnerName = "总量"
-			}else {
+			} else {
 				//若对手国或第二对手国是相同的，只需翻译一个；
 				finalpartnerName = partnerCnName
 			}
@@ -1057,7 +1105,6 @@ func syncAllComTradeMapping3()  {
 			//}
 		}
 
-
 		if len(dataList) >= 2000 {
 			tmpErr := models.AddMultiComTradeCodeMapping(dataList)
 			if tmpErr != nil {
@@ -1076,4 +1123,4 @@ func syncAllComTradeMapping3()  {
 	}
 	fmt.Println("整体结束了=============")
 	return
-}
+}

services/task.go

@@ -7,12 +7,10 @@ import (
 )
 
 func Task() {
-	sync()
-	//sync2()
 	//sync3()
 	//syncAllComTradeMapping()
 	//syncAllComTradeMapping2()
-	syncAllComTradeMapping3()
+	//syncAllComTradeMapping3()
 	//return
 	fmt.Println("start crawler")
 	refreshData := task.NewTask("refreshData", "0 0,30 16-20 * * *", RefreshData)
@@ -20,12 +18,16 @@ func Task() {
 	refreshCoal := task.NewTask("refreshData", "0 0,30 17-23 * * *", RefreshCoal)
 	refreshVisitors := task.NewTask("RefreshChangesVisitorsCovid", "0 30 2-22/10 * * *", RefreshChangesVisitorsCovid)
 	syncEiaSteoData := task.NewTask("SyncEiaSteoData", "0 0 22 * * *", SyncEiaSteoData)
+	syncYearComTrade := task.NewTask("SyncEiaSteoData", "0 0 3 1 1 *", SyncYearComTrade)           // 每年一月一号同步
+	syncYearMonthComTrade := task.NewTask("SyncEiaSteoData", "0 0 3 1 * *", SyncYearMonthComTrade) // 每月1号同步
 
 	task.AddTask("数据爬取", refreshData)
 	task.AddTask("欧洲天然气爬取", refreshEic)
 	task.AddTask("中国煤炭网爬取", refreshCoal)
 	task.AddTask("谷歌出行指数爬取", refreshVisitors)
-	task.AddTask("eia steo报告", syncEiaSteoData) //每天22点爬一次
+	task.AddTask("eia steo报告", syncEiaSteoData)     //每天22点爬一次
+	task.AddTask("UN年度数据", syncYearComTrade)      //每年一月一号的3点同步
+	task.AddTask("UN月度数据", syncYearMonthComTrade) //每月1号的3点同步
 	task.StartTask()
 	//FileCoalJsm()
 	//FileCoalFirm()