月度供需根据类别拆分成多个请求

services/usda_psd.go

@@ -74,7 +74,16 @@ func DownloadUsdaPsdDataTask(cont context.Context) (err error) {
 	//todo 设置下载频率
 	// 获取最近两年的年份
 	years := []int{time.Now().Year() + 1, time.Now().Year()}
-	err = DownloadUsdaPsdData(years)
+	var commodities []string
+	commodities = append(commodities, "0813800", "0813200", "0813600", "0813100", "0813500", "4242000", "4233000", "4235000", "4243000", "4244000", "4234000", "4239100", "4232000", "4236000", "2223000", "2232000", "2221000", "2226000", "2222000", "2224000")
+	for _, commodity := range commodities {
+		err = DownloadUsdaPsdData(commodity, years)
+		if err != nil {
+			utils.FileLog.Info("DownloadUsdaPsdData " + commodity + "ErrMsg:" + err.Error())
+		}
+		//time.Sleep(time.Duration(utils.RangeRand(2, 10)*10) * time.Second)
+		utils.FileLog.Info("DownloadUsdaPsdData " + commodity + " 爬取成功")
+	}
 	return
 }
 
@@ -109,7 +118,7 @@ func DownloadUsdaFmsDataTask(cont context.Context) (err error) {
 // Oilseed, Soybean:2222000
 // Oilseed, Sunflowerseed:2224000
 // 美国农业部月度供需平衡表数据
-func DownloadUsdaPsdData(years []int) (err error) {
+func DownloadUsdaPsdData(commodityCode string, years []int) (err error) {
 	defer func() {
 		if err != nil {
 			msg := "失败提醒" + "downloadUsdaPsdData ErrMsg:" + err.Error()
@@ -123,10 +132,11 @@ func DownloadUsdaPsdData(years []int) (err error) {
 	dataUrl := "https://apps.fas.usda.gov/PSDOnlineApi/api/query/RunQuery"
 
 	var commodities []string
-	commodities = append(commodities, "0813800", "0813200", "0813600", "0813100", "0813500", "4242000", "4233000", "4235000", "4243000", "4244000", "4234000", "4239100", "4232000", "4236000", "2223000", "2232000", "2221000", "2226000", "2222000", "2224000")
-	//commodities = append(commodities, "0430000")
+	//commodities = append(commodities, "0813800", "0813200", "0813600", "0813100", "0813500", "4242000", "4233000", "4235000", "4243000", "4244000", "4234000", "4239100", "4232000", "4236000", "2223000", "2232000", "2221000", "2226000", "2222000", "2224000")
+	commodities = append(commodities, commodityCode)
 	commodityCodes := strings.Join(commodities, ",")
 	attributeUrl = attributeUrl + "commodityCodes=" + commodityCodes
+	fmt.Println("attributeUrl", attributeUrl)
 	// 定义请求参数
 	// 获取属性入参
 	attributeBody, e := utils.HttpGetNoCookie(attributeUrl)
@@ -162,12 +172,14 @@ func DownloadUsdaPsdData(years []int) (err error) {
 	req.Countries = countries
 	req.MarketYears = marketYears
 	req.OptionColumn = "year"
-	//req.ChkPrevMonth = true
+	req.ChkPrevMonth = true
+	req.ChkYearChange = true
 	req.SortOrder = "Commodity/Country/Attribute"
 
 	// 构造httppost请求
 	reqBody, _ := json.Marshal(req)
 	// 解析返回值
+	fmt.Println("reqBody", string(reqBody))
 
 	headerParams := make(map[string]string)
 	//headerParams["Cookie"] = "CT6T=312900; SF_cookie_3=68941398"
@@ -525,6 +537,7 @@ func ParseUsdaFmsExcel(path string) (err error) {
 	return
 }
 
+// 月度供需
 func handleUsdaFasPsd(item *UsdaPsdData) (err error) {
 	//设置缓存key，防止重复处理
 	errMsg := ""
@@ -570,7 +583,7 @@ func handleUsdaFasPsd(item *UsdaPsdData) (err error) {
 		for _, k := range headerSlice {
 			col, ok := row[k]
 			if !ok || col == nil {
-				utils.FileLog.Info("col is nil")
+				//utils.FileLog.Info("col is nil")
 				continue
 			}
 			if k == "commodity" {
@@ -584,7 +597,6 @@ func handleUsdaFasPsd(item *UsdaPsdData) (err error) {
 			} else {
 				//数据列
 				year, _ := strconv.Atoi(strings.Split(k, "/")[0])
-				month := 0
 				indexName := ""
 				classifyName := ""
 				classifySort := 0
@@ -592,7 +604,7 @@ func handleUsdaFasPsd(item *UsdaPsdData) (err error) {
 				fre := "年度"
 				lastStr := "Yearly"
 				// year年度的最后一天日期
-				dateT := time.Date(year, time.December, 31, 0, 0, 0, 0, time.UTC)
+				dateT := time.Date(year, time.December, 31, 0, 0, 0, 0, time.Local)
 				if strings.Contains(k, "(") {
 					fre = "月度"
 					lastStr = "Monthly"
@@ -605,9 +617,9 @@ func handleUsdaFasPsd(item *UsdaPsdData) (err error) {
 						errMsg += fmt.Sprintf("月份转换错误:%s%s\n", monthStr, e.Error())
 						continue
 					}
-					month = int(monthT.Month())
 					// 将year和month拼接成日期，该月的最后一天日期
-					dateT = time.Date(year, time.Month(month), 31, 0, 0, 0, 0, time.UTC)
+					// 获取下一个月份的第一天，减去1天为当前月份的最后一天
+					dateT = time.Date(year, monthT.Month(), 1, 0, 0, 0, 0, time.Now().Location()).AddDate(0, 1, -1)
 				}
 				date := dateT.Format("2006-01-02")
 				// 封装成指标数据
@@ -666,7 +678,7 @@ func handleUsdaFasPsd(item *UsdaPsdData) (err error) {
 	if err != nil {
 		return
 	}
-	fmt.Println("月度供需 执行成功")
+	fmt.Println("月度供需 " + commodityRow + "执行成功")
 	return
 }