处理美国农业部数据

services/usda_psd.go

@@ -108,20 +108,6 @@ func DownloadUsdaFmsDataTask(cont context.Context) (err error) {
 // Oilseed, Sunflowerseed:2224000
 // 美国农业部月度供需平衡表数据
 func DownloadUsdaPsdData() (err error) {
-	// 从test.json文件中读取json串
-	/*body, err := ioutil.ReadFile("test.json")
-	if err != nil {
-		return
-	}
-	// 解析json串
-	item := new(UsdaPsdData)
-	err = json.Unmarshal(body, &item)
-	if err != nil {
-		fmt.Println("json.Unmarshal err:" + err.Error())
-		return
-	}
-	indexList, err = handleUsdaFasPsd(item)
-	return*/
 	defer func() {
 		if err != nil {
 			msg := "失败提醒" + "downloadUsdaPsdData ErrMsg:" + err.Error()
@@ -165,7 +151,7 @@ func DownloadUsdaPsdData() (err error) {
 	var countries []string
 	countries = append(countries, "R00", "ALL")
 	var marketYears []int
-	marketYears = append(marketYears, 2024, 2023, 2022, 2021, 2020, 2019, 2018, 2017, 2016, 2015, 2014)
+	marketYears = append(marketYears, 2025, 2024, 2023, 2022, 2021, 2020, 2019, 2018, 2017, 2016, 2015, 2014)
 	// {"queryId":0,"commodityGroupCode":null,"commodities":["0430000"],"attributes":[4,20,28,57,81,84,86,88,113,130,192,125,176,178,184],"countries":["R00","ALL"],"marketYears":[2024,2023,2022,2021,2020,2019,2018,2017,2016,2015,2014],"chkCommoditySummary":false,"chkAttribSummary":false,"chkCountrySummary":false,"commoditySummaryText":"","attribSummaryText":"","countrySummaryText":"","optionColumn":"year","chkTopCountry":false,"topCountryCount":"","chkfileFormat":false,"chkPrevMonth":true,"chkMonthChange":false,"chkCodes":false,"chkYearChange":false,"queryName":"","sortOrder":"Commodity/Attribute/Country","topCountryState":false}
 	var req UsdaPsdDataQueryParams
 	req.Commodities = commodities
@@ -195,9 +181,24 @@ func DownloadUsdaPsdData() (err error) {
 		fmt.Println("json.Unmarshal err:" + err.Error())
 		return
 	}
+
+	// 使用通道等待解析完成
+	done := make(chan error)
 	go func() {
-		err = handleUsdaFasPsd(item)
+		done <- handleUsdaFasPsd(item)
 	}()
+	// 等待解析完成或超时
+	select {
+	case err = <-done:
+		if err != nil {
+			err = fmt.Errorf("handleUsdaFasPsd, Err:%w", err)
+			return
+		}
+	case <-time.After(20 * time.Minute): // 假设20分钟超时
+		err = fmt.Errorf("parse excel timed out")
+		return
+	}
+
 	return
 }
 
@@ -212,7 +213,7 @@ func DownloadUsdaFmsData(startDate, endDate string) (err error) {
 			go alarm_msg.SendAlarmMsg(msg, 3)
 		}
 	}()
-	downloadFile := fmt.Sprintf("./static/usda_fms_excel_%s.xls", time.Now().Format(utils.FormatDate))
+	downloadFile := fmt.Sprintf("./static/usda_fms_excel_%s.xls", time.Now().Format(utils.FormatDateTime))
 	//请求首页，获取入参
 	dataUrl := "https://apps.fas.usda.gov/esrquery/esrq.aspx"
 	body1, err := utils.HttpGetNoCookie(dataUrl)
@@ -460,8 +461,10 @@ func ParseUsdaFmsExcel(path string) (err error) {
 					}
 					date := timeT.Format(utils.FormatDate)
 					dataVal = text
+					firstCommodity := utils.GetFirstLetter(commodity)
+					firstKind := utils.GetFirstLetter(kind)
 					indexName = fmt.Sprintf("%s: %s: %s", commodity, country, kind)
-					inCode := "usda" + utils.GetFirstLetter(indexName)
+					inCode := fmt.Sprintf("usda%s%s%s", firstCommodity, strings.ToLower(strings.ReplaceAll(country, " ", "")), firstKind)
 					indexItem, okIndex := indexMap[indexName]
 					// 首字母大写
 					classifyName := commodity
@@ -602,8 +605,11 @@ func handleUsdaFasPsd(item *UsdaPsdData) (err error) {
 					errMsg += fmt.Sprintf("指标名称为空 commodityRow:%s,countriesRow:%s,attributesRow:%s\n", commodityRow, countriesRow, attributesRow)
 					continue
 				}
+				firstCommodityRow := utils.GetFirstLetter(commodityRow)
+				firstAttributesRow := utils.GetFirstLetter(attributesRow)
+				firstLastStr := utils.GetFirstLetter(lastStr)
 
-				inCode = "usda" + utils.GetFirstLetter(indexName)
+				inCode = fmt.Sprintf("usda%s%s%s%s", firstCommodityRow, strings.ToLower(strings.ReplaceAll(countriesRow, " ", "")), firstAttributesRow, firstLastStr)
 				indexItem, okIndex := indexMap[indexName]
 				// 首字母大写
 				classifyName = commodityRow
@@ -622,6 +628,7 @@ func handleUsdaFasPsd(item *UsdaPsdData) (err error) {
 					sort++
 				}
 				val := col.(float64)
+				val = utils.FloatFormatRound(val, 2)
 				indexItem.ExcelDataMap[date] = fmt.Sprintf("%.4f", val)
 				indexMap[indexName] = indexItem
 				continue

utils/common.go

@@ -11,6 +11,7 @@ import (
 	"errors"
 	"fmt"
 	"github.com/mozillazg/go-pinyin"
+	"github.com/shopspring/decimal"
 	"image"
 	"image/png"
 	"io"
@@ -1248,3 +1249,8 @@ func httpPostEtaLib(url, postData string, params ...string) ([]byte, error) {
 	}
 	return b, err
 }
+
+func FloatFormatRound(val float64, num int32) float64 {
+	val, _ = decimal.NewFromFloat(val).Round(num).Float64()
+	return val
+}