美农出库销售数据

services/usda_psd.go

@@ -1,10 +1,16 @@
 package services
 
 import (
+	"bytes"
 	"encoding/json"
 	"eta/eta_crawler/services/alarm_msg"
 	"eta/eta_crawler/utils"
 	"fmt"
+	"github.com/PuerkitoBio/goquery"
+	"io"
+	"mime/multipart"
+	"net/http"
+	"os"
 	"strings"
 )
 
@@ -66,6 +72,7 @@ type UsdaPsdDataAttribute struct {
 // Oilseed, Rapeseed:2226000
 // Oilseed, Soybean:2222000
 // Oilseed, Sunflowerseed:2224000
+// 美国农业部月度供需平衡表数据
 func DownloadUsdaPsdData() {
 	var err error
 	defer func() {
@@ -135,11 +142,6 @@ func DownloadUsdaPsdData() {
 		err = e
 		return
 	}
-	fmt.Println("body :" + string(body))
-	utils.FileLog.Info("body:" + string(body))
-	if strings.Contains(string(body), "很抱歉！ 因系统检测到您的请求可能对网站造成威胁") {
-		return
-	}
 	item := new(UsdaPsdData)
 	err = json.Unmarshal(body, &item)
 	if err != nil {
@@ -158,3 +160,145 @@ func DownloadUsdaPsdData() {
 	}
 	return
 }
+
+// 美国农业出库销售数据
+func DownloadUsdaFmsData() {
+	var err error
+	defer func() {
+		if err != nil {
+			msg := "失败提醒" + "DownloadUsdaFmsData ErrMsg:" + err.Error()
+			fmt.Println("msg:", msg)
+			utils.FileLog.Info(msg)
+			go alarm_msg.SendAlarmMsg(msg, 3)
+		}
+	}()
+	downloadFile := "downloaded_excel.xlsx"
+	//请求首页，获取入参
+	dataUrl := "https://apps.fas.usda.gov/esrquery/esrq.aspx"
+	body1, err := utils.HttpGetNoCookie(dataUrl)
+	if err != nil {
+		return
+	}
+	htmlString := string(body1)
+	// 解析返回值，截取htmlinput标签，input标签里，id=“__EVENTVALIDATION”的input标签里的值
+	// 使用goquery读取HTML字符串
+	doc, err := goquery.NewDocumentFromReader(strings.NewReader(htmlString))
+	if err != nil {
+		return
+	}
+	stateValue := doc.Find("input#__VIEWSTATE").AttrOr("value", "")
+	stateEneratorValue := doc.Find("input#__VIEWSTATEGENERATOR").AttrOr("value", "")
+	// 查询并获取input标签的值
+	validValue := doc.Find("input#__EVENTVALIDATION").AttrOr("value", "")
+
+	var body bytes.Buffer
+	multipartWriter := multipart.NewWriter(&body)
+
+	// 添加表单字段（如果需要的话）
+	if err = multipartWriter.WriteField("__EVENTTARGET", ""); err != nil {
+		err = fmt.Errorf("set __EVENTTARGET, Err:%s", err)
+		return
+	}
+	if err = multipartWriter.WriteField("__EVENTARGUMENT", ""); err != nil {
+		err = fmt.Errorf("set __EVENTARGUMENT, Err:%s", err)
+		return
+	}
+	if err = multipartWriter.WriteField("__LASTFOCUS", ""); err != nil {
+		err = fmt.Errorf("set __LASTFOCUS, Err:%s", err)
+		return
+	}
+	if err = multipartWriter.WriteField("__VIEWSTATE", stateValue); err != nil {
+		err = fmt.Errorf("set __VIEWSTATE, Err:%s", err)
+		return
+	}
+	if err = multipartWriter.WriteField("__VIEWSTATEGENERATOR", stateEneratorValue); err != nil {
+		err = fmt.Errorf("set __VIEWSTATEGENERATOR, Err:%s", err)
+		return
+	}
+	if err = multipartWriter.WriteField("__EVENTVALIDATION", validValue); err != nil {
+		err = fmt.Errorf("set __EVENTVALIDATION, Err:%s", err)
+		return
+	}
+	//整理需要下载的品种ID
+	//Soybeans：801，Soybean cake & meal：901，Soybean Oil：902
+	CommodityIds := []string{"801", "901", "902"}
+	for _, v := range CommodityIds {
+		if err = multipartWriter.WriteField("ctl00$MainContent$lbCommodity", v); err != nil {
+			err = fmt.Errorf("set ctl00$MainContent$lbCommodity, Err:%s", err)
+			return
+		}
+	}
+	startDate := "08/22/2019"
+	endDate := "08/22/2024"
+	if err = multipartWriter.WriteField("ctl00$MainContent$lbCountry", "0:0"); err != nil {
+		err = fmt.Errorf("set ctl00$MainContent$lbCountry, Err:%s", err)
+		return
+	}
+	if err = multipartWriter.WriteField("ctl00$MainContent$ddlReportFormat", "10"); err != nil {
+
+		return
+	}
+	if err = multipartWriter.WriteField("ctl00$MainContent$rblOutputType", "2"); err != nil {
+		return
+	}
+	if err = multipartWriter.WriteField("ctl00$MainContent$tbStartDate", startDate); err != nil {
+		return
+	}
+	if err = multipartWriter.WriteField("ctl00$MainContent$tbEndDate", endDate); err != nil {
+		return
+	}
+	if err = multipartWriter.WriteField("ctl00$MainContent$rblColumnSelection", "regular"); err != nil {
+		return
+	}
+	if err = multipartWriter.WriteField("ctl00$MainContent$btnSubmit", "Submit"); err != nil {
+		return
+	}
+	// 注意：如果接口需要文件上传，这里应该使用multipartWriter.CreateFormFile来添加文件
+
+	// 关闭multipart writer以添加最后的边界
+	if err = multipartWriter.Close(); err != nil {
+		err = fmt.Errorf("close multipart writer, Err:%s", err)
+		return
+	}
+
+	// 构造请求
+	req, err := http.NewRequest("POST", dataUrl, &body)
+	if err != nil {
+		err = fmt.Errorf("create request, Err:%s", err)
+		return
+	}
+
+	// 设置请求头
+	req.Header.Set("Content-Type", multipartWriter.FormDataContentType())
+
+	// 发送请求
+	client := &http.Client{}
+	resp, err := client.Do(req)
+	if err != nil {
+		err = fmt.Errorf("send request, Err:%s", err)
+		return
+	}
+	defer resp.Body.Close()
+
+	// 检查响应状态码
+	if resp.StatusCode != http.StatusOK {
+		err = fmt.Errorf("unexpected status code: %d", resp.StatusCode)
+		return
+	}
+
+	// 读取响应体
+	out, err := os.Create(downloadFile)
+	if err != nil {
+		return
+	}
+	defer out.Close()
+
+	// 将响应体写入到文件
+	_, err = io.Copy(out, resp.Body)
+	if err != nil {
+		return
+	}
+
+	fmt.Println("Excel file downloaded successfully")
+	return
+}