国家统计局数据获取

services/national_data/national_data.go

@@ -0,0 +1,86 @@
+package national_data
+
+import (
+	"crypto/tls"
+	"fmt"
+	"io/ioutil"
+	"net/http"
+	"strings"
+)
+
+func NationalHttpPost(reqUrl, payload string) (result []byte, err error) {
+	tr := &http.Transport{
+		TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
+	}
+	client := &http.Client{
+		Transport: tr,
+	}
+	req, err := http.NewRequest("POST", reqUrl, strings.NewReader(payload))
+	if err != nil {
+		return
+	}
+	req.Header.Add("Accept", "text/plain, */*; q=0.01")
+	req.Header.Add("Accept-Encoding", "tgzip, deflate, br")
+	req.Header.Add("Accept-Language", "zh-CN,zh;q=0.9")
+	req.Header.Add("Connection", "keep-alive")
+	req.Header.Add("Content-Length", "37")
+	req.Header.Add("Content-Type", "application/x-www-form-urlencoded")
+	req.Header.Add("Cookie", "wzws_sessionid=gDExNS4xOTQuMTAyLjEyN6BkERzUgmZjNWVlMYFiOWNiZDg=; JSESSIONID=UOri2Cu3f3c-Y3rPgXWJ04E8pfbeyAUGG-s7zJ7Tt0JhlEiLi0EU!412929168; u=5")
+	req.Header.Add("Host", "data.stats.gov.cn")
+	req.Header.Add("Origin", "https://data.stats.gov.cn")
+	req.Header.Set("Referer", "https://data.stats.gov.cn/easyquery.htm?cn=A01")
+	req.Header.Set("sec-ch-ua", "\"Not_A Brand\";v=\"99\", \"Google Chrome\";v=\"109\", \"Chromium\";v=\"109\"")
+	req.Header.Set("sec-ch-ua-mobile", "?0")
+	req.Header.Set("sec-ch-ua-platform", "\"Windows\"")
+	req.Header.Set("Sec-Fetch-Dest", "empty")
+	req.Header.Set("Sec-Fetch-Mode", "cors")
+	req.Header.Set("Sec-Fetch-Site", "same-origin")
+	req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/109.0.0.0 Safari/537.36")
+	req.Header.Set("X-Requested-With", "XMLHttpRequest")
+	res, err := client.Do(req)
+	if err != nil {
+		return
+	}
+	defer res.Body.Close()
+	body, err := ioutil.ReadAll(res.Body)
+	if err != nil {
+		return
+	}
+	result = body
+	return
+}
+
+func NationalGet(reqUrl, payload string) (err error) {
+	tr := &http.Transport{
+		TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
+	}
+	client := &http.Client{
+		Transport: tr,
+	}
+	req, err := http.NewRequest("GET", reqUrl, strings.NewReader(payload))
+	if err != nil {
+		return
+	}
+	res, err := client.Do(req)
+	if err != nil {
+		return
+	}
+	defer res.Body.Close()
+	_, err = ioutil.ReadAll(res.Body)
+	if err != nil {
+		return
+	}
+
+	Cookie := res.Header.Get("Cookie")
+	fmt.Println(Cookie)
+	rcookie := req.Header.Get("Cookie")
+	fmt.Println("rcookie")
+	fmt.Println(rcookie)
+	//fmt.Println("body:" + string(body))
+	cookiesArr := res.Cookies()
+	fmt.Println("cookiesArrLen:", len(cookiesArr))
+	for k, v := range cookiesArr {
+		fmt.Println(k, v)
+	}
+	return
+}

services/task.go

@@ -6,7 +6,7 @@ import (
 	"github.com/beego/beego/v2/task"
 )
 
-func Task() {
+func Task123() {
 	fmt.Println("start crawler")
 	refreshData := task.NewTask("refreshData", "0 0,30 16-20 * * *", RefreshData)
 	refreshEic := task.NewTask("refreshData", "0 0 2,6 * * *", RefreshEic)
@@ -57,3 +57,20 @@ func RefreshChangesVisitorsCovid(cont context.Context) (err error) {
 	err = AddSourceChangesVisitorsCovid()
 	return
 }
+
+//func Task() {
+//	fmt.Println("start")
+//	reqUrl := "https://data.stats.gov.cn/easyquery.htm"
+//
+//	formData := url.Values{}
+//	formData.Add("id", "zb")
+//	formData.Add("dbcode", "hgyd")
+//	formData.Add("wdcode", "zb")
+//	formData.Add("m", "getTree")
+//
+//	formDataStr := formData.Encode()
+//	result, err := national_data.NationalHttpPost(reqUrl, formDataStr)
+//	fmt.Println(err)
+//	fmt.Println(string(result))
+//	fmt.Println("end")
+//}