eta_crawler/services/national_data/common.go

package national_data

import (
	"bytes"
	"crypto/tls"
	"encoding/json"
	"eta/eta_crawler/utils"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"net/url"
	"strings"
	"time"
)

const (
	NationalStatisticsBaseReqUrl = "https://data.stats.gov.cn/easyquery.htm"
)

func NationalHttpPost(reqUrl, payload string) (result []byte, err error) {
	// 随机延迟执行
	r := utils.RangeRand(5000, 8000)
	//fmt.Printf("随机延迟%d\n", r)
	if r > 6100 && r < 6250 {
		time.Sleep(15 * time.Second)
	} else {
		time.Sleep(time.Duration(r) * time.Millisecond)
	}

	tr := &http.Transport{
		TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
	}
	client := &http.Client{
		Transport: tr,
	}
	req, err := http.NewRequest("POST", reqUrl, strings.NewReader(payload))
	if err != nil {
		return
	}
	req.Header.Add("Accept", "text/plain, */*; q=0.01")
	req.Header.Add("Accept-Encoding", "tgzip, deflate, br")
	req.Header.Add("Accept-Language", "zh-CN,zh;q=0.9")
	req.Header.Add("Connection", "keep-alive")
	req.Header.Add("Content-Length", "37")
	req.Header.Add("Content-Type", "application/x-www-form-urlencoded")
	req.Header.Add("Cookie", "wzws_sessionid=gDExNS4xOTQuMTAyLjEyN6BkERzUgmZjNWVlMYFiOWNiZDg=; JSESSIONID=UOri2Cu3f3c-Y3rPgXWJ04E8pfbeyAUGG-s7zJ7Tt0JhlEiLi0EU!412929168; u=5")
	req.Header.Add("Host", "data.stats.gov.cn")
	req.Header.Add("Origin", "https://data.stats.gov.cn")
	req.Header.Set("Referer", "https://data.stats.gov.cn/easyquery.htm?cn=A01")
	req.Header.Set("sec-ch-ua", "\"Not_A Brand\";v=\"99\", \"Google Chrome\";v=\"109\", \"Chromium\";v=\"109\"")
	req.Header.Set("sec-ch-ua-mobile", "?0")
	req.Header.Set("sec-ch-ua-platform", "\"Windows\"")
	req.Header.Set("Sec-Fetch-Dest", "empty")
	req.Header.Set("Sec-Fetch-Mode", "cors")
	req.Header.Set("Sec-Fetch-Site", "same-origin")
	req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/109.0.0.0 Safari/537.36")
	req.Header.Set("X-Requested-With", "XMLHttpRequest")
	res, err := client.Do(req)
	if err != nil {
		return
	}
	defer func() {
		_ = res.Body.Close()
	}()
	// 此处用io.Copy替代ioutil.ReadAll方法避免数据过大读取不完整
	var b []byte
	buf := bytes.NewBuffer(b)
	_, err = io.Copy(buf, res.Body)
	if err != nil {
		return
	}
	result = buf.Bytes()
	return
}

func NationalGet(reqUrl, payload string) (err error) {
	tr := &http.Transport{
		TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
	}
	client := &http.Client{
		Transport: tr,
	}
	req, err := http.NewRequest("GET", reqUrl, strings.NewReader(payload))
	if err != nil {
		return
	}
	res, err := client.Do(req)
	if err != nil {
		return
	}
	defer res.Body.Close()
	_, err = ioutil.ReadAll(res.Body)
	if err != nil {
		return
	}

	Cookie := res.Header.Get("Cookie")
	fmt.Println(Cookie)
	rcookie := req.Header.Get("Cookie")
	fmt.Println("rcookie")
	fmt.Println(rcookie)
	//fmt.Println("body:" + string(body))
	cookiesArr := res.Cookies()
	fmt.Println("cookiesArrLen:", len(cookiesArr))
	for k, v := range cookiesArr {
		fmt.Println(k, v)
	}
	return
}

// DataApiReq 数据接口请求体
type DataApiReq struct {
	Method    string `description:"方法: QueryData-查询数据; getOtherWds-获取其他维度" json:"method"`
	DbCode    string `description:"数据库编码" json:"dbcode"`
	RowCode   string `description:"行-维度: zb; sj; reg" json:"rowcode"`
	ColCode   string `description:"列-维度: zb; sj; reg" json:"colcode"`
	WdsList   []Wds  `description:"维度列表" json:"wdsList"`
	DfwdsList []Wds  `description:"df不知道啥意思...反正也是维度相关的" json:"dfwdsList"`
}

// Wds 维度
type Wds struct {
	WdCode    string `description:"维度: zb-指标; sj-时间; reg-地区" json:"wdcode"`
	ValueCode string `description:"维度编码" json:"valuecode"`
}

// CommonDataApiRequest 数据接口请求
func CommonDataApiRequest(req DataApiReq) (resp QuotaListDataResp, err error) {
	var b []byte
	defer func() {
		if err != nil {
			r, _ := json.Marshal(req)
			utils.FileLog.Error("CommonDataApiRequest Err request: %s", r)
			utils.FileLog.Info("CommonDataApiRequest Err result: %s", string(b))
		}
	}()
	if req.DbCode == "" {
		return
	}
	if req.Method == "" {
		req.Method = "QueryData"
	}
	if req.RowCode == "" {
		req.RowCode = "zb"
	}
	if req.ColCode == "" {
		req.ColCode = "sj"
	}

	// 构建查询
	f := url.Values{}
	f.Add("m", req.Method)
	f.Add("dbcode", req.DbCode)
	f.Add("rowcode", req.RowCode)
	f.Add("colcode", req.ColCode)
	wds := `[]`
	if len(req.WdsList) > 0 {
		wdsByte, e := json.Marshal(req.WdsList)
		if e != nil {
			err = fmt.Errorf("wds json marshal err: %s", e.Error())
			return
		}
		wds = string(wdsByte)
	}
	dfwds := `[]`
	if len(req.DfwdsList) > 0 {
		dfwdsByte, e := json.Marshal(req.DfwdsList)
		if e != nil {
			err = fmt.Errorf("dfwds json marshal err: %s", e.Error())
			return
		}
		dfwds = string(dfwdsByte)
	}
	f.Add("wds", wds)
	f.Add("dfwds", dfwds)
	f.Add("k1", fmt.Sprint(time.Now().UnixNano()/1e6))
	f.Add("h", "1")

	// 响应
	b, e := NationalHttpPost(NationalStatisticsBaseReqUrl, f.Encode())
	if e != nil {
		err = fmt.Errorf("http request err: %s", e.Error())
		return
	}
	if len(b) == 0 {
		err = fmt.Errorf("http result empty")
		return
	}
	if e = json.Unmarshal(b, &resp); e != nil {
		err = fmt.Errorf("resp unmarshal err: %s", e.Error())
		return
	}
	if resp.ReturnCode != 200 {
		err = fmt.Errorf("resp code err: %d", resp.ReturnCode)
		return
	}
	return
}

// QuotaListDataResp 指标数据列表响应体
type QuotaListDataResp struct {
	ReturnCode int `description:"状态码" json:"returncode"`
	ReturnData struct {
		DataNodes []QuotaDataNode `json:"datanodes"`
		WdNodes   []QuotaWdNode   `json:"wdnodes"`
	}
}

// QuotaDataNode 指标数据节点
type QuotaDataNode struct {
	Code string `description:"编码"`
	Data struct {
		Data    float64 `description:"指标值"`
		HasData bool    `description:"是否有值" json:"hasdata"`
		StrData string  `description:"指标值(字符串)" json:"strdata"`
	}
	Wds []Wds
}

// QuotaWdNode 维度节点
type QuotaWdNode struct {
	WdCode string `description:"示例: zb; sj; reg;" json:"wdcode"`
	WdName string `description:"示例: 指标; 时间; 地区" json:"wdname"`
	Nodes  []QuotaWdNodeData
}

// QuotaWdNodeData 维度节点数据
type QuotaWdNodeData struct {
	Code     string `description:"指标编码"`
	Name     string `description:"指标名称"`
	Unit     string `description:"单位"`
	SortCode int    `description:"编码排序" json:"sortcode"`
}

// OtherWdResp 其他维度信息响应体
type OtherWdResp struct {
	ReturnCode int           `description:"状态码" json:"returncode"`
	ReturnData []OtherWdData `description:"响应数据" json:"returndata"`
}

// OtherWdData 其他维度数据
type OtherWdData struct {
	IsSj   bool           `description:"是否为时间" json:"issj"`
	WdCode string         `description:"维度编码" json:"wdcode"`
	WdName string         `description:"维度名称" json:"wdname"`
	Nodes  []OtherWdNodes `description:"维度数据" json:"nodes"`
}

type OtherWdNodes struct {
	Code string `description:"编码" json:"code"`
	Name string `description:"名称" json:"name"`
	Sort string `description:"排序" json:"sort"`
}

// formatMonth2YearDateCode 将日期code转为对应日期
func formatMonth2YearDateCode(dateCode string) (date time.Time, err error) {
	if dateCode == "" {
		return
	}
	// 根据日期code长度进行区分, 格式为三种: 月度-200601; 季度-2006A; 年度-2006
	switch len([]rune(dateCode)) {
	case 6:
		// 月度日期取每月最后一天
		m := dateCode[4:]
		replaceMonth := map[string]string{
			"01": "0131",
			"02": "0228",
			"03": "0331",
			"04": "0430",
			"05": "0531",
			"06": "0630",
			"07": "0731",
			"08": "0831",
			"09": "0930",
			"10": "1031",
			"11": "1130",
			"12": "1231",
		}
		md := fmt.Sprintf("%s%s", dateCode[:4], replaceMonth[m])
		t, e := time.ParseInLocation("20060102", md, time.Local)
		if e != nil {
			err = fmt.Errorf("月度指标日期转换失败, Err: %s", e.Error())
			return
		}
		date = t
		break
	case 5:
		// 季度ABCD转换成对应日期
		dateSuffixMap := map[string]string{
			"A": "03-31",
			"B": "06-30",
			"C": "09-30",
			"D": "12-31",
		}
		dateCode = strings.ToUpper(dateCode)
		quarterTab := dateCode[4:]
		dateStr := fmt.Sprintf("%s-%s", dateCode[:4], dateSuffixMap[quarterTab])
		t, e := time.ParseInLocation(utils.FormatDate, dateStr, time.Local)
		if e != nil {
			err = fmt.Errorf("季度指标日期转换失败, Err: %s", e.Error())
			return
		}
		date = t
		break
	case 4:
		dateStr := fmt.Sprintf("%s-%s", dateCode, "12-31")
		t, e := time.ParseInLocation(utils.FormatDate, dateStr, time.Local)
		if e != nil {
			err = fmt.Errorf("年度指标日期转换失败, Err: %s", e.Error())
			return
		}
		date = t
		break
	default:
		err = fmt.Errorf("日期code格式有误, code: %s", dateCode)
		return
	}
	return
}

// GetOtherWd 获取Db下其他维度信息
func GetOtherWd(dbCode, rowCode, colCode string) (wdList []OtherWdData, err error) {
	if dbCode == "" {
		return
	}
	if rowCode == "" {
		rowCode = "zb"
	}
	if colCode == "" {
		colCode = "sj"
	}

	// 构建查询
	f := url.Values{}
	f.Add("m", "getOtherWds")
	f.Add("dbcode", dbCode)
	f.Add("rowcode", rowCode)
	f.Add("colcode", colCode)
	f.Add("wds", `[]`)
	f.Add("k1", fmt.Sprint(time.Now().UnixNano()/1e6))
	f.Add("h", "1")
	r, e := NationalHttpPost(NationalStatisticsBaseReqUrl, f.Encode())
	if e != nil {
		err = fmt.Errorf("请求其他维度信息失败, Err: %s", e.Error())
		return
	}
	utils.FileLog.Info("GetOtherWdInfo Result: %s", string(r))

	// 响应
	resp := new(OtherWdResp)
	if e = json.Unmarshal(r, &resp); e != nil {
		err = fmt.Errorf("其他维度信息Unmarshal Err: %s", e.Error())
		return
	}
	if resp == nil {
		err = fmt.Errorf("其他维度信息请求结果为空")
		return
	}
	if resp.ReturnCode != 200 {
		err = fmt.Errorf("其他维度信息请求有误, Code: %d", resp.ReturnCode)
		return
	}
	wdList = resp.ReturnData
	return
}

func ApiTest() (err error) {
	//f := url.Values{}
	//f.Add("m", "QueryData")
	//f.Add("dbcode", "fsyd")
	//f.Add("rowcode", "zb")
	//f.Add("colcode", "sj")
	//f.Add("wds", `[{"wdcode":"reg","valuecode":"000"}]`)
	//f.Add("dfwds", `[{"wdcode":"zb","valuecode":"A01"}]`)
	f := url.Values{}
	f.Add("m", "QueryData")
	f.Add("dbcode", "fsjd")
	f.Add("rowcode", "zb")
	f.Add("colcode", "sj")
	//f.Add("wds", `[{"wdcode":"reg","valuecode":"110000"}]`)
	f.Add("wds", `[{"wdcode":"reg","valuecode":"310000"}]`)
	f.Add("dfwds", `[{"wdcode":"zb","valuecode":"A0501"},{"wdcode":"sj","valuecode":"LAST18"}]`)
	f.Add("k1", fmt.Sprint(time.Now().UnixNano()/1e6))
	f.Add("h", "1")

	//f := url.Values{}
	//f.Add("m", "QueryData")
	//f.Add("dbcode", "gatyd")
	//f.Add("rowcode", "sj")
	//f.Add("colcode", "reg")
	//f.Add("wds", `[{"wdcode":"zb","valuecode":"A010A"}]`)
	//f.Add("dfwds", `[{"wdcode":"sj","valuecode":"LAST36"}]`)
	//f := url.Values{}
	//f.Add("m", "QueryData")
	//f.Add("dbcode", "fsyd")
	//f.Add("rowcode", "zb")
	//f.Add("colcode", "sj")
	//f.Add("wds", `[{"wdcode":"reg","valuecode":"000"}]`)
	//f.Add("dfwds", `[{"wdcode":"zb","valuecode":"A01"}]`)
	//f.Add("k1", fmt.Sprint(time.Now().UnixNano()/1e6))
	//f.Add("h", "1")

	r, e := NationalHttpPost(NationalStatisticsBaseReqUrl, f.Encode())
	if e != nil {
		fmt.Println("请求失败, Err: ", e.Error())
		return
	}
	utils.FileLog.Info("test result: %s", string(r))
	return
}