package national_data import ( "bytes" "crypto/tls" "encoding/json" "eta/eta_crawler/utils" "fmt" "io" "io/ioutil" "net/http" "net/url" "strings" "time" ) const ( NationalStatisticsBaseReqUrl = "https://data.stats.gov.cn/easyquery.htm" ) func NationalHttpPost(reqUrl, payload string) (result []byte, err error) { // 随机延迟执行 r := utils.RangeRand(5000, 8000) //fmt.Printf("随机延迟%d\n", r) if r > 6100 && r < 6250 { time.Sleep(15 * time.Second) } else { time.Sleep(time.Duration(r) * time.Millisecond) } tr := &http.Transport{ TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, } client := &http.Client{ Transport: tr, } req, err := http.NewRequest("POST", reqUrl, strings.NewReader(payload)) if err != nil { return } req.Header.Add("Accept", "text/plain, */*; q=0.01") req.Header.Add("Accept-Encoding", "tgzip, deflate, br") req.Header.Add("Accept-Language", "zh-CN,zh;q=0.9") req.Header.Add("Connection", "keep-alive") req.Header.Add("Content-Length", "37") req.Header.Add("Content-Type", "application/x-www-form-urlencoded") req.Header.Add("Cookie", "wzws_sessionid=gDExNS4xOTQuMTAyLjEyN6BkERzUgmZjNWVlMYFiOWNiZDg=; JSESSIONID=UOri2Cu3f3c-Y3rPgXWJ04E8pfbeyAUGG-s7zJ7Tt0JhlEiLi0EU!412929168; u=5") req.Header.Add("Host", "data.stats.gov.cn") req.Header.Add("Origin", "https://data.stats.gov.cn") req.Header.Set("Referer", "https://data.stats.gov.cn/easyquery.htm?cn=A01") req.Header.Set("sec-ch-ua", "\"Not_A Brand\";v=\"99\", \"Google Chrome\";v=\"109\", \"Chromium\";v=\"109\"") req.Header.Set("sec-ch-ua-mobile", "?0") req.Header.Set("sec-ch-ua-platform", "\"Windows\"") req.Header.Set("Sec-Fetch-Dest", "empty") req.Header.Set("Sec-Fetch-Mode", "cors") req.Header.Set("Sec-Fetch-Site", "same-origin") req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/109.0.0.0 Safari/537.36") req.Header.Set("X-Requested-With", "XMLHttpRequest") res, err := client.Do(req) if err != nil { return } defer func() { _ = res.Body.Close() }() // 此处用io.Copy替代ioutil.ReadAll方法避免数据过大读取不完整 var b []byte buf := bytes.NewBuffer(b) _, err = io.Copy(buf, res.Body) if err != nil { return } result = buf.Bytes() return } func NationalGet(reqUrl, payload string) (err error) { tr := &http.Transport{ TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, } client := &http.Client{ Transport: tr, } req, err := http.NewRequest("GET", reqUrl, strings.NewReader(payload)) if err != nil { return } res, err := client.Do(req) if err != nil { return } defer res.Body.Close() _, err = ioutil.ReadAll(res.Body) if err != nil { return } Cookie := res.Header.Get("Cookie") fmt.Println(Cookie) rcookie := req.Header.Get("Cookie") fmt.Println("rcookie") fmt.Println(rcookie) //fmt.Println("body:" + string(body)) cookiesArr := res.Cookies() fmt.Println("cookiesArrLen:", len(cookiesArr)) for k, v := range cookiesArr { fmt.Println(k, v) } return } // DataApiReq 数据接口请求体 type DataApiReq struct { Method string `description:"方法: QueryData-查询数据; getOtherWds-获取其他维度" json:"method"` DbCode string `description:"数据库编码" json:"dbcode"` RowCode string `description:"行-维度: zb; sj; reg" json:"rowcode"` ColCode string `description:"列-维度: zb; sj; reg" json:"colcode"` WdsList []Wds `description:"维度列表" json:"wdsList"` DfwdsList []Wds `description:"df不知道啥意思...反正也是维度相关的" json:"dfwdsList"` } // Wds 维度 type Wds struct { WdCode string `description:"维度: zb-指标; sj-时间; reg-地区" json:"wdcode"` ValueCode string `description:"维度编码" json:"valuecode"` } // CommonDataApiRequest 数据接口请求 func CommonDataApiRequest(req DataApiReq) (resp QuotaListDataResp, err error) { var b []byte defer func() { if err != nil { r, _ := json.Marshal(req) utils.FileLog.Error("CommonDataApiRequest Err request: %s", r) utils.FileLog.Info("CommonDataApiRequest Err result: %s", string(b)) } }() if req.DbCode == "" { return } if req.Method == "" { req.Method = "QueryData" } if req.RowCode == "" { req.RowCode = "zb" } if req.ColCode == "" { req.ColCode = "sj" } // 构建查询 f := url.Values{} f.Add("m", req.Method) f.Add("dbcode", req.DbCode) f.Add("rowcode", req.RowCode) f.Add("colcode", req.ColCode) wds := `[]` if len(req.WdsList) > 0 { wdsByte, e := json.Marshal(req.WdsList) if e != nil { err = fmt.Errorf("wds json marshal err: %s", e.Error()) return } wds = string(wdsByte) } dfwds := `[]` if len(req.DfwdsList) > 0 { dfwdsByte, e := json.Marshal(req.DfwdsList) if e != nil { err = fmt.Errorf("dfwds json marshal err: %s", e.Error()) return } dfwds = string(dfwdsByte) } f.Add("wds", wds) f.Add("dfwds", dfwds) f.Add("k1", fmt.Sprint(time.Now().UnixNano()/1e6)) f.Add("h", "1") // 响应 b, e := NationalHttpPost(NationalStatisticsBaseReqUrl, f.Encode()) if e != nil { err = fmt.Errorf("http request err: %s", e.Error()) return } if len(b) == 0 { err = fmt.Errorf("http result empty") return } if e = json.Unmarshal(b, &resp); e != nil { err = fmt.Errorf("resp unmarshal err: %s", e.Error()) return } if resp.ReturnCode != 200 { err = fmt.Errorf("resp code err: %d", resp.ReturnCode) return } return } // QuotaListDataResp 指标数据列表响应体 type QuotaListDataResp struct { ReturnCode int `description:"状态码" json:"returncode"` ReturnData struct { DataNodes []QuotaDataNode `json:"datanodes"` WdNodes []QuotaWdNode `json:"wdnodes"` } } // QuotaDataNode 指标数据节点 type QuotaDataNode struct { Code string `description:"编码"` Data struct { Data float64 `description:"指标值"` HasData bool `description:"是否有值" json:"hasdata"` StrData string `description:"指标值(字符串)" json:"strdata"` } Wds []Wds } // QuotaWdNode 维度节点 type QuotaWdNode struct { WdCode string `description:"示例: zb; sj; reg;" json:"wdcode"` WdName string `description:"示例: 指标; 时间; 地区" json:"wdname"` Nodes []QuotaWdNodeData } // QuotaWdNodeData 维度节点数据 type QuotaWdNodeData struct { Code string `description:"指标编码"` Name string `description:"指标名称"` Unit string `description:"单位"` SortCode int `description:"编码排序" json:"sortcode"` } // OtherWdResp 其他维度信息响应体 type OtherWdResp struct { ReturnCode int `description:"状态码" json:"returncode"` ReturnData []OtherWdData `description:"响应数据" json:"returndata"` } // OtherWdData 其他维度数据 type OtherWdData struct { IsSj bool `description:"是否为时间" json:"issj"` WdCode string `description:"维度编码" json:"wdcode"` WdName string `description:"维度名称" json:"wdname"` Nodes []OtherWdNodes `description:"维度数据" json:"nodes"` } type OtherWdNodes struct { Code string `description:"编码" json:"code"` Name string `description:"名称" json:"name"` Sort string `description:"排序" json:"sort"` } // formatMonth2YearDateCode 将日期code转为对应日期 func formatMonth2YearDateCode(dateCode string) (date time.Time, err error) { if dateCode == "" { return } // 根据日期code长度进行区分, 格式为三种: 月度-200601; 季度-2006A; 年度-2006 switch len([]rune(dateCode)) { case 6: // 月度日期取每月最后一天 m := dateCode[4:] replaceMonth := map[string]string{ "01": "0131", "02": "0228", "03": "0331", "04": "0430", "05": "0531", "06": "0630", "07": "0731", "08": "0831", "09": "0930", "10": "1031", "11": "1130", "12": "1231", } md := fmt.Sprintf("%s%s", dateCode[:4], replaceMonth[m]) t, e := time.ParseInLocation("20060102", md, time.Local) if e != nil { err = fmt.Errorf("月度指标日期转换失败, Err: %s", e.Error()) return } date = t break case 5: // 季度ABCD转换成对应日期 dateSuffixMap := map[string]string{ "A": "03-31", "B": "06-30", "C": "09-30", "D": "12-31", } dateCode = strings.ToUpper(dateCode) quarterTab := dateCode[4:] dateStr := fmt.Sprintf("%s-%s", dateCode[:4], dateSuffixMap[quarterTab]) t, e := time.ParseInLocation(utils.FormatDate, dateStr, time.Local) if e != nil { err = fmt.Errorf("季度指标日期转换失败, Err: %s", e.Error()) return } date = t break case 4: dateStr := fmt.Sprintf("%s-%s", dateCode, "12-31") t, e := time.ParseInLocation(utils.FormatDate, dateStr, time.Local) if e != nil { err = fmt.Errorf("年度指标日期转换失败, Err: %s", e.Error()) return } date = t break default: err = fmt.Errorf("日期code格式有误, code: %s", dateCode) return } return } // GetOtherWd 获取Db下其他维度信息 func GetOtherWd(dbCode, rowCode, colCode string) (wdList []OtherWdData, err error) { if dbCode == "" { return } if rowCode == "" { rowCode = "zb" } if colCode == "" { colCode = "sj" } // 构建查询 f := url.Values{} f.Add("m", "getOtherWds") f.Add("dbcode", dbCode) f.Add("rowcode", rowCode) f.Add("colcode", colCode) f.Add("wds", `[]`) f.Add("k1", fmt.Sprint(time.Now().UnixNano()/1e6)) f.Add("h", "1") r, e := NationalHttpPost(NationalStatisticsBaseReqUrl, f.Encode()) if e != nil { err = fmt.Errorf("请求其他维度信息失败, Err: %s", e.Error()) return } utils.FileLog.Info("GetOtherWdInfo Result: %s", string(r)) // 响应 resp := new(OtherWdResp) if e = json.Unmarshal(r, &resp); e != nil { err = fmt.Errorf("其他维度信息Unmarshal Err: %s", e.Error()) return } if resp == nil { err = fmt.Errorf("其他维度信息请求结果为空") return } if resp.ReturnCode != 200 { err = fmt.Errorf("其他维度信息请求有误, Code: %d", resp.ReturnCode) return } wdList = resp.ReturnData return } func ApiTest() (err error) { //f := url.Values{} //f.Add("m", "QueryData") //f.Add("dbcode", "fsyd") //f.Add("rowcode", "zb") //f.Add("colcode", "sj") //f.Add("wds", `[{"wdcode":"reg","valuecode":"000"}]`) //f.Add("dfwds", `[{"wdcode":"zb","valuecode":"A01"}]`) f := url.Values{} f.Add("m", "QueryData") f.Add("dbcode", "fsjd") f.Add("rowcode", "zb") f.Add("colcode", "sj") //f.Add("wds", `[{"wdcode":"reg","valuecode":"110000"}]`) f.Add("wds", `[{"wdcode":"reg","valuecode":"310000"}]`) f.Add("dfwds", `[{"wdcode":"zb","valuecode":"A0501"},{"wdcode":"sj","valuecode":"LAST18"}]`) f.Add("k1", fmt.Sprint(time.Now().UnixNano()/1e6)) f.Add("h", "1") //f := url.Values{} //f.Add("m", "QueryData") //f.Add("dbcode", "gatyd") //f.Add("rowcode", "sj") //f.Add("colcode", "reg") //f.Add("wds", `[{"wdcode":"zb","valuecode":"A010A"}]`) //f.Add("dfwds", `[{"wdcode":"sj","valuecode":"LAST36"}]`) //f := url.Values{} //f.Add("m", "QueryData") //f.Add("dbcode", "fsyd") //f.Add("rowcode", "zb") //f.Add("colcode", "sj") //f.Add("wds", `[{"wdcode":"reg","valuecode":"000"}]`) //f.Add("dfwds", `[{"wdcode":"zb","valuecode":"A01"}]`) //f.Add("k1", fmt.Sprint(time.Now().UnixNano()/1e6)) //f.Add("h", "1") r, e := NationalHttpPost(NationalStatisticsBaseReqUrl, f.Encode()) if e != nil { fmt.Println("请求失败, Err: ", e.Error()) return } utils.FileLog.Info("test result: %s", string(r)) return }