package national_data import ( "crypto/tls" "encoding/json" "fmt" "hongze/hongze_data_crawler/models" "io/ioutil" "net/http" "net/url" "strings" "time" ) const ( NationalStatisticsBaseReqUrl = "https://data.stats.gov.cn/easyquery.htm" ) func NationalHttpPost(reqUrl, payload string) (result []byte, err error) { tr := &http.Transport{ TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, } client := &http.Client{ Transport: tr, } req, err := http.NewRequest("POST", reqUrl, strings.NewReader(payload)) if err != nil { return } req.Header.Add("Accept", "text/plain, */*; q=0.01") req.Header.Add("Accept-Encoding", "tgzip, deflate, br") req.Header.Add("Accept-Language", "zh-CN,zh;q=0.9") req.Header.Add("Connection", "keep-alive") req.Header.Add("Content-Length", "37") req.Header.Add("Content-Type", "application/x-www-form-urlencoded") req.Header.Add("Cookie", "wzws_sessionid=gDExNS4xOTQuMTAyLjEyN6BkERzUgmZjNWVlMYFiOWNiZDg=; JSESSIONID=UOri2Cu3f3c-Y3rPgXWJ04E8pfbeyAUGG-s7zJ7Tt0JhlEiLi0EU!412929168; u=5") req.Header.Add("Host", "data.stats.gov.cn") req.Header.Add("Origin", "https://data.stats.gov.cn") req.Header.Set("Referer", "https://data.stats.gov.cn/easyquery.htm?cn=A01") req.Header.Set("sec-ch-ua", "\"Not_A Brand\";v=\"99\", \"Google Chrome\";v=\"109\", \"Chromium\";v=\"109\"") req.Header.Set("sec-ch-ua-mobile", "?0") req.Header.Set("sec-ch-ua-platform", "\"Windows\"") req.Header.Set("Sec-Fetch-Dest", "empty") req.Header.Set("Sec-Fetch-Mode", "cors") req.Header.Set("Sec-Fetch-Site", "same-origin") req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/109.0.0.0 Safari/537.36") req.Header.Set("X-Requested-With", "XMLHttpRequest") res, err := client.Do(req) if err != nil { return } defer res.Body.Close() body, err := ioutil.ReadAll(res.Body) if err != nil { return } result = body return } func NationalGet(reqUrl, payload string) (err error) { tr := &http.Transport{ TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, } client := &http.Client{ Transport: tr, } req, err := http.NewRequest("GET", reqUrl, strings.NewReader(payload)) if err != nil { return } res, err := client.Do(req) if err != nil { return } defer res.Body.Close() _, err = ioutil.ReadAll(res.Body) if err != nil { return } Cookie := res.Header.Get("Cookie") fmt.Println(Cookie) rcookie := req.Header.Get("Cookie") fmt.Println("rcookie") fmt.Println(rcookie) //fmt.Println("body:" + string(body)) cookiesArr := res.Cookies() fmt.Println("cookiesArrLen:", len(cookiesArr)) for k, v := range cookiesArr { fmt.Println(k, v) } return } // QuotaClassifyTreeResp 指标分类树响应 type QuotaClassifyTreeResp struct { Id string `description:"分类ID(字符串)"` IsParent bool `description:"是否为父级"` Name string `description:"分类名称"` Pid string `description:"父级分类ID"` Dbcode string `description:"源-dbocde"` Wdcode string `description:"源-wdcode"` } // SyncQuotaClassifyTree 同步指标分类树 // dbcode: hgyd-月度; hgjd-季度; ... func SyncQuotaClassifyTree() (err error) { defer func() { if err != nil { fmt.Println(err.Error()) } }() items := make([]*models.BaseFromNationalStatisticsClassify, 0) initId := "zb" initDbcode := "hgyd" initWdcode := "zb" resp, e := curlAndFormatQuotaClassify(initId, initDbcode, initWdcode, items) if e != nil { err = fmt.Errorf("递归指标分类失败, Err: %s", e.Error()) return } items = resp // 去重 classifyMap := make(map[string]bool) classifyOB := new(models.BaseFromNationalStatisticsClassify) classifyPars := make([]interface{}, 0) classifies, e := classifyOB.GetItemsByCondition("", classifyPars, []string{}, "") if e != nil { err = fmt.Errorf("获取指标分类列表失败, Err: %s", e.Error()) return } for _, c := range classifies { classifyMap[c.Id] = true } finalList := make([]*models.BaseFromNationalStatisticsClassify, 0) for _, v := range items { if classifyMap[v.Id] { continue } finalList = append(finalList, v) } if e = classifyOB.CreateMulti(items); e != nil { err = fmt.Errorf("批量新增指标分类失败, Err: %s", e.Error()) return } return } // curlAndFormatQuotaClassify 递归请求分类树 func curlAndFormatQuotaClassify(id, dbcode, wdcode string, items []*models.BaseFromNationalStatisticsClassify) (resp []*models.BaseFromNationalStatisticsClassify, err error) { f := url.Values{} f.Add("id", id) f.Add("dbcode", dbcode) f.Add("wdcode", wdcode) f.Add("m", "getTree") r, e := NationalHttpPost(NationalStatisticsBaseReqUrl, f.Encode()) if e != nil { err = fmt.Errorf("请求指标分类树失败, Err: %s", e.Error()) return } list := make([]*QuotaClassifyTreeResp, 0) if e = json.Unmarshal(r, &list); e != nil { err = fmt.Errorf("解析响应数据失败, Err: %s", e.Error()) return } nowTime := time.Now().Local() for _, v := range list { isParent := 0 if v.IsParent { isParent = 1 } items = append(items, &models.BaseFromNationalStatisticsClassify{ ClassifyName: v.Name, Id: v.Id, Dbcode: v.Dbcode, Wdcode: v.Wdcode, Pid: v.Pid, IsParent: isParent, CreateTime: nowTime, ModifyTime: nowTime, }) // 向下递归 if isParent == 1 { time.Sleep(1 * time.Second) // 缓缓...毕竟接口是人家的... items, e = curlAndFormatQuotaClassify(v.Id, v.Dbcode, v.Wdcode, items) if e != nil { err = fmt.Errorf("递归请求分类树失败, Err: %s", e.Error()) return } } } return items, nil } type QuotaListDataResp struct { ReturnCode int `description:"状态码" json:"returncode"` ReturnData struct { DataNodes []QuotaDataNode `json:"datanodes"` WdNodes struct{} `json:"wdnodes"` } } type QuotaDataNode struct { Code string `description:"编码"` Data struct { Data float64 `description:"指标值"` HasData bool `description:"是否有值" json:"hasdata"` StrData string `description:"指标值(字符串)" json:"strdata"` } Wds []QuotaDataWds } type QuotaDataWds struct { ValueCode string `json:"valuecode"` WdCode string `json:"wdcode"` } // SyncQuotaDataFromDbCodeAndId 同步指标值 func SyncQuotaDataFromDbCodeAndId() (err error) { // 查询无父级的指标分类 //classifyOB := new(models.BaseFromNationalStatisticsClassify) //classifyCond := ` AND is_parent = 0` //classifyPars := make([]interface{}, 0) //classifyOrder := ` ORDER BY base_from_national_statistics_classify_id ASC` //classifies, e := classifyOB.GetItemsByCondition(classifyCond, classifyPars, []string{}, classifyOrder) //if e != nil { // err = fmt.Errorf("获取指标分类列表失败, Err: %s", e.Error()) // return //} // m: QueryData // dbcode: hgyd // rowcode: zb // colcode: sj // wds: [] // dfwds: [{"wdcode":"zb","valuecode":"A010101"}] // k1: 1678872233342 // h: 1 time.Sleep(2 * time.Second) // 构建查询 f := url.Values{} f.Add("m", "QueryData") f.Add("dbcode", "hgyd") f.Add("rowcode", "zb") f.Add("colcode", "sj") f.Add("wds", "[]") f.Add("dfwds", `[{"wdcode":"zb","valuecode":"A010101"}]`) f.Add("k1", fmt.Sprint(time.Now().UnixNano()/1e6)) f.Add("h", "1") r, e := NationalHttpPost(NationalStatisticsBaseReqUrl, f.Encode()) fmt.Println("err: ", e) if e != nil { fmt.Println("获取失败, Err: ", e.Error()) //err = fmt.Errorf("请求指标分类树失败, Err: %s", e.Error()) return } fmt.Println("res: ", string(r)) return }