eta_crawler/services/national_data/national_data.go

package national_data

import (
	"crypto/tls"
	"encoding/json"
	"fmt"
	"hongze/hongze_data_crawler/models"
	"io/ioutil"
	"net/http"
	"net/url"
	"strings"
	"time"
)

const (
	NationalStatisticsBaseReqUrl = "https://data.stats.gov.cn/easyquery.htm"
)

func NationalHttpPost(reqUrl, payload string) (result []byte, err error) {
	tr := &http.Transport{
		TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
	}
	client := &http.Client{
		Transport: tr,
	}
	req, err := http.NewRequest("POST", reqUrl, strings.NewReader(payload))
	if err != nil {
		return
	}
	req.Header.Add("Accept", "text/plain, */*; q=0.01")
	req.Header.Add("Accept-Encoding", "tgzip, deflate, br")
	req.Header.Add("Accept-Language", "zh-CN,zh;q=0.9")
	req.Header.Add("Connection", "keep-alive")
	req.Header.Add("Content-Length", "37")
	req.Header.Add("Content-Type", "application/x-www-form-urlencoded")
	req.Header.Add("Cookie", "wzws_sessionid=gDExNS4xOTQuMTAyLjEyN6BkERzUgmZjNWVlMYFiOWNiZDg=; JSESSIONID=UOri2Cu3f3c-Y3rPgXWJ04E8pfbeyAUGG-s7zJ7Tt0JhlEiLi0EU!412929168; u=5")
	req.Header.Add("Host", "data.stats.gov.cn")
	req.Header.Add("Origin", "https://data.stats.gov.cn")
	req.Header.Set("Referer", "https://data.stats.gov.cn/easyquery.htm?cn=A01")
	req.Header.Set("sec-ch-ua", "\"Not_A Brand\";v=\"99\", \"Google Chrome\";v=\"109\", \"Chromium\";v=\"109\"")
	req.Header.Set("sec-ch-ua-mobile", "?0")
	req.Header.Set("sec-ch-ua-platform", "\"Windows\"")
	req.Header.Set("Sec-Fetch-Dest", "empty")
	req.Header.Set("Sec-Fetch-Mode", "cors")
	req.Header.Set("Sec-Fetch-Site", "same-origin")
	req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/109.0.0.0 Safari/537.36")
	req.Header.Set("X-Requested-With", "XMLHttpRequest")
	res, err := client.Do(req)
	if err != nil {
		return
	}
	defer res.Body.Close()
	body, err := ioutil.ReadAll(res.Body)
	if err != nil {
		return
	}
	result = body
	return
}

func NationalGet(reqUrl, payload string) (err error) {
	tr := &http.Transport{
		TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
	}
	client := &http.Client{
		Transport: tr,
	}
	req, err := http.NewRequest("GET", reqUrl, strings.NewReader(payload))
	if err != nil {
		return
	}
	res, err := client.Do(req)
	if err != nil {
		return
	}
	defer res.Body.Close()
	_, err = ioutil.ReadAll(res.Body)
	if err != nil {
		return
	}

	Cookie := res.Header.Get("Cookie")
	fmt.Println(Cookie)
	rcookie := req.Header.Get("Cookie")
	fmt.Println("rcookie")
	fmt.Println(rcookie)
	//fmt.Println("body:" + string(body))
	cookiesArr := res.Cookies()
	fmt.Println("cookiesArrLen:", len(cookiesArr))
	for k, v := range cookiesArr {
		fmt.Println(k, v)
	}
	return
}

// QuotaClassifyTreeResp 指标分类树响应
type QuotaClassifyTreeResp struct {
	Id       string `description:"分类ID(字符串)"`
	IsParent bool   `description:"是否为父级"`
	Name     string `description:"分类名称"`
	Pid      string `description:"父级分类ID"`
	Dbcode   string `description:"源-dbocde"`
	Wdcode   string `description:"源-wdcode"`
}

// SyncQuotaClassifyTree 同步指标分类树
// dbcode: hgyd-月度; hgjd-季度; ...
func SyncQuotaClassifyTree() (err error) {
	defer func() {
		if err != nil {
			fmt.Println(err.Error())
		}
	}()

	items := make([]*models.BaseFromNationalStatisticsClassify, 0)
	initId := "zb"
	initDbcode := "hgyd"
	initWdcode := "zb"
	resp, e := curlAndFormatQuotaClassify(initId, initDbcode, initWdcode, items)
	if e != nil {
		err = fmt.Errorf("递归指标分类失败, Err: %s", e.Error())
		return
	}
	items = resp

	// 去重
	classifyMap := make(map[string]bool)
	classifyOB := new(models.BaseFromNationalStatisticsClassify)
	classifyPars := make([]interface{}, 0)
	classifies, e := classifyOB.GetItemsByCondition("", classifyPars, []string{}, "")
	if e != nil {
		err = fmt.Errorf("获取指标分类列表失败, Err: %s", e.Error())
		return
	}
	for _, c := range classifies {
		classifyMap[c.Id] = true
	}

	finalList := make([]*models.BaseFromNationalStatisticsClassify, 0)
	for _, v := range items {
		if classifyMap[v.Id] {
			continue
		}
		finalList = append(finalList, v)
	}
	if e = classifyOB.CreateMulti(items); e != nil {
		err = fmt.Errorf("批量新增指标分类失败, Err: %s", e.Error())
		return
	}
	return
}

// curlAndFormatQuotaClassify 递归请求分类树
func curlAndFormatQuotaClassify(id, dbcode, wdcode string, items []*models.BaseFromNationalStatisticsClassify) (resp []*models.BaseFromNationalStatisticsClassify, err error) {
	f := url.Values{}
	f.Add("id", id)
	f.Add("dbcode", dbcode)
	f.Add("wdcode", wdcode)
	f.Add("m", "getTree")
	r, e := NationalHttpPost(NationalStatisticsBaseReqUrl, f.Encode())
	if e != nil {
		err = fmt.Errorf("请求指标分类树失败, Err: %s", e.Error())
		return
	}
	list := make([]*QuotaClassifyTreeResp, 0)
	if e = json.Unmarshal(r, &list); e != nil {
		err = fmt.Errorf("解析响应数据失败, Err: %s", e.Error())
		return
	}

	nowTime := time.Now().Local()
	for _, v := range list {
		isParent := 0
		if v.IsParent {
			isParent = 1
		}
		items = append(items, &models.BaseFromNationalStatisticsClassify{
			ClassifyName: v.Name,
			Id:           v.Id,
			Dbcode:       v.Dbcode,
			Wdcode:       v.Wdcode,
			Pid:          v.Pid,
			IsParent:     isParent,
			CreateTime:   nowTime,
			ModifyTime:   nowTime,
		})

		// 向下递归
		if isParent == 1 {
			time.Sleep(1 * time.Second) // 缓缓...毕竟接口是人家的...
			items, e = curlAndFormatQuotaClassify(v.Id, v.Dbcode, v.Wdcode, items)
			if e != nil {
				err = fmt.Errorf("递归请求分类树失败, Err: %s", e.Error())
				return
			}
		}
	}
	return items, nil
}

type QuotaListDataResp struct {
	ReturnCode int `description:"状态码" json:"returncode"`
	ReturnData struct {
		DataNodes []QuotaDataNode `json:"datanodes"`
		WdNodes   struct{}        `json:"wdnodes"`
	}
}

type QuotaDataNode struct {
	Code string `description:"编码"`
	Data struct {
		Data    float64 `description:"指标值"`
		HasData bool    `description:"是否有值" json:"hasdata"`
		StrData string  `description:"指标值(字符串)" json:"strdata"`
	}
	Wds []QuotaDataWds
}

type QuotaDataWds struct {
	ValueCode string `json:"valuecode"`
	WdCode    string `json:"wdcode"`
}

// SyncQuotaDataFromDbCodeAndId 同步指标值
func SyncQuotaDataFromDbCodeAndId() (err error) {
	// 查询无父级的指标分类
	//classifyOB := new(models.BaseFromNationalStatisticsClassify)
	//classifyCond := ` AND is_parent = 0`
	//classifyPars := make([]interface{}, 0)
	//classifyOrder := ` ORDER BY base_from_national_statistics_classify_id ASC`
	//classifies, e := classifyOB.GetItemsByCondition(classifyCond, classifyPars, []string{}, classifyOrder)
	//if e != nil {
	//	err = fmt.Errorf("获取指标分类列表失败, Err: %s", e.Error())
	//	return
	//}

	//	m: QueryData
	//	dbcode: hgyd
	//	rowcode: zb
	//	colcode: sj
	//	wds: []
	//	dfwds: [{"wdcode":"zb","valuecode":"A010101"}]
	//	k1: 1678872233342
	//	h: 1
	time.Sleep(2 * time.Second)

	// 构建查询
	f := url.Values{}
	f.Add("m", "QueryData")
	f.Add("dbcode", "hgyd")
	f.Add("rowcode", "zb")
	f.Add("colcode", "sj")
	f.Add("wds", "[]")
	f.Add("dfwds", `[{"wdcode":"zb","valuecode":"A010101"}]`)
	f.Add("k1", fmt.Sprint(time.Now().UnixNano()/1e6))
	f.Add("h", "1")
	r, e := NationalHttpPost(NationalStatisticsBaseReqUrl, f.Encode())
	fmt.Println("err: ", e)
	if e != nil {
		fmt.Println("获取失败, Err: ", e.Error())
		//err = fmt.Errorf("请求指标分类树失败, Err: %s", e.Error())
		return
	}
	fmt.Println("res: ", string(r))
	return
}