temp commit

models/base_from_national_statistics_classify.go

@@ -0,0 +1,95 @@
+package models
+
+import (
+	"fmt"
+	"github.com/beego/beego/v2/client/orm"
+	"strings"
+	"time"
+)
+
+// BaseFromNationalStatisticsClassify 国统局-指标分类
+type BaseFromNationalStatisticsClassify struct {
+	BaseFromNationalStatisticsClassifyId int       `orm:"column(base_from_national_statistics_classify_id);pk"`
+	ClassifyName                         string    `description:"分类名称"`
+	Id                                   string    `description:"分类ID(字符串)"`
+	Dbcode                               string    `description:"dbcode"`
+	Wdcode                               string    `description:"wdcode"`
+	Pid                                  string    `description:"父级分类ID"`
+	IsParent                             int       `description:"是否存在子分类: 0-否; 1-是"`
+	CreateTime                           time.Time `description:"创建时间"`
+	ModifyTime                           time.Time `description:"更新时间"`
+}
+
+func (m *BaseFromNationalStatisticsClassify) TableName() string {
+	return "base_from_national_statistics_classify"
+}
+
+func (m *BaseFromNationalStatisticsClassify) Create() (err error) {
+	o := orm.NewOrmUsingDB("data")
+	id, err := o.Insert(m)
+	if err != nil {
+		return
+	}
+	m.BaseFromNationalStatisticsClassifyId = int(id)
+	return
+}
+
+func (m *BaseFromNationalStatisticsClassify) CreateMulti(items []*BaseFromNationalStatisticsClassify) (err error) {
+	if len(items) == 0 {
+		return
+	}
+	o := orm.NewOrmUsingDB("data")
+	_, err = o.InsertMulti(len(items), items)
+	return
+}
+
+func (m *BaseFromNationalStatisticsClassify) Update(cols []string) (err error) {
+	o := orm.NewOrmUsingDB("data")
+	_, err = o.Update(m, cols...)
+	return
+}
+
+func (m *BaseFromNationalStatisticsClassify) Del() (err error) {
+	o := orm.NewOrmUsingDB("data")
+	sql := `DELETE FROM base_from_national_statistics_classify WHERE base_from_national_statistics_classify_id = ? LIMIT 1`
+	_, err = o.Raw(sql, m.BaseFromNationalStatisticsClassifyId).Exec()
+	return
+}
+
+func (m *BaseFromNationalStatisticsClassify) GetItemById(id int) (err error) {
+	o := orm.NewOrmUsingDB("data")
+	sql := `SELECT * FROM base_from_national_statistics_classify WHERE base_from_national_statistics_classify_id = ? LIMIT 1`
+	err = o.Raw(sql, id).QueryRow(&m)
+	return
+}
+
+func (m *BaseFromNationalStatisticsClassify) GetItemByCondition(condition string, pars []interface{}) (err error) {
+	o := orm.NewOrmUsingDB("data")
+	sql := `SELECT * FROM base_from_national_statistics_classify WHERE 1=1 `
+	sql += condition
+	sql += ` LIMIT 1`
+	err = o.Raw(sql, pars).QueryRow(&m)
+	return
+}
+
+func (m *BaseFromNationalStatisticsClassify) GetCountByCondition(condition string, pars []interface{}) (count int, err error) {
+	o := orm.NewOrmUsingDB("data")
+	sql := fmt.Sprintf(`SELECT COUNT(1) FROM %s WHERE 1=1 %s`, m.TableName(), condition)
+	err = o.Raw(sql, pars).QueryRow(&count)
+	return
+}
+
+func (m *BaseFromNationalStatisticsClassify) GetItemsByCondition(condition string, pars []interface{}, fieldArr []string, orderRule string) (items []*BaseFromNationalStatisticsClassify, err error) {
+	o := orm.NewOrmUsingDB("data")
+	fields := strings.Join(fieldArr, ",")
+	if len(fieldArr) == 0 {
+		fields = `*`
+	}
+	order := `ORDER BY create_time DESC`
+	if orderRule != "" {
+		order = ` ORDER BY ` + orderRule
+	}
+	sql := fmt.Sprintf(`SELECT %s FROM %s WHERE 1=1 %s %s`, fields, m.TableName(), condition, order)
+	_, err = o.Raw(sql, pars).QueryRows(&items)
+	return
+}

models/db.go

@@ -49,5 +49,6 @@ func init() {
 		new(BaseFromEiaSteoIndex),
 		new(BaseFromEiaSteoData),
 		new(BaseFromEiaSteoClassify),
+		new(BaseFromNationalStatisticsClassify),
 	)
 }

services/national_data/national_data.go

@@ -2,10 +2,18 @@ package national_data
 
 import (
 	"crypto/tls"
+	"encoding/json"
 	"fmt"
+	"hongze/hongze_data_crawler/models"
 	"io/ioutil"
 	"net/http"
+	"net/url"
 	"strings"
+	"time"
+)
+
+const (
+	NationalStatisticsBaseReqUrl = "https://data.stats.gov.cn/easyquery.htm"
 )
 
 func NationalHttpPost(reqUrl, payload string) (result []byte, err error) {
@@ -84,3 +92,175 @@ func NationalGet(reqUrl, payload string) (err error) {
 	}
 	return
 }
+
+// QuotaClassifyTreeResp 指标分类树响应
+type QuotaClassifyTreeResp struct {
+	Id       string `description:"分类ID(字符串)"`
+	IsParent bool   `description:"是否为父级"`
+	Name     string `description:"分类名称"`
+	Pid      string `description:"父级分类ID"`
+	Dbcode   string `description:"源-dbocde"`
+	Wdcode   string `description:"源-wdcode"`
+}
+
+// SyncQuotaClassifyTree 同步指标分类树
+// dbcode: hgyd-月度; hgjd-季度; ...
+func SyncQuotaClassifyTree() (err error) {
+	defer func() {
+		if err != nil {
+			fmt.Println(err.Error())
+		}
+	}()
+
+	items := make([]*models.BaseFromNationalStatisticsClassify, 0)
+	initId := "zb"
+	initDbcode := "hgyd"
+	initWdcode := "zb"
+	resp, e := curlAndFormatQuotaClassify(initId, initDbcode, initWdcode, items)
+	if e != nil {
+		err = fmt.Errorf("递归指标分类失败, Err: %s", e.Error())
+		return
+	}
+	items = resp
+
+	// 去重
+	classifyMap := make(map[string]bool)
+	classifyOB := new(models.BaseFromNationalStatisticsClassify)
+	classifyPars := make([]interface{}, 0)
+	classifies, e := classifyOB.GetItemsByCondition("", classifyPars, []string{}, "")
+	if e != nil {
+		err = fmt.Errorf("获取指标分类列表失败, Err: %s", e.Error())
+		return
+	}
+	for _, c := range classifies {
+		classifyMap[c.Id] = true
+	}
+
+	finalList := make([]*models.BaseFromNationalStatisticsClassify, 0)
+	for _, v := range items {
+		if classifyMap[v.Id] {
+			continue
+		}
+		finalList = append(finalList, v)
+	}
+	if e = classifyOB.CreateMulti(items); e != nil {
+		err = fmt.Errorf("批量新增指标分类失败, Err: %s", e.Error())
+		return
+	}
+	return
+}
+
+// curlAndFormatQuotaClassify 递归请求分类树
+func curlAndFormatQuotaClassify(id, dbcode, wdcode string, items []*models.BaseFromNationalStatisticsClassify) (resp []*models.BaseFromNationalStatisticsClassify, err error) {
+	f := url.Values{}
+	f.Add("id", id)
+	f.Add("dbcode", dbcode)
+	f.Add("wdcode", wdcode)
+	f.Add("m", "getTree")
+	r, e := NationalHttpPost(NationalStatisticsBaseReqUrl, f.Encode())
+	if e != nil {
+		err = fmt.Errorf("请求指标分类树失败, Err: %s", e.Error())
+		return
+	}
+	list := make([]*QuotaClassifyTreeResp, 0)
+	if e = json.Unmarshal(r, &list); e != nil {
+		err = fmt.Errorf("解析响应数据失败, Err: %s", e.Error())
+		return
+	}
+
+	nowTime := time.Now().Local()
+	for _, v := range list {
+		isParent := 0
+		if v.IsParent {
+			isParent = 1
+		}
+		items = append(items, &models.BaseFromNationalStatisticsClassify{
+			ClassifyName: v.Name,
+			Id:           v.Id,
+			Dbcode:       v.Dbcode,
+			Wdcode:       v.Wdcode,
+			Pid:          v.Pid,
+			IsParent:     isParent,
+			CreateTime:   nowTime,
+			ModifyTime:   nowTime,
+		})
+
+		// 向下递归
+		if isParent == 1 {
+			time.Sleep(1 * time.Second) // 缓缓...毕竟接口是人家的...
+			items, e = curlAndFormatQuotaClassify(v.Id, v.Dbcode, v.Wdcode, items)
+			if e != nil {
+				err = fmt.Errorf("递归请求分类树失败, Err: %s", e.Error())
+				return
+			}
+		}
+	}
+	return items, nil
+}
+
+type QuotaListDataResp struct {
+	ReturnCode int `description:"状态码" json:"returncode"`
+	ReturnData struct {
+		DataNodes []QuotaDataNode `json:"datanodes"`
+		WdNodes   struct{}        `json:"wdnodes"`
+	}
+}
+
+type QuotaDataNode struct {
+	Code string `description:"编码"`
+	Data struct {
+		Data    float64 `description:"指标值"`
+		HasData bool    `description:"是否有值" json:"hasdata"`
+		StrData string  `description:"指标值(字符串)" json:"strdata"`
+	}
+	Wds []QuotaDataWds
+}
+
+type QuotaDataWds struct {
+	ValueCode string `json:"valuecode"`
+	WdCode    string `json:"wdcode"`
+}
+
+// SyncQuotaDataFromDbCodeAndId 同步指标值
+func SyncQuotaDataFromDbCodeAndId() (err error) {
+	// 查询无父级的指标分类
+	//classifyOB := new(models.BaseFromNationalStatisticsClassify)
+	//classifyCond := ` AND is_parent = 0`
+	//classifyPars := make([]interface{}, 0)
+	//classifyOrder := ` ORDER BY base_from_national_statistics_classify_id ASC`
+	//classifies, e := classifyOB.GetItemsByCondition(classifyCond, classifyPars, []string{}, classifyOrder)
+	//if e != nil {
+	//	err = fmt.Errorf("获取指标分类列表失败, Err: %s", e.Error())
+	//	return
+	//}
+
+	//	m: QueryData
+	//	dbcode: hgyd
+	//	rowcode: zb
+	//	colcode: sj
+	//	wds: []
+	//	dfwds: [{"wdcode":"zb","valuecode":"A010101"}]
+	//	k1: 1678872233342
+	//	h: 1
+	time.Sleep(2 * time.Second)
+
+	// 构建查询
+	f := url.Values{}
+	f.Add("m", "QueryData")
+	f.Add("dbcode", "hgyd")
+	f.Add("rowcode", "zb")
+	f.Add("colcode", "sj")
+	f.Add("wds", "[]")
+	f.Add("dfwds", `[{"wdcode":"zb","valuecode":"A010101"}]`)
+	f.Add("k1", fmt.Sprint(time.Now().UnixNano()/1e6))
+	f.Add("h", "1")
+	r, e := NationalHttpPost(NationalStatisticsBaseReqUrl, f.Encode())
+	fmt.Println("err: ", e)
+	if e != nil {
+		fmt.Println("获取失败, Err: ", e.Error())
+		//err = fmt.Errorf("请求指标分类树失败, Err: %s", e.Error())
+		return
+	}
+	fmt.Println("res: ", string(r))
+	return
+}

services/task.go

@@ -58,19 +58,23 @@ func RefreshChangesVisitorsCovid(cont context.Context) (err error) {
 	return
 }
 
-//func Task() {
-//	fmt.Println("start")
-//	reqUrl := "https://data.stats.gov.cn/easyquery.htm"
-//
-//	formData := url.Values{}
-//	formData.Add("id", "zb")
-//	formData.Add("dbcode", "hgyd")
-//	formData.Add("wdcode", "zb")
-//	formData.Add("m", "getTree")
-//
-//	formDataStr := formData.Encode()
-//	result, err := national_data.NationalHttpPost(reqUrl, formDataStr)
-//	fmt.Println(err)
-//	fmt.Println(string(result))
-//	fmt.Println("end")
-//}
+func Task() {
+	fmt.Println("start")
+
+	//_ = national_data.SyncQuotaClassifyTree()
+	//_ = national_data.SyncQuotaDataFromDbCodeAndId()
+
+	//reqUrl := "https://data.stats.gov.cn/easyquery.htm"
+
+	//formData := url.Values{}
+	//formData.Add("id", "zb")
+	//formData.Add("dbcode", "hgyd")
+	//formData.Add("wdcode", "zb")
+	//formData.Add("m", "getTree")
+	//
+	//formDataStr := formData.Encode()
+	//result, err := national_data.NationalHttpPost(reqUrl, formDataStr)
+	//fmt.Println(err)
+	//fmt.Println(string(result))
+	fmt.Println("end")
+}