Merge branch 'aj_national_data'

models/base_from_national_statistics_classify.go

@@ -0,0 +1,95 @@
+package models
+
+import (
+	"fmt"
+	"github.com/beego/beego/v2/client/orm"
+	"strings"
+	"time"
+)
+
+// BaseFromNationalStatisticsClassify 国统局-指标分类
+type BaseFromNationalStatisticsClassify struct {
+	BaseFromNationalStatisticsClassifyId int       `orm:"column(base_from_national_statistics_classify_id);pk"`
+	ClassifyName                         string    `description:"分类名称"`
+	Id                                   string    `description:"分类ID(字符串)"`
+	Dbcode                               string    `description:"dbcode"`
+	Wdcode                               string    `description:"wdcode"`
+	Pid                                  string    `description:"父级分类ID"`
+	IsParent                             int       `description:"是否存在子分类: 0-否; 1-是"`
+	CreateTime                           time.Time `description:"创建时间"`
+	ModifyTime                           time.Time `description:"更新时间"`
+}
+
+func (m *BaseFromNationalStatisticsClassify) TableName() string {
+	return "base_from_national_statistics_classify"
+}
+
+func (m *BaseFromNationalStatisticsClassify) Create() (err error) {
+	o := orm.NewOrmUsingDB("data")
+	id, err := o.Insert(m)
+	if err != nil {
+		return
+	}
+	m.BaseFromNationalStatisticsClassifyId = int(id)
+	return
+}
+
+func (m *BaseFromNationalStatisticsClassify) CreateMulti(items []*BaseFromNationalStatisticsClassify) (err error) {
+	if len(items) == 0 {
+		return
+	}
+	o := orm.NewOrmUsingDB("data")
+	_, err = o.InsertMulti(len(items), items)
+	return
+}
+
+func (m *BaseFromNationalStatisticsClassify) Update(cols []string) (err error) {
+	o := orm.NewOrmUsingDB("data")
+	_, err = o.Update(m, cols...)
+	return
+}
+
+func (m *BaseFromNationalStatisticsClassify) Del() (err error) {
+	o := orm.NewOrmUsingDB("data")
+	sql := `DELETE FROM base_from_national_statistics_classify WHERE base_from_national_statistics_classify_id = ? LIMIT 1`
+	_, err = o.Raw(sql, m.BaseFromNationalStatisticsClassifyId).Exec()
+	return
+}
+
+func (m *BaseFromNationalStatisticsClassify) GetItemById(id int) (err error) {
+	o := orm.NewOrmUsingDB("data")
+	sql := `SELECT * FROM base_from_national_statistics_classify WHERE base_from_national_statistics_classify_id = ? LIMIT 1`
+	err = o.Raw(sql, id).QueryRow(&m)
+	return
+}
+
+func (m *BaseFromNationalStatisticsClassify) GetItemByCondition(condition string, pars []interface{}) (err error) {
+	o := orm.NewOrmUsingDB("data")
+	sql := `SELECT * FROM base_from_national_statistics_classify WHERE 1=1 `
+	sql += condition
+	sql += ` LIMIT 1`
+	err = o.Raw(sql, pars).QueryRow(&m)
+	return
+}
+
+func (m *BaseFromNationalStatisticsClassify) GetCountByCondition(condition string, pars []interface{}) (count int, err error) {
+	o := orm.NewOrmUsingDB("data")
+	sql := fmt.Sprintf(`SELECT COUNT(1) FROM %s WHERE 1=1 %s`, m.TableName(), condition)
+	err = o.Raw(sql, pars).QueryRow(&count)
+	return
+}
+
+func (m *BaseFromNationalStatisticsClassify) GetItemsByCondition(condition string, pars []interface{}, fieldArr []string, orderRule string) (items []*BaseFromNationalStatisticsClassify, err error) {
+	o := orm.NewOrmUsingDB("data")
+	fields := strings.Join(fieldArr, ",")
+	if len(fieldArr) == 0 {
+		fields = `*`
+	}
+	order := `ORDER BY create_time DESC`
+	if orderRule != "" {
+		order = ` ORDER BY ` + orderRule
+	}
+	sql := fmt.Sprintf(`SELECT %s FROM %s WHERE 1=1 %s %s`, fields, m.TableName(), condition, order)
+	_, err = o.Raw(sql, pars).QueryRows(&items)
+	return
+}

models/base_from_national_statistics_data.go

@@ -0,0 +1,93 @@
+package models
+
+import (
+	"fmt"
+	"github.com/beego/beego/v2/client/orm"
+	"strings"
+	"time"
+)
+
+// BaseFromNationalStatisticsData 国统局-指标
+type BaseFromNationalStatisticsData struct {
+	BaseFromNationalStatisticsDataId  int       `orm:"column(base_from_national_statistics_data_id);pk"`
+	BaseFromNationalStatisticsIndexId int       `description:"指标ID"`
+	IndexCode                         string    `description:"指标编码"`
+	DataTime                          time.Time `description:"数据日期"`
+	Value                             float64   `description:"数据值"`
+	CreateTime                        time.Time `description:"创建时间"`
+	ModifyTime                        time.Time `description:"更新时间"`
+}
+
+func (m *BaseFromNationalStatisticsData) TableName() string {
+	return "base_from_national_statistics_data"
+}
+
+func (m *BaseFromNationalStatisticsData) Create() (err error) {
+	o := orm.NewOrmUsingDB("data")
+	id, err := o.Insert(m)
+	if err != nil {
+		return
+	}
+	m.BaseFromNationalStatisticsDataId = int(id)
+	return
+}
+
+func (m *BaseFromNationalStatisticsData) CreateMulti(items []*BaseFromNationalStatisticsData) (err error) {
+	if len(items) == 0 {
+		return
+	}
+	o := orm.NewOrmUsingDB("data")
+	_, err = o.InsertMulti(len(items), items)
+	return
+}
+
+func (m *BaseFromNationalStatisticsData) Update(cols []string) (err error) {
+	o := orm.NewOrmUsingDB("data")
+	_, err = o.Update(m, cols...)
+	return
+}
+
+func (m *BaseFromNationalStatisticsData) Del() (err error) {
+	o := orm.NewOrmUsingDB("data")
+	sql := `DELETE FROM base_from_national_statistics_data WHERE base_from_national_statistics_data_id = ? LIMIT 1`
+	_, err = o.Raw(sql, m.BaseFromNationalStatisticsDataId).Exec()
+	return
+}
+
+func (m *BaseFromNationalStatisticsData) GetItemById(id int) (err error) {
+	o := orm.NewOrmUsingDB("data")
+	sql := `SELECT * FROM base_from_national_statistics_data WHERE base_from_national_statistics_data_id = ? LIMIT 1`
+	err = o.Raw(sql, id).QueryRow(&m)
+	return
+}
+
+func (m *BaseFromNationalStatisticsData) GetItemByCondition(condition string, pars []interface{}) (err error) {
+	o := orm.NewOrmUsingDB("data")
+	sql := `SELECT * FROM base_from_national_statistics_data WHERE 1=1 `
+	sql += condition
+	sql += ` LIMIT 1`
+	err = o.Raw(sql, pars).QueryRow(&m)
+	return
+}
+
+func (m *BaseFromNationalStatisticsData) GetCountByCondition(condition string, pars []interface{}) (count int, err error) {
+	o := orm.NewOrmUsingDB("data")
+	sql := fmt.Sprintf(`SELECT COUNT(1) FROM %s WHERE 1=1 %s`, m.TableName(), condition)
+	err = o.Raw(sql, pars).QueryRow(&count)
+	return
+}
+
+func (m *BaseFromNationalStatisticsData) GetItemsByCondition(condition string, pars []interface{}, fieldArr []string, orderRule string) (items []*BaseFromNationalStatisticsData, err error) {
+	o := orm.NewOrmUsingDB("data")
+	fields := strings.Join(fieldArr, ",")
+	if len(fieldArr) == 0 {
+		fields = `*`
+	}
+	order := `ORDER BY create_time DESC`
+	if orderRule != "" {
+		order = ` ORDER BY ` + orderRule
+	}
+	sql := fmt.Sprintf(`SELECT %s FROM %s WHERE 1=1 %s %s`, fields, m.TableName(), condition, order)
+	_, err = o.Raw(sql, pars).QueryRows(&items)
+	return
+}

models/base_from_national_statistics_index.go

@@ -0,0 +1,200 @@
+package models
+
+import (
+	"fmt"
+	"github.com/beego/beego/v2/client/orm"
+	"strings"
+	"time"
+)
+
+// BaseFromNationalStatisticsIndex 国统局-指标
+type BaseFromNationalStatisticsIndex struct {
+	BaseFromNationalStatisticsIndexId    int       `orm:"column(base_from_national_statistics_index_id);pk"`
+	BaseFromNationalStatisticsClassifyId int       `description:"指标分类ID"`
+	Dbcode                               string    `description:"dbcode"`
+	IndexCode                            string    `description:"指标编码"`
+	IndexName                            string    `description:"指标名称"`
+	Unit                                 string    `description:"单位"`
+	Frequency                            string    `description:"频度"`
+	Reg                                  string    `description:"地区"`
+	StartDate                            time.Time `description:"开始日期"`
+	EndDate                              time.Time `description:"结束日期"`
+	CreateTime                           time.Time `description:"创建时间"`
+	ModifyTime                           time.Time `description:"更新时间"`
+}
+
+func (m *BaseFromNationalStatisticsIndex) TableName() string {
+	return "base_from_national_statistics_index"
+}
+
+func (m *BaseFromNationalStatisticsIndex) Create() (err error) {
+	o := orm.NewOrmUsingDB("data")
+	id, err := o.Insert(m)
+	if err != nil {
+		return
+	}
+	m.BaseFromNationalStatisticsIndexId = int(id)
+	return
+}
+
+func (m *BaseFromNationalStatisticsIndex) CreateMulti(items []*BaseFromNationalStatisticsIndex) (err error) {
+	if len(items) == 0 {
+		return
+	}
+	o := orm.NewOrmUsingDB("data")
+	_, err = o.InsertMulti(len(items), items)
+	return
+}
+
+func (m *BaseFromNationalStatisticsIndex) Update(cols []string) (err error) {
+	o := orm.NewOrmUsingDB("data")
+	_, err = o.Update(m, cols...)
+	return
+}
+
+func (m *BaseFromNationalStatisticsIndex) Del() (err error) {
+	o := orm.NewOrmUsingDB("data")
+	sql := `DELETE FROM base_from_national_statistics_index WHERE base_from_national_statistics_index_id = ? LIMIT 1`
+	_, err = o.Raw(sql, m.BaseFromNationalStatisticsIndexId).Exec()
+	return
+}
+
+func (m *BaseFromNationalStatisticsIndex) GetItemById(id int) (err error) {
+	o := orm.NewOrmUsingDB("data")
+	sql := `SELECT * FROM base_from_national_statistics_index WHERE base_from_national_statistics_index_id = ? LIMIT 1`
+	err = o.Raw(sql, id).QueryRow(&m)
+	return
+}
+
+func (m *BaseFromNationalStatisticsIndex) GetItemByCondition(condition string, pars []interface{}) (err error) {
+	o := orm.NewOrmUsingDB("data")
+	sql := `SELECT * FROM base_from_national_statistics_index WHERE 1=1 `
+	sql += condition
+	sql += ` LIMIT 1`
+	err = o.Raw(sql, pars).QueryRow(&m)
+	return
+}
+
+func (m *BaseFromNationalStatisticsIndex) GetCountByCondition(condition string, pars []interface{}) (count int, err error) {
+	o := orm.NewOrmUsingDB("data")
+	sql := fmt.Sprintf(`SELECT COUNT(1) FROM %s WHERE 1=1 %s`, m.TableName(), condition)
+	err = o.Raw(sql, pars).QueryRow(&count)
+	return
+}
+
+func (m *BaseFromNationalStatisticsIndex) GetItemsByCondition(condition string, pars []interface{}, fieldArr []string, orderRule string) (items []*BaseFromNationalStatisticsIndex, err error) {
+	o := orm.NewOrmUsingDB("data")
+	fields := strings.Join(fieldArr, ",")
+	if len(fieldArr) == 0 {
+		fields = `*`
+	}
+	order := `ORDER BY create_time DESC`
+	if orderRule != "" {
+		order = ` ORDER BY ` + orderRule
+	}
+	sql := fmt.Sprintf(`SELECT %s FROM %s WHERE 1=1 %s %s`, fields, m.TableName(), condition, order)
+	_, err = o.Raw(sql, pars).QueryRows(&items)
+	return
+}
+
+// SaveNationalStatisticsIndexAndDataReq 保存指标和数据请求体
+type SaveNationalStatisticsIndexAndDataReq struct {
+	Index      *BaseFromNationalStatisticsIndex  `description:"指标信息"`
+	IndexExist bool                              `description:"指标是否存在"`
+	DataList   []*BaseFromNationalStatisticsData `description:"新增的指标数据"`
+}
+
+// SaveNationalStatisticsIndexAndData 保存指标和值
+func SaveNationalStatisticsIndexAndData(req *SaveNationalStatisticsIndexAndDataReq) (err error) {
+	if req.Index == nil {
+		return
+	}
+	o := orm.NewOrmUsingDB("data")
+	tx, err := o.Begin()
+	if err != nil {
+		return
+	}
+	defer func() {
+		if err != nil {
+			_ = tx.Rollback()
+			return
+		}
+		_ = tx.Commit()
+	}()
+
+	if !req.IndexExist {
+		id, e := tx.Insert(req.Index)
+		if e != nil {
+			return e
+		}
+		req.Index.BaseFromNationalStatisticsIndexId = int(id)
+	}
+	indexId := req.Index.BaseFromNationalStatisticsIndexId
+	if req.DataList != nil && len(req.DataList) > 0 {
+		for _, d := range req.DataList {
+			d.BaseFromNationalStatisticsIndexId = indexId
+		}
+		_, e := tx.InsertMulti(len(req.DataList), req.DataList)
+		if e != nil {
+			return e
+		}
+	}
+	return
+}
+
+// BatchSaveNationalStatisticsIndexAndData 批量保存指标和值
+func BatchSaveNationalStatisticsIndexAndData(indexArr []*BaseFromNationalStatisticsIndex, indexDataMap map[string][]*BaseFromNationalStatisticsData) (err error) {
+	o := orm.NewOrmUsingDB("data")
+	tx, err := o.Begin()
+	if err != nil {
+		return
+	}
+	defer func() {
+		if err != nil {
+			_ = tx.Rollback()
+			return
+		}
+		_ = tx.Commit()
+	}()
+
+	// 指标
+	for _, v := range indexArr {
+		id, e := tx.Insert(v)
+		if e != nil {
+			return e
+		}
+		indexId := int(id)
+
+		// 数据
+		dataList := indexDataMap[v.IndexCode]
+		if dataList != nil && len(dataList) > 0 {
+			for _, d := range dataList {
+				d.BaseFromNationalStatisticsIndexId = indexId
+			}
+			_, e = tx.InsertMulti(len(dataList), dataList)
+			if e != nil {
+				return e
+			}
+		}
+	}
+	return
+}
+
+// UpdateNationalStatisticsIndexStartEndDate 更新指标开始结束日期
+func UpdateNationalStatisticsIndexStartEndDate() (err error) {
+	o := orm.NewOrmUsingDB("data")
+	sql := `UPDATE base_from_national_statistics_index AS a
+			JOIN (
+				SELECT
+					index_code,
+					MIN(data_time) AS min_time,
+					MAX(data_time) AS max_time
+				FROM
+					base_from_national_statistics_data
+				GROUP BY
+					index_code
+			) AS b ON a.index_code = b.index_code
+			SET a.start_date = b.min_time, a.end_date = b.max_time`
+	_, err = o.Raw(sql).Exec()
+	return
+}

models/db.go

@@ -49,5 +49,8 @@ func init() {
 		new(BaseFromEiaSteoIndex),
 		new(BaseFromEiaSteoData),
 		new(BaseFromEiaSteoClassify),
+		new(BaseFromNationalStatisticsClassify),
+		new(BaseFromNationalStatisticsIndex),
+		new(BaseFromNationalStatisticsData),
 	)
 }

services/national_data/classify.go

@@ -0,0 +1,142 @@
+package national_data
+
+import (
+	"encoding/json"
+	"fmt"
+	"hongze/hongze_data_crawler/models"
+	"hongze/hongze_data_crawler/services/alarm_msg"
+	"hongze/hongze_data_crawler/utils"
+	"net/url"
+	"time"
+)
+
+// QuotaClassifyTreeResp 指标分类树响应
+type QuotaClassifyTreeResp struct {
+	Id       string `description:"分类ID(字符串)"`
+	IsParent bool   `description:"是否为父级"`
+	Name     string `description:"分类名称"`
+	Pid      string `description:"父级分类ID"`
+	Dbcode   string `description:"源-dbocde"`
+	Wdcode   string `description:"源-wdcode"`
+}
+
+// SyncQuotaClassifyTree 同步指标分类树
+func SyncQuotaClassifyTree() (err error) {
+	defer func() {
+		if err != nil {
+			utils.FileLog.Error("统计局-同步指标分类树失败, ErrMsg: %s", err.Error())
+			go alarm_msg.SendAlarmMsg(fmt.Sprintf("统计局-同步指标分类树失败, ErrMsg: %s", err.Error()), 3)
+		}
+	}()
+
+	// 含指标维度
+	dbCodes := []string{
+		"hgyd", "hgjd", "hgnd", // 月度/季度/年度
+		"fsyd", "fsjd", "fsnd", "csyd", "csnd", "gatyd", "gatnd", // 分省月度/分省季度/分省年度/城市月度价格/城市年度数据/港澳台月度/港澳台年度
+		"gjyd", "gjydsdj", "gjydsc", "gjnd", // 主要国家月度/三大经济体月度/国际市场月度商品/国家年度
+	}
+	// 含地区维度
+	regCodes := []string{
+		"fsyd", "fsjd", "fsnd", "csyd", "csnd", "gatyd", "gatnd", // 分省月度/分省季度/分省年度/城市月度价格/城市年度数据/港澳台月度/港澳台年度
+		"gjyd", "gjydsdj", "gjydsc", "gjnd", // 主要国家月度/三大经济体月度/国际市场月度商品/国家年度
+	}
+
+	for _, code := range dbCodes {
+		fmt.Println("开始同步DbCode: ", code)
+
+		// 指标维度
+		items := make([]*models.BaseFromNationalStatisticsClassify, 0)
+		resp, e := curlAndFormatQuotaClassify("zb", code, "zb", items)
+		if e != nil {
+			err = fmt.Errorf("递归指标分类失败, Err: %s", e.Error())
+			return
+		}
+		items = resp
+
+		// 地区维度
+		if utils.InArrayByStr(regCodes, code) {
+			regResp, e := curlAndFormatQuotaClassify("reg", code, "reg", items)
+			if e != nil {
+				err = fmt.Errorf("递归地区分类失败, Err: %s", e.Error())
+				return
+			}
+			items = append(items, regResp...)
+		}
+
+		// 去重-code+维度code+ID
+		classifyMap := make(map[string]bool)
+		classifyOB := new(models.BaseFromNationalStatisticsClassify)
+		classifyPars := make([]interface{}, 0)
+		classifies, e := classifyOB.GetItemsByCondition("", classifyPars, []string{}, "")
+		if e != nil {
+			err = fmt.Errorf("获取指标分类列表失败, Err: %s", e.Error())
+			return
+		}
+		for _, c := range classifies {
+			classifyMap[fmt.Sprintf("%s%s%s", code, c.Wdcode, c.Id)] = true
+		}
+
+		finalList := make([]*models.BaseFromNationalStatisticsClassify, 0)
+		for _, v := range items {
+			if classifyMap[fmt.Sprintf("%s%s%s", code, v.Wdcode, v.Id)] {
+				continue
+			}
+			finalList = append(finalList, v)
+		}
+		if e = classifyOB.CreateMulti(items); e != nil {
+			err = fmt.Errorf("批量新增指标分类失败, Err: %s", e.Error())
+			return
+		}
+		fmt.Println("结束同步DbCode: ", code)
+	}
+	return
+}
+
+// curlAndFormatQuotaClassify 递归请求分类树
+func curlAndFormatQuotaClassify(id, dbcode, wdcode string, items []*models.BaseFromNationalStatisticsClassify) (resp []*models.BaseFromNationalStatisticsClassify, err error) {
+	f := url.Values{}
+	f.Add("id", id)
+	f.Add("dbcode", dbcode)
+	f.Add("wdcode", wdcode)
+	f.Add("m", "getTree")
+	r, e := NationalHttpPost(NationalStatisticsBaseReqUrl, f.Encode())
+	if e != nil {
+		err = fmt.Errorf("请求指标分类树失败, Err: %s", e.Error())
+		return
+	}
+	list := make([]*QuotaClassifyTreeResp, 0)
+	if e = json.Unmarshal(r, &list); e != nil {
+		err = fmt.Errorf("解析响应数据失败, Err: %s", e.Error())
+		return
+	}
+
+	nowTime := time.Now().Local()
+	for _, v := range list {
+		isParent := 0
+		if v.IsParent {
+			isParent = 1
+		}
+		fmt.Printf("Dbcode: %s, ClassifyName: %s, Id: %s\n", v.Dbcode, v.Name, v.Id)
+		items = append(items, &models.BaseFromNationalStatisticsClassify{
+			ClassifyName: v.Name,
+			Id:           v.Id,
+			Dbcode:       v.Dbcode,
+			Wdcode:       v.Wdcode,
+			Pid:          v.Pid,
+			IsParent:     isParent,
+			CreateTime:   nowTime,
+			ModifyTime:   nowTime,
+		})
+
+		// 向下递归
+		if isParent == 1 {
+			time.Sleep(5 * time.Second) // 缓缓...毕竟接口是人家的...
+			items, e = curlAndFormatQuotaClassify(v.Id, v.Dbcode, v.Wdcode, items)
+			if e != nil {
+				err = fmt.Errorf("递归请求分类树失败, Err: %s", e.Error())
+				return
+			}
+		}
+	}
+	return items, nil
+}

services/national_data/common.go

@@ -0,0 +1,394 @@
+package national_data
+
+import (
+	"bytes"
+	"crypto/tls"
+	"encoding/json"
+	"fmt"
+	"hongze/hongze_data_crawler/utils"
+	"io"
+	"io/ioutil"
+	"net/http"
+	"net/url"
+	"strings"
+	"time"
+)
+
+const (
+	NationalStatisticsBaseReqUrl = "https://data.stats.gov.cn/easyquery.htm"
+)
+
+func NationalHttpPost(reqUrl, payload string) (result []byte, err error) {
+	// 随机延迟执行
+	r := utils.RangeRand(5000, 8000)
+	//fmt.Printf("随机延迟%d\n", r)
+	if r > 6100 && r < 6250 {
+		time.Sleep(15 * time.Second)
+	} else {
+		time.Sleep(time.Duration(r) * time.Millisecond)
+	}
+
+	tr := &http.Transport{
+		TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
+	}
+	client := &http.Client{
+		Transport: tr,
+	}
+	req, err := http.NewRequest("POST", reqUrl, strings.NewReader(payload))
+	if err != nil {
+		return
+	}
+	req.Header.Add("Accept", "text/plain, */*; q=0.01")
+	req.Header.Add("Accept-Encoding", "tgzip, deflate, br")
+	req.Header.Add("Accept-Language", "zh-CN,zh;q=0.9")
+	req.Header.Add("Connection", "keep-alive")
+	req.Header.Add("Content-Length", "37")
+	req.Header.Add("Content-Type", "application/x-www-form-urlencoded")
+	req.Header.Add("Cookie", "wzws_sessionid=gDExNS4xOTQuMTAyLjEyN6BkERzUgmZjNWVlMYFiOWNiZDg=; JSESSIONID=UOri2Cu3f3c-Y3rPgXWJ04E8pfbeyAUGG-s7zJ7Tt0JhlEiLi0EU!412929168; u=5")
+	req.Header.Add("Host", "data.stats.gov.cn")
+	req.Header.Add("Origin", "https://data.stats.gov.cn")
+	req.Header.Set("Referer", "https://data.stats.gov.cn/easyquery.htm?cn=A01")
+	req.Header.Set("sec-ch-ua", "\"Not_A Brand\";v=\"99\", \"Google Chrome\";v=\"109\", \"Chromium\";v=\"109\"")
+	req.Header.Set("sec-ch-ua-mobile", "?0")
+	req.Header.Set("sec-ch-ua-platform", "\"Windows\"")
+	req.Header.Set("Sec-Fetch-Dest", "empty")
+	req.Header.Set("Sec-Fetch-Mode", "cors")
+	req.Header.Set("Sec-Fetch-Site", "same-origin")
+	req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/109.0.0.0 Safari/537.36")
+	req.Header.Set("X-Requested-With", "XMLHttpRequest")
+	res, err := client.Do(req)
+	if err != nil {
+		return
+	}
+	defer func() {
+		_ = res.Body.Close()
+	}()
+	// 此处用io.Copy替代ioutil.ReadAll方法避免数据过大读取不完整
+	var b []byte
+	buf := bytes.NewBuffer(b)
+	_, err = io.Copy(buf, res.Body)
+	if err != nil {
+		return
+	}
+	result = buf.Bytes()
+	return
+}
+
+func NationalGet(reqUrl, payload string) (err error) {
+	tr := &http.Transport{
+		TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
+	}
+	client := &http.Client{
+		Transport: tr,
+	}
+	req, err := http.NewRequest("GET", reqUrl, strings.NewReader(payload))
+	if err != nil {
+		return
+	}
+	res, err := client.Do(req)
+	if err != nil {
+		return
+	}
+	defer res.Body.Close()
+	_, err = ioutil.ReadAll(res.Body)
+	if err != nil {
+		return
+	}
+
+	Cookie := res.Header.Get("Cookie")
+	fmt.Println(Cookie)
+	rcookie := req.Header.Get("Cookie")
+	fmt.Println("rcookie")
+	fmt.Println(rcookie)
+	//fmt.Println("body:" + string(body))
+	cookiesArr := res.Cookies()
+	fmt.Println("cookiesArrLen:", len(cookiesArr))
+	for k, v := range cookiesArr {
+		fmt.Println(k, v)
+	}
+	return
+}
+
+// DataApiReq 数据接口请求体
+type DataApiReq struct {
+	Method    string `description:"方法: QueryData-查询数据; getOtherWds-获取其他维度" json:"method"`
+	DbCode    string `description:"数据库编码" json:"dbcode"`
+	RowCode   string `description:"行-维度: zb; sj; reg" json:"rowcode"`
+	ColCode   string `description:"列-维度: zb; sj; reg" json:"colcode"`
+	WdsList   []Wds  `description:"维度列表" json:"wdsList"`
+	DfwdsList []Wds  `description:"df不知道啥意思...反正也是维度相关的" json:"dfwdsList"`
+}
+
+// Wds 维度
+type Wds struct {
+	WdCode    string `description:"维度: zb-指标; sj-时间; reg-地区" json:"wdcode"`
+	ValueCode string `description:"维度编码" json:"valuecode"`
+}
+
+// CommonDataApiRequest 数据接口请求
+func CommonDataApiRequest(req DataApiReq) (resp QuotaListDataResp, err error) {
+	var b []byte
+	defer func() {
+		if err != nil {
+			r, _ := json.Marshal(req)
+			utils.FileLog.Error("CommonDataApiRequest Err request: %s", r)
+			utils.FileLog.Info("CommonDataApiRequest Err result: %s", string(b))
+		}
+	}()
+	if req.DbCode == "" {
+		return
+	}
+	if req.Method == "" {
+		req.Method = "QueryData"
+	}
+	if req.RowCode == "" {
+		req.RowCode = "zb"
+	}
+	if req.ColCode == "" {
+		req.ColCode = "sj"
+	}
+
+	// 构建查询
+	f := url.Values{}
+	f.Add("m", req.Method)
+	f.Add("dbcode", req.DbCode)
+	f.Add("rowcode", req.RowCode)
+	f.Add("colcode", req.ColCode)
+	wds := `[]`
+	if len(req.WdsList) > 0 {
+		wdsByte, e := json.Marshal(req.WdsList)
+		if e != nil {
+			err = fmt.Errorf("wds json marshal err: %s", e.Error())
+			return
+		}
+		wds = string(wdsByte)
+	}
+	dfwds := `[]`
+	if len(req.DfwdsList) > 0 {
+		dfwdsByte, e := json.Marshal(req.DfwdsList)
+		if e != nil {
+			err = fmt.Errorf("dfwds json marshal err: %s", e.Error())
+			return
+		}
+		dfwds = string(dfwdsByte)
+	}
+	f.Add("wds", wds)
+	f.Add("dfwds", dfwds)
+	f.Add("k1", fmt.Sprint(time.Now().UnixNano()/1e6))
+	f.Add("h", "1")
+
+	// 响应
+	b, e := NationalHttpPost(NationalStatisticsBaseReqUrl, f.Encode())
+	if e != nil {
+		err = fmt.Errorf("http request err: %s", e.Error())
+		return
+	}
+	if len(b) == 0 {
+		err = fmt.Errorf("http result empty")
+		return
+	}
+	if e = json.Unmarshal(b, &resp); e != nil {
+		err = fmt.Errorf("resp unmarshal err: %s", e.Error())
+		return
+	}
+	if resp.ReturnCode != 200 {
+		err = fmt.Errorf("resp code err: %d", resp.ReturnCode)
+		return
+	}
+	return
+}
+
+// QuotaListDataResp 指标数据列表响应体
+type QuotaListDataResp struct {
+	ReturnCode int `description:"状态码" json:"returncode"`
+	ReturnData struct {
+		DataNodes []QuotaDataNode `json:"datanodes"`
+		WdNodes   []QuotaWdNode   `json:"wdnodes"`
+	}
+}
+
+// QuotaDataNode 指标数据节点
+type QuotaDataNode struct {
+	Code string `description:"编码"`
+	Data struct {
+		Data    float64 `description:"指标值"`
+		HasData bool    `description:"是否有值" json:"hasdata"`
+		StrData string  `description:"指标值(字符串)" json:"strdata"`
+	}
+	Wds []Wds
+}
+
+// QuotaWdNode 维度节点
+type QuotaWdNode struct {
+	WdCode string `description:"示例: zb; sj; reg;" json:"wdcode"`
+	WdName string `description:"示例: 指标; 时间; 地区" json:"wdname"`
+	Nodes  []QuotaWdNodeData
+}
+
+// QuotaWdNodeData 维度节点数据
+type QuotaWdNodeData struct {
+	Code     string `description:"指标编码"`
+	Name     string `description:"指标名称"`
+	Unit     string `description:"单位"`
+	SortCode int    `description:"编码排序" json:"sortcode"`
+}
+
+// OtherWdResp 其他维度信息响应体
+type OtherWdResp struct {
+	ReturnCode int           `description:"状态码" json:"returncode"`
+	ReturnData []OtherWdData `description:"响应数据" json:"returndata"`
+}
+
+// OtherWdData 其他维度数据
+type OtherWdData struct {
+	IsSj   bool           `description:"是否为时间" json:"issj"`
+	WdCode string         `description:"维度编码" json:"wdcode"`
+	WdName string         `description:"维度名称" json:"wdname"`
+	Nodes  []OtherWdNodes `description:"维度数据" json:"nodes"`
+}
+
+type OtherWdNodes struct {
+	Code string `description:"编码" json:"code"`
+	Name string `description:"名称" json:"name"`
+	Sort string `description:"排序" json:"sort"`
+}
+
+// formatMonth2YearDateCode 将日期code转为对应日期
+func formatMonth2YearDateCode(dateCode string) (date time.Time, err error) {
+	if dateCode == "" {
+		return
+	}
+	// 根据日期code长度进行区分, 格式为三种: 月度-200601; 季度-2006A; 年度-2006
+	switch len([]rune(dateCode)) {
+	case 6:
+		t, e := time.ParseInLocation("200601", dateCode, time.Local)
+		if e != nil {
+			err = fmt.Errorf("月度指标日期转换失败, Err: %s", e.Error())
+			return
+		}
+		date = t
+		break
+	case 5:
+		// 季度ABCD转换成对应日期
+		dateSuffixMap := map[string]string{
+			"A": "03-31",
+			"B": "06-30",
+			"C": "09-30",
+			"D": "12-31",
+		}
+		dateCode = strings.ToUpper(dateCode)
+		quarterTab := dateCode[4:]
+		dateStr := fmt.Sprintf("%s-%s", dateCode[:4], dateSuffixMap[quarterTab])
+		t, e := time.ParseInLocation(utils.FormatDate, dateStr, time.Local)
+		if e != nil {
+			err = fmt.Errorf("季度指标日期转换失败, Err: %s", e.Error())
+			return
+		}
+		date = t
+		break
+	case 4:
+		dateStr := fmt.Sprintf("%s-%s", dateCode, "12-31")
+		t, e := time.ParseInLocation(utils.FormatDate, dateStr, time.Local)
+		if e != nil {
+			err = fmt.Errorf("年度指标日期转换失败, Err: %s", e.Error())
+			return
+		}
+		date = t
+		break
+	default:
+		err = fmt.Errorf("日期code格式有误, code: %s", dateCode)
+		return
+	}
+	return
+}
+
+// GetOtherWd 获取Db下其他维度信息
+func GetOtherWd(dbCode, rowCode, colCode string) (wdList []OtherWdData, err error) {
+	if dbCode == "" {
+		return
+	}
+	if rowCode == "" {
+		rowCode = "zb"
+	}
+	if colCode == "" {
+		colCode = "sj"
+	}
+
+	// 构建查询
+	f := url.Values{}
+	f.Add("m", "getOtherWds")
+	f.Add("dbcode", dbCode)
+	f.Add("rowcode", rowCode)
+	f.Add("colcode", colCode)
+	f.Add("wds", `[]`)
+	f.Add("k1", fmt.Sprint(time.Now().UnixNano()/1e6))
+	f.Add("h", "1")
+	r, e := NationalHttpPost(NationalStatisticsBaseReqUrl, f.Encode())
+	if e != nil {
+		err = fmt.Errorf("请求其他维度信息失败, Err: %s", e.Error())
+		return
+	}
+	utils.FileLog.Info("GetOtherWdInfo Result: %s", string(r))
+
+	// 响应
+	resp := new(OtherWdResp)
+	if e = json.Unmarshal(r, &resp); e != nil {
+		err = fmt.Errorf("其他维度信息Unmarshal Err: %s", e.Error())
+		return
+	}
+	if resp == nil {
+		err = fmt.Errorf("其他维度信息请求结果为空")
+		return
+	}
+	if resp.ReturnCode != 200 {
+		err = fmt.Errorf("其他维度信息请求有误, Code: %d", resp.ReturnCode)
+		return
+	}
+	wdList = resp.ReturnData
+	return
+}
+
+func ApiTest() (err error) {
+	//f := url.Values{}
+	//f.Add("m", "QueryData")
+	//f.Add("dbcode", "fsyd")
+	//f.Add("rowcode", "zb")
+	//f.Add("colcode", "sj")
+	//f.Add("wds", `[{"wdcode":"reg","valuecode":"000"}]`)
+	//f.Add("dfwds", `[{"wdcode":"zb","valuecode":"A01"}]`)
+	f := url.Values{}
+	f.Add("m", "QueryData")
+	f.Add("dbcode", "fsjd")
+	f.Add("rowcode", "zb")
+	f.Add("colcode", "sj")
+	//f.Add("wds", `[{"wdcode":"reg","valuecode":"110000"}]`)
+	f.Add("wds", `[{"wdcode":"reg","valuecode":"310000"}]`)
+	f.Add("dfwds", `[{"wdcode":"zb","valuecode":"A0501"},{"wdcode":"sj","valuecode":"LAST18"}]`)
+	f.Add("k1", fmt.Sprint(time.Now().UnixNano()/1e6))
+	f.Add("h", "1")
+
+	//f := url.Values{}
+	//f.Add("m", "QueryData")
+	//f.Add("dbcode", "gatyd")
+	//f.Add("rowcode", "sj")
+	//f.Add("colcode", "reg")
+	//f.Add("wds", `[{"wdcode":"zb","valuecode":"A010A"}]`)
+	//f.Add("dfwds", `[{"wdcode":"sj","valuecode":"LAST36"}]`)
+	//f := url.Values{}
+	//f.Add("m", "QueryData")
+	//f.Add("dbcode", "fsyd")
+	//f.Add("rowcode", "zb")
+	//f.Add("colcode", "sj")
+	//f.Add("wds", `[{"wdcode":"reg","valuecode":"000"}]`)
+	//f.Add("dfwds", `[{"wdcode":"zb","valuecode":"A01"}]`)
+	//f.Add("k1", fmt.Sprint(time.Now().UnixNano()/1e6))
+	//f.Add("h", "1")
+
+	r, e := NationalHttpPost(NationalStatisticsBaseReqUrl, f.Encode())
+	if e != nil {
+		fmt.Println("请求失败, Err: ", e.Error())
+		return
+	}
+	utils.FileLog.Info("test result: %s", string(r))
+	return
+}

services/national_data/national_data.go

@@ -0,0 +1,856 @@
+package national_data
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"hongze/hongze_data_crawler/models"
+	"hongze/hongze_data_crawler/services/alarm_msg"
+	"hongze/hongze_data_crawler/utils"
+	"strings"
+	"time"
+)
+
+// 每个库大概同步时间
+// hgyd-4h fsyd-9h csyd-3h gatyd-1h gjyd、gjydsdj、gjydsc-15min
+// hgjd-4h fsjd-4h
+// hgnd-9h fsnd-2day csnd、gatnd、gjnd-1.5h
+
+// RefreshNationalDbs 刷新统计局数据(所有)
+func RefreshNationalDbs(cont context.Context) (err error) {
+	utils.FileLog.Info("开始刷新统计局数据")
+
+	_ = SyncXDateYQuotaDb([]string{})
+
+	_ = SyncXDateYQuotaZRegDb([]string{})
+
+	_ = SyncXRegYDateZQuotaDb([]string{})
+
+	// 最后更新一下每个指标的开始结束日期
+	if e := models.UpdateNationalStatisticsIndexStartEndDate(); e != nil {
+		alarm_msg.SendAlarmMsg(fmt.Sprintf("统计局-更新指标开始结束日期失败, ErrMsg: %s", e.Error()), 3)
+	}
+
+	utils.FileLog.Info("统计局数据刷新成功")
+	return
+}
+
+// RefreshNationalMonthDbA 刷新月度指标库
+func RefreshNationalMonthDbA(cont context.Context) (err error) {
+	utils.FileLog.Info("统计局-开始同步月度指标库A")
+	if err = SelectSyncFunc([]string{"hgyd", "csyd", "gatyd", "gjyd", "gjydsdj", "gjydsc"}); err != nil {
+		utils.FileLog.Info("统计局-同步月度指标库A失败")
+		return
+	}
+	utils.FileLog.Info("统计局-同步月度指标库A成功")
+	return
+}
+
+// RefreshNationalMonthDbB 刷新月度指标库(分省月度)
+func RefreshNationalMonthDbB(cont context.Context) (err error) {
+	utils.FileLog.Info("统计局-开始同步月度指标库B")
+	if err = SelectSyncFunc([]string{"fsyd"}); err != nil {
+		utils.FileLog.Info("统计局-同步月度指标库B失败")
+		return
+	}
+	utils.FileLog.Info("统计局-同步月度指标库B成功")
+	return
+}
+
+// RefreshNationalQuarterDb 刷新季度指标库
+func RefreshNationalQuarterDb(cont context.Context) (err error) {
+	utils.FileLog.Info("统计局-开始同步季度指标库")
+	if err = SelectSyncFunc([]string{"hgjd", "fsjd"}); err != nil {
+		utils.FileLog.Info("统计局-同步季度指标库失败")
+		return
+	}
+	utils.FileLog.Info("统计局-同步季度指标库成功")
+	return
+}
+
+// RefreshNationalYearDbA 刷新年度指标库
+func RefreshNationalYearDbA(cont context.Context) (err error) {
+	utils.FileLog.Info("统计局-开始同步年度指标库A")
+	if err = SelectSyncFunc([]string{"hgnd", "csnd", "gatnd", "gjnd"}); err != nil {
+		utils.FileLog.Info("统计局-同步年度指标库A失败")
+		return
+	}
+	utils.FileLog.Info("统计局-同步年度指标库A成功")
+	return
+}
+
+// RefreshNationalYearDbB 刷新年度指标库(分省年度数据)
+func RefreshNationalYearDbB(cont context.Context) (err error) {
+	utils.FileLog.Info("统计局-开始同步年度指标库B")
+	if err = SelectSyncFunc([]string{"fsnd"}); err != nil {
+		utils.FileLog.Info("统计局-同步年度指标库B失败")
+		return
+	}
+	utils.FileLog.Info("统计局-同步年度指标库B成功")
+	return
+}
+
+func SelectSyncFunc(dbs []string) (err error) {
+	funcA := []string{"hgyd", "hgjd", "hgnd"}
+	funcB := []string{"fsyd", "fsjd", "fsnd", "csyd", "csnd", "gjydsc"}
+	funcC := []string{"gatyd", "gatnd", "gjyd", "gjydsdj", "gjnd"}
+	// 此处要根据不同的指标库选择同步方式
+	for _, q := range dbs {
+		if utils.InArrayByStr(funcA, q) {
+			if err = SyncXDateYQuotaDb([]string{q}); err != nil {
+				return
+			}
+			continue
+		}
+		if utils.InArrayByStr(funcB, q) {
+			if err = SyncXDateYQuotaZRegDb([]string{q}); err != nil {
+				return
+			}
+			continue
+		}
+		if utils.InArrayByStr(funcC, q) {
+			if err = SyncXRegYDateZQuotaDb([]string{q}); err != nil {
+				return
+			}
+		}
+	}
+	// 最后更新一下每个指标的开始结束日期
+	if e := models.UpdateNationalStatisticsIndexStartEndDate(); e != nil {
+		alarm_msg.SendAlarmMsg(fmt.Sprintf("统计局-更新指标开始结束日期失败, ErrMsg: %s", e.Error()), 3)
+	}
+	return
+}
+
+// SyncXDateYQuotaDb 同步两维度X轴-日期, Y轴-指标数据库(月度/季度/年度数据指标)
+func SyncXDateYQuotaDb(dbs []string) (err error) {
+	if len(dbs) == 0 {
+		dbs = []string{"hgyd", "hgjd", "hgnd"}
+	}
+	defer func() {
+		d := strings.Join(dbs, ",")
+		if err != nil {
+			utils.FileLog.Error("统计局-同步%s数据库失败, ErrMsg: %s", d, err.Error())
+			go alarm_msg.SendAlarmMsg(fmt.Sprintf("统计局-同步%s数据库失败, ErrMsg: %s", d, err.Error()), 3)
+			return
+		}
+		utils.FileLog.Info("统计局-同步%s数据库成功", d)
+	}()
+
+	// 查询无父级的指标分类
+	for _, d := range dbs {
+		classifyOB := new(models.BaseFromNationalStatisticsClassify)
+		classifyCond := ` AND is_parent = 0 AND dbcode = ?`
+		classifyPars := make([]interface{}, 0)
+		classifyPars = append(classifyPars, d)
+		classifyOrder := ` base_from_national_statistics_classify_id ASC`
+		classifyList, e := classifyOB.GetItemsByCondition(classifyCond, classifyPars, []string{}, classifyOrder)
+		if e != nil {
+			err = fmt.Errorf("获取指标分类列表失败, Err: %s", e.Error())
+			return
+		}
+		utils.FileLog.Info("%s分类长度: %d\n", d, len(classifyList))
+
+		// 同步指标和数据
+		for _, c := range classifyList {
+			utils.FileLog.Info("开始同步分类-%d: %s", c.BaseFromNationalStatisticsClassifyId, c.ClassifyName)
+			if e = SyncXDateYQuotaData(c.BaseFromNationalStatisticsClassifyId, c.Dbcode, c.Id); e != nil {
+				err = fmt.Errorf("同步指标数据失败, DbCode: %s, ClassifyId: %d, Err: %s", c.Dbcode, c.BaseFromNationalStatisticsClassifyId, e.Error())
+				return
+			}
+			utils.FileLog.Info("结束同步分类-%d: %s", c.BaseFromNationalStatisticsClassifyId, c.ClassifyName)
+		}
+	}
+	return
+}
+
+// SyncXDateYQuotaData 同步两维度X轴-日期, Y轴-指标的数据
+func SyncXDateYQuotaData(classifyId int, dbCode, classifyCode string) (err error) {
+	defer func() {
+		if err != nil {
+			utils.FileLog.Error("同步数据库DbCode: %s, 分类ClassifyId: %d失败, ErrMsg: %s", dbCode, classifyId, err.Error())
+		}
+	}()
+	// yd-月度 jd-季度 nd-年度
+	frequency := ""
+	timeParam := ""
+	if strings.Contains(dbCode, "yd") {
+		timeParam = "LAST36" // 最近36个月
+		frequency = "月度"
+	}
+	if strings.Contains(dbCode, "jd") {
+		timeParam = "LAST18" // 最近18个季度
+		frequency = "季度"
+	}
+	if strings.Contains(dbCode, "nd") {
+		timeParam = "LAST20" // 最近20年
+		frequency = "年度"
+	}
+
+	var dataReq DataApiReq
+	dataReq.DbCode = dbCode
+	dataReq.DfwdsList = append(dataReq.DfwdsList, Wds{
+		WdCode:    "zb",
+		ValueCode: classifyCode,
+	}, Wds{
+		WdCode:    "sj",
+		ValueCode: timeParam,
+	})
+	attempt := 0
+	resp, e := CommonDataApiRequest(dataReq)
+	if e != nil {
+		//if !strings.Contains(e.Error(), "connection attempt failed") {
+		//	err = fmt.Errorf("查询数据失败, Err: %s", e.Error())
+		//	return
+		//}
+		// 连接失败重新尝试3次
+		for {
+			time.Sleep(2 * time.Minute)
+			attempt += 1
+			utils.FileLog.Info("当前第%d次重新请求", attempt)
+			resp, e = CommonDataApiRequest(dataReq)
+			if e == nil {
+				break
+			}
+			if attempt >= 3 {
+				s, _ := json.Marshal(dataReq)
+				err = fmt.Errorf("查询数据重试失败, DataReq: %s", s)
+				return
+			}
+		}
+	}
+
+	// 数据集
+	dataNodes := resp.ReturnData.DataNodes
+	dataMap := make(map[string]QuotaDataNode)
+	for _, d := range dataNodes {
+		dataMap[d.Code] = d
+	}
+
+	// 取出指标(Y轴), 日期(X轴)
+	wdNodes := resp.ReturnData.WdNodes
+	var quotaNodes, dateNodes []QuotaWdNodeData
+	for _, w := range wdNodes {
+		if w.WdCode == "zb" {
+			quotaNodes = w.Nodes
+			continue
+		}
+		if w.WdCode == "sj" {
+			dateNodes = w.Nodes
+		}
+	}
+
+	// 指标编码去重, 指标编码+日期数据去重
+	indexOB := new(models.BaseFromNationalStatisticsIndex)
+	indexCond := ` AND dbcode = ?`
+	indexPars := make([]interface{}, 0)
+	indexPars = append(indexPars, dbCode)
+	indexList, e := indexOB.GetItemsByCondition(indexCond, indexPars, []string{"index_code"}, "")
+	if e != nil {
+		err = fmt.Errorf("获取指标列表失败, Err: %s", e.Error())
+		return
+	}
+	indexExistMap := make(map[string]bool)
+	for _, v := range indexList {
+		indexExistMap[v.IndexCode] = true
+	}
+
+	// 遍历XY轴
+	indexDataList := make([]*models.SaveNationalStatisticsIndexAndDataReq, 0)
+	indexDataMap := make(map[string][]*models.BaseFromNationalStatisticsData)
+	for _, q := range quotaNodes {
+		indexCode := fmt.Sprintf("%s%s", dbCode, q.Code)
+
+		// 数据去重
+		dataExistMap := make(map[string]bool)
+		dataOB := new(models.BaseFromNationalStatisticsData)
+		dataCond := ` AND index_code = ?`
+		dataPars := make([]interface{}, 0)
+		dataPars = append(dataPars, indexCode)
+		dataList, e := dataOB.GetItemsByCondition(dataCond, dataPars, []string{"index_code", "data_time"}, "")
+		if e != nil {
+			err = fmt.Errorf("获取指标数据列表失败, Err: %s", e.Error())
+			return
+		}
+		for _, v := range dataList {
+			dataExistMap[v.DataTime.Format(utils.FormatDate)] = true
+		}
+
+		// 指标
+		r := new(models.SaveNationalStatisticsIndexAndDataReq)
+		r.Index = &models.BaseFromNationalStatisticsIndex{
+			BaseFromNationalStatisticsClassifyId: classifyId,
+			Dbcode:                               dbCode,
+			IndexCode:                            indexCode,
+			IndexName:                            q.Name,
+			Frequency:                            frequency,
+			Unit:                                 q.Unit,
+			CreateTime:                           time.Now().Local(),
+			ModifyTime:                           time.Now().Local(),
+		}
+		if indexExistMap[indexCode] {
+			r.IndexExist = true
+		}
+
+		// 数据
+		for _, d := range dateNodes {
+			k := fmt.Sprintf("%s.%s_%s.%s", "zb", q.Code, "sj", d.Code)
+			v := dataMap[k]
+			if !v.Data.HasData {
+				continue
+			}
+
+			t, e := formatMonth2YearDateCode(d.Code)
+			if e != nil {
+				err = fmt.Errorf("格式化日期code失败, Err: %s", e.Error())
+				return
+			}
+			if dataExistMap[t.Format(utils.FormatDate)] {
+				continue
+			}
+
+			// 数据map
+			if indexDataMap[indexCode] == nil {
+				indexDataMap[indexCode] = make([]*models.BaseFromNationalStatisticsData, 0)
+			}
+			indexDataMap[indexCode] = append(indexDataMap[indexCode], &models.BaseFromNationalStatisticsData{
+				IndexCode:  indexCode,
+				DataTime:   t,
+				Value:      v.Data.Data,
+				CreateTime: time.Now().Local(),
+				ModifyTime: time.Now().Local(),
+			})
+		}
+		indexDataList = append(indexDataList, r)
+	}
+
+	// 保存指标
+	for _, v := range indexDataList {
+		ds := indexDataMap[v.Index.IndexCode]
+		if ds == nil || (ds != nil && len(ds) == 0) {
+			continue
+		}
+		v.DataList = ds
+		if e := models.SaveNationalStatisticsIndexAndData(v); e != nil {
+			err = fmt.Errorf("保存指标和数据失败, Err: %s", e.Error())
+			return
+		}
+	}
+	return
+}
+
+// SyncXDateYQuotaZRegDb 同步三维度X轴-日期, Y轴-指标, Z轴-地区的数据库(分省月季年度、主要城市月年度、国际市场月度商品价格)
+func SyncXDateYQuotaZRegDb(dbs []string) (err error) {
+	if len(dbs) == 0 {
+		dbs = []string{"fsyd", "fsjd", "fsnd", "csyd", "csnd", "gjydsc"}
+	}
+	defer func() {
+		d := strings.Join(dbs, ",")
+		if err != nil {
+			utils.FileLog.Error("统计局-同步%s数据库失败, ErrMsg: %s", d, err.Error())
+			go alarm_msg.SendAlarmMsg(fmt.Sprintf("统计局-同步%s数据库失败, ErrMsg: %s", d, err.Error()), 3)
+			return
+		}
+		utils.FileLog.Info("统计局-同步%s数据库成功", d)
+	}()
+
+	// 需要同步的数据库
+	for _, d := range dbs {
+		classifyOB := new(models.BaseFromNationalStatisticsClassify)
+		// 注意此处只需要同步分类中为指标的即可, 分类为地区的数据在指标中均有包含
+		classifyCond := ` AND is_parent = 0 AND wdcode = 'zb' AND dbcode = ?`
+		classifyPars := make([]interface{}, 0)
+		classifyPars = append(classifyPars, d)
+		classifyOrder := ` base_from_national_statistics_classify_id ASC`
+		classifyList, e := classifyOB.GetItemsByCondition(classifyCond, classifyPars, []string{}, classifyOrder)
+		if e != nil {
+			err = fmt.Errorf("获取%s分类列表失败, Err: %s", d, e.Error())
+			return
+		}
+		utils.FileLog.Info("%s分类长度: %d\n", d, len(classifyList))
+
+		// 查询其他维度-地区
+		wdList, e := GetOtherWd(d, "", "")
+		var regList []OtherWdNodes
+		for _, wd := range wdList {
+			if wd.WdCode == "reg" {
+				regList = wd.Nodes
+				break
+			}
+		}
+		if len(regList) == 0 {
+			err = fmt.Errorf("其他维度为空, DbCode: %s", d)
+			return
+		}
+
+		// 同步指标和数据
+		for _, c := range classifyList {
+			utils.FileLog.Info("开始同步分类-%d: %s", c.BaseFromNationalStatisticsClassifyId, c.ClassifyName)
+			if e = SyncXDateYQuotaZRegData(c.BaseFromNationalStatisticsClassifyId, c.Dbcode, c.Id, regList); e != nil {
+				err = fmt.Errorf("同步指标数据失败, DbCode: %s, ClassifyId: %d, Err: %s", c.Dbcode, c.BaseFromNationalStatisticsClassifyId, e.Error())
+				return
+			}
+			utils.FileLog.Info("结束同步分类-%d: %s", c.BaseFromNationalStatisticsClassifyId, c.ClassifyName)
+		}
+	}
+	return
+}
+
+// SyncXDateYQuotaZRegData 同步三维度X轴-日期, Y轴-指标, Z轴-地区的数据
+func SyncXDateYQuotaZRegData(classifyId int, dbCode, classifyCode string, regList []OtherWdNodes) (err error) {
+	defer func() {
+		if err != nil {
+			utils.FileLog.Error("同步数据库DbCode: %s, 分类ClassifyId: %d失败, ErrMsg: %s", dbCode, classifyId, err.Error())
+		}
+	}()
+
+	// 根据DbCode判断频度和查询的时间区间
+	frequency := ""
+	timeParam := ""
+	if strings.Contains(dbCode, "yd") {
+		timeParam = "LAST36" // 最近36个月
+		frequency = "月度"
+	}
+	if strings.Contains(dbCode, "jd") {
+		timeParam = "LAST18" // 最近18个季度
+		frequency = "季度"
+	}
+	if strings.Contains(dbCode, "nd") {
+		timeParam = "LAST20" // 最近20年
+		frequency = "年度"
+	}
+
+	// 遍历地区维度, 查询指标和数据
+	for _, reg := range regList {
+		var dataReq DataApiReq
+		dataReq.DbCode = dbCode
+		dataReq.WdsList = append(dataReq.WdsList, Wds{
+			WdCode:    "reg",
+			ValueCode: reg.Code,
+		})
+		dataReq.DfwdsList = append(dataReq.DfwdsList, Wds{
+			WdCode:    "zb",
+			ValueCode: classifyCode,
+		}, Wds{
+			WdCode:    "sj",
+			ValueCode: timeParam,
+		})
+		attempt := 0
+		resp, e := CommonDataApiRequest(dataReq)
+		if e != nil {
+			//if !strings.Contains(e.Error(), "connection attempt failed") {
+			//	err = fmt.Errorf("查询数据失败, Err: %s", e.Error())
+			//	return
+			//}
+			// 连接失败重新尝试3次
+			for {
+				time.Sleep(2 * time.Minute)
+				attempt += 1
+				utils.FileLog.Info("当前第%d次重新请求", attempt)
+				resp, e = CommonDataApiRequest(dataReq)
+				if e == nil {
+					break
+				}
+				if attempt >= 3 {
+					s, _ := json.Marshal(dataReq)
+					err = fmt.Errorf("查询数据重试失败, DataReq: %s", s)
+					return
+				}
+			}
+		}
+
+		// 数据集
+		dataNodes := resp.ReturnData.DataNodes
+		dataMap := make(map[string]QuotaDataNode)
+		for _, d := range dataNodes {
+			dataMap[d.Code] = d
+		}
+
+		// 取出指标(Y轴), 日期(X轴)
+		wdNodes := resp.ReturnData.WdNodes
+		var quotaNodes, dateNodes []QuotaWdNodeData
+		for _, w := range wdNodes {
+			if w.WdCode == "zb" {
+				quotaNodes = w.Nodes
+				continue
+			}
+			if w.WdCode == "sj" {
+				dateNodes = w.Nodes
+			}
+		}
+
+		// 指标编码去重, 指标编码+日期数据去重
+		indexOB := new(models.BaseFromNationalStatisticsIndex)
+		indexCond := ` AND dbcode = ?`
+		indexPars := make([]interface{}, 0)
+		indexPars = append(indexPars, dbCode)
+		indexList, e := indexOB.GetItemsByCondition(indexCond, indexPars, []string{"index_code"}, "")
+		if e != nil {
+			err = fmt.Errorf("获取指标列表失败, Err: %s", e.Error())
+			return
+		}
+		indexExistMap := make(map[string]bool)
+		for _, v := range indexList {
+			indexExistMap[v.IndexCode] = true
+		}
+
+		// 遍历XY轴
+		indexDataList := make([]*models.SaveNationalStatisticsIndexAndDataReq, 0)
+		indexDataMap := make(map[string][]*models.BaseFromNationalStatisticsData)
+		for _, q := range quotaNodes {
+			// dbcode+指标code+地区code
+			indexCode := fmt.Sprintf("%s%s%s", dbCode, q.Code, reg.Code)
+
+			// 数据去重
+			dataExistMap := make(map[string]bool)
+			dataOB := new(models.BaseFromNationalStatisticsData)
+			dataCond := ` AND index_code = ?`
+			dataPars := make([]interface{}, 0)
+			dataPars = append(dataPars, indexCode)
+			dataList, e := dataOB.GetItemsByCondition(dataCond, dataPars, []string{"index_code", "data_time"}, "")
+			if e != nil {
+				err = fmt.Errorf("获取指标数据列表失败, Err: %s", e.Error())
+				return
+			}
+			for _, v := range dataList {
+				dataExistMap[v.DataTime.Format(utils.FormatDate)] = true
+			}
+
+			// 指标
+			r := new(models.SaveNationalStatisticsIndexAndDataReq)
+			r.Index = &models.BaseFromNationalStatisticsIndex{
+				BaseFromNationalStatisticsClassifyId: classifyId,
+				Dbcode:                               dbCode,
+				IndexCode:                            indexCode,
+				IndexName:                            q.Name,
+				Frequency:                            frequency,
+				Unit:                                 q.Unit,
+				Reg:                                  reg.Name,
+				CreateTime:                           time.Now().Local(),
+				ModifyTime:                           time.Now().Local(),
+			}
+			if indexExistMap[indexCode] {
+				r.IndexExist = true
+			}
+
+			// 数据
+			// zb.A01010201_reg.110000_sj.201608
+			for _, d := range dateNodes {
+				k := fmt.Sprintf("%s.%s_%s.%s_%s.%s", "zb", q.Code, "reg", reg.Code, "sj", d.Code)
+				v := dataMap[k]
+				if !v.Data.HasData {
+					continue
+				}
+
+				t, e := formatMonth2YearDateCode(d.Code)
+				if e != nil {
+					err = fmt.Errorf("格式化日期code失败, Err: %s", e.Error())
+					return
+				}
+				if dataExistMap[t.Format(utils.FormatDate)] {
+					continue
+				}
+
+				// 数据map
+				if indexDataMap[indexCode] == nil {
+					indexDataMap[indexCode] = make([]*models.BaseFromNationalStatisticsData, 0)
+				}
+				indexDataMap[indexCode] = append(indexDataMap[indexCode], &models.BaseFromNationalStatisticsData{
+					IndexCode:  indexCode,
+					DataTime:   t,
+					Value:      v.Data.Data,
+					CreateTime: time.Now().Local(),
+					ModifyTime: time.Now().Local(),
+				})
+			}
+			indexDataList = append(indexDataList, r)
+		}
+
+		// 保存指标
+		for _, v := range indexDataList {
+			ds := indexDataMap[v.Index.IndexCode]
+			if ds == nil || (ds != nil && len(ds) == 0) {
+				continue
+			}
+			v.DataList = ds
+			if e := models.SaveNationalStatisticsIndexAndData(v); e != nil {
+				err = fmt.Errorf("保存指标和数据失败, Err: %s", e.Error())
+				return
+			}
+		}
+	}
+	return
+}
+
+// SyncXRegYDateZQuotaDb 同步三维度X轴-地区, Y轴-日期的数据库(港澳台、国际数据指标)
+func SyncXRegYDateZQuotaDb(dbs []string) (err error) {
+	if len(dbs) == 0 {
+		dbs = []string{"gatyd", "gatnd", "gjyd", "gjydsdj", "gjnd"}
+	}
+	defer func() {
+		d := strings.Join(dbs, ",")
+		if err != nil {
+			utils.FileLog.Error("统计局-同步%s数据库失败, ErrMsg: %s", d, err.Error())
+			go alarm_msg.SendAlarmMsg(fmt.Sprintf("统计局-同步%s数据库失败, ErrMsg: %s", d, err.Error()), 3)
+			return
+		}
+		utils.FileLog.Info("统计局-同步%s数据库成功", d)
+	}()
+
+	// 需要同步的数据库
+	for _, d := range dbs {
+		classifyOB := new(models.BaseFromNationalStatisticsClassify)
+		// 注意此处只需要同步分类中为指标的即可, 分类为地区的数据在指标中均有包含
+		classifyCond := ` AND is_parent = 0 AND wdcode = 'zb' AND dbcode = ?`
+		classifyPars := make([]interface{}, 0)
+		classifyPars = append(classifyPars, d)
+		classifyOrder := ` base_from_national_statistics_classify_id ASC`
+		classifyList, e := classifyOB.GetItemsByCondition(classifyCond, classifyPars, []string{}, classifyOrder)
+		if e != nil {
+			err = fmt.Errorf("获取%s分类列表失败, Err: %s", d, e.Error())
+			return
+		}
+		utils.FileLog.Info("%s分类长度: %d\n", d, len(classifyList))
+
+		// 同步指标和数据
+		for _, c := range classifyList {
+			utils.FileLog.Info("开始同步分类-%d: %s", c.BaseFromNationalStatisticsClassifyId, c.ClassifyName)
+			if e = SyncXRegYDateZQuotaDbData(c.BaseFromNationalStatisticsClassifyId, c.Dbcode, c.Id); e != nil {
+				err = fmt.Errorf("同步指标数据失败, DbCode: %s, ClassifyId: %d, Err: %s", c.Dbcode, c.BaseFromNationalStatisticsClassifyId, e.Error())
+				return
+			}
+			utils.FileLog.Info("结束同步分类-%d: %s", c.BaseFromNationalStatisticsClassifyId, c.ClassifyName)
+		}
+	}
+	return
+}
+
+// SyncXRegYDateZQuotaDbData 同步三维度X轴-地区, Y轴-日期, Z轴-指标的数据
+func SyncXRegYDateZQuotaDbData(classifyId int, dbCode, classifyCode string) (err error) {
+	defer func() {
+		if err != nil {
+			utils.FileLog.Error("同步数据库DbCode: %s, 分类ClassifyId: %d失败, ErrMsg: %s", dbCode, classifyId, err.Error())
+		}
+	}()
+
+	// 根据DbCode判断频度和查询的时间区间
+	frequency := ""
+	timeParam := ""
+	if strings.Contains(dbCode, "yd") {
+		timeParam = "LAST36" // 最近36个月
+		frequency = "月度"
+	}
+	if strings.Contains(dbCode, "jd") {
+		timeParam = "LAST18" // 最近18个季度
+		frequency = "季度"
+	}
+	if strings.Contains(dbCode, "nd") {
+		timeParam = "LAST20" // 最近20年
+		frequency = "年度"
+	}
+
+	// 先以指标作为行进行默认查询, 取出其中的指标作为当前分类的指标
+	//f := url.Values{}
+	//f.Add("m", "QueryData")
+	//f.Add("dbcode", "gatyd")
+	//f.Add("rowcode", "zb")
+	//f.Add("colcode", "reg")
+	//f.Add("wds", `[{"wdcode":"sj","valuecode":"LAST36"}]`)
+	//f.Add("dfwds", `[{"wdcode":"zb","valuecode":"A01"}]`)
+	var defaultReq DataApiReq
+	defaultReq.DbCode = dbCode
+	defaultReq.RowCode = "zb"
+	defaultReq.ColCode = "reg"
+	defaultReq.WdsList = append(defaultReq.WdsList, Wds{
+		WdCode:    "sj",
+		ValueCode: timeParam,
+	})
+	defaultReq.DfwdsList = append(defaultReq.DfwdsList, Wds{
+		WdCode:    "zb",
+		ValueCode: classifyCode,
+	})
+	defaultResult, e := CommonDataApiRequest(defaultReq)
+	if e != nil {
+		err = fmt.Errorf("默认查询数据失败, Err: %s", e.Error())
+		return
+	}
+	var quotaWds []QuotaWdNodeData
+	for _, n := range defaultResult.ReturnData.WdNodes {
+		if n.WdCode == "zb" {
+			quotaWds = n.Nodes
+			break
+		}
+	}
+
+	// 遍历指标维度
+	for _, quota := range quotaWds {
+		//f := url.Values{}
+		//f.Add("m", "QueryData")
+		//f.Add("dbcode", "gatyd")
+		//f.Add("rowcode", "sj")
+		//f.Add("colcode", "reg")
+		//f.Add("wds", `[{"wdcode":"zb","valuecode":"A010A"}]`)
+		//f.Add("dfwds", `[{"wdcode":"sj","valuecode":"LAST36"}]`)
+		var dataReq DataApiReq
+		dataReq.DbCode = dbCode
+		dataReq.RowCode = "sj"
+		dataReq.ColCode = "reg"
+		dataReq.WdsList = append(defaultReq.WdsList, Wds{
+			WdCode:    "zb",
+			ValueCode: quota.Code,
+		})
+		dataReq.DfwdsList = append(defaultReq.DfwdsList, Wds{
+			WdCode:    "sj",
+			ValueCode: timeParam,
+		})
+		attempt := 0
+		resp, e := CommonDataApiRequest(dataReq)
+		if e != nil {
+			//if !strings.Contains(e.Error(), "connection attempt failed") {
+			//	err = fmt.Errorf("查询数据失败, Err: %s", e.Error())
+			//	return
+			//}
+			// 连接失败重新尝试3次
+			for {
+				time.Sleep(2 * time.Minute)
+				attempt += 1
+				utils.FileLog.Info("当前第%d次重新请求", attempt)
+				resp, e = CommonDataApiRequest(dataReq)
+				if e == nil {
+					break
+				}
+				if attempt >= 3 {
+					s, _ := json.Marshal(dataReq)
+					err = fmt.Errorf("查询数据重试失败, DataReq: %s", s)
+					return
+				}
+			}
+		}
+
+		// 数据集
+		dataNodes := resp.ReturnData.DataNodes
+		dataMap := make(map[string]QuotaDataNode)
+		for _, d := range dataNodes {
+			dataMap[d.Code] = d
+		}
+
+		// 取出指标(Y轴), 日期(X轴)
+		wdNodes := resp.ReturnData.WdNodes
+		//var quotaNodes, dateNodes, regNodes []QuotaWdNodeData
+		var dateNodes, regNodes []QuotaWdNodeData
+		for _, w := range wdNodes {
+			switch w.WdCode {
+			case "zb":
+				//quotaNodes = w.Nodes
+				break
+			case "sj":
+				dateNodes = w.Nodes
+				break
+			case "reg":
+				regNodes = w.Nodes
+				break
+			}
+		}
+
+		// 指标编码去重, 指标编码+日期数据去重
+		indexOB := new(models.BaseFromNationalStatisticsIndex)
+		indexCond := ` AND dbcode = ?`
+		indexPars := make([]interface{}, 0)
+		indexPars = append(indexPars, dbCode)
+		indexList, e := indexOB.GetItemsByCondition(indexCond, indexPars, []string{"index_code"}, "")
+		if e != nil {
+			err = fmt.Errorf("获取指标列表失败, Err: %s", e.Error())
+			return
+		}
+		indexExistMap := make(map[string]bool)
+		for _, v := range indexList {
+			indexExistMap[v.IndexCode] = true
+		}
+
+		// 遍历XY轴
+		indexDataList := make([]*models.SaveNationalStatisticsIndexAndDataReq, 0)
+		indexDataMap := make(map[string][]*models.BaseFromNationalStatisticsData)
+
+		// 遍历X轴-地区
+		for _, reg := range regNodes {
+			// 指标: dbcode+指标code+地区code
+			indexCode := fmt.Sprintf("%s%s%s", dbCode, quota.Code, reg.Code)
+
+			// 数据去重
+			dataExistMap := make(map[string]bool)
+			dataOB := new(models.BaseFromNationalStatisticsData)
+			dataCond := ` AND index_code = ?`
+			dataPars := make([]interface{}, 0)
+			dataPars = append(dataPars, indexCode)
+			dataList, e := dataOB.GetItemsByCondition(dataCond, dataPars, []string{"index_code", "data_time"}, "")
+			if e != nil {
+				err = fmt.Errorf("获取指标数据列表失败, Err: %s", e.Error())
+				return
+			}
+			for _, v := range dataList {
+				dataExistMap[v.DataTime.Format(utils.FormatDate)] = true
+			}
+
+			r := new(models.SaveNationalStatisticsIndexAndDataReq)
+			r.Index = &models.BaseFromNationalStatisticsIndex{
+				BaseFromNationalStatisticsClassifyId: classifyId,
+				Dbcode:                               dbCode,
+				IndexCode:                            indexCode,
+				IndexName:                            quota.Name,
+				Frequency:                            frequency,
+				Unit:                                 quota.Unit,
+				Reg:                                  reg.Name,
+				CreateTime:                           time.Now().Local(),
+				ModifyTime:                           time.Now().Local(),
+			}
+			if indexExistMap[indexCode] {
+				r.IndexExist = true
+			}
+
+			// 遍历Y轴-日期
+			for _, d := range dateNodes {
+				k := fmt.Sprintf("%s.%s_%s.%s_%s.%s", "zb", quota.Code, "reg", reg.Code, "sj", d.Code)
+				v := dataMap[k]
+				if !v.Data.HasData {
+					continue
+				}
+
+				// 日期去重
+				t, e := formatMonth2YearDateCode(d.Code)
+				if e != nil {
+					err = fmt.Errorf("格式化日期code失败, Err: %s", e.Error())
+					return
+				}
+				if dataExistMap[t.Format(utils.FormatDate)] {
+					continue
+				}
+
+				// 数据map
+				if indexDataMap[indexCode] == nil {
+					indexDataMap[indexCode] = make([]*models.BaseFromNationalStatisticsData, 0)
+				}
+				indexDataMap[indexCode] = append(indexDataMap[indexCode], &models.BaseFromNationalStatisticsData{
+					IndexCode:  indexCode,
+					DataTime:   t,
+					Value:      v.Data.Data,
+					CreateTime: time.Now().Local(),
+					ModifyTime: time.Now().Local(),
+				})
+			}
+			indexDataList = append(indexDataList, r)
+		}
+
+		// 保存指标
+		for _, v := range indexDataList {
+			ds := indexDataMap[v.Index.IndexCode]
+			if ds == nil || (ds != nil && len(ds) == 0) {
+				continue
+			}
+			v.DataList = ds
+			if e := models.SaveNationalStatisticsIndexAndData(v); e != nil {
+				err = fmt.Errorf("保存指标和数据失败, Err: %s", e.Error())
+				return
+			}
+		}
+	}
+	return
+}

services/task.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"fmt"
 	"github.com/beego/beego/v2/task"
+	"hongze/hongze_data_crawler/services/national_data"
 )
 
 func Task() {
@@ -14,11 +15,25 @@ func Task() {
 	refreshVisitors := task.NewTask("RefreshChangesVisitorsCovid", "0 30 2-22/10 * * *", RefreshChangesVisitorsCovid)
 	syncEiaSteoData := task.NewTask("SyncEiaSteoData", "0 0 22 * * *", SyncEiaSteoData)
 
+	// 统计局-分月季年爬
+	refreshNationalMonthA := task.NewTask("RefreshNationalMonthDbA", "0 15 2 8 * *", national_data.RefreshNationalMonthDbA)
+	refreshNationalMonthB := task.NewTask("RefreshNationalMonthDbB", "0 15 2 16 * *", national_data.RefreshNationalMonthDbB)
+	refreshNationalQuarter := task.NewTask("RefreshNationalQuarterDb", "0 25 1 15 1,4,7,10 *", national_data.RefreshNationalQuarterDb)
+	refreshNationalYearA := task.NewTask("RefreshNationalYearDbA", "0 45 1 13 2 *", national_data.RefreshNationalYearDbA)
+	refreshNationalYearB := task.NewTask("RefreshNationalYearDbB", "0 45 1 26 2 *", national_data.RefreshNationalYearDbB)
+
 	task.AddTask("数据爬取", refreshData)
 	task.AddTask("欧洲天然气爬取", refreshEic)
 	task.AddTask("中国煤炭网爬取", refreshCoal)
 	task.AddTask("谷歌出行指数爬取", refreshVisitors)
 	task.AddTask("eia steo报告", syncEiaSteoData) //每天22点爬一次
+
+	task.AddTask("统计局数据爬取-月度A", refreshNationalMonthA) // 每月8号2:15执行
+	task.AddTask("统计局数据爬取-月度B", refreshNationalMonthB) // 每月16号2:15执行
+	task.AddTask("统计局数据爬取-季度", refreshNationalQuarter) // 每年1/4/7/10月15日1:25执行
+	task.AddTask("统计局数据爬取-年度A", refreshNationalYearA)  // 每年2月13日1:45执行
+	task.AddTask("统计局数据爬取-年度B", refreshNationalYearB)  // 每年2月26日1:45执行
+
 	task.StartTask()
 	//FileCoalJsm()
 	//FileCoalFirm()
@@ -57,3 +72,15 @@ func RefreshChangesVisitorsCovid(cont context.Context) (err error) {
 	err = AddSourceChangesVisitorsCovid()
 	return
 }
+
+//func Task2() {
+//	fmt.Println("start")
+//
+//	var cont context.Context
+//	_ = national_data.RefreshNationalYearDbA(cont)
+//
+//	//_ = national_data.RefreshNationalYearDbB(cont)
+//
+//	//_ = national_data.ApiTest()
+//	fmt.Println("end")
+//}

utils/common.go

@@ -3,6 +3,7 @@ package utils
 import (
 	"bufio"
 	"crypto/md5"
+	cryRand "crypto/rand"
 	"crypto/sha1"
 	"encoding/base64"
 	"encoding/hex"
@@ -13,6 +14,7 @@ import (
 	"image/png"
 	"io"
 	"math"
+	"math/big"
 	"math/rand"
 	"net/http"
 	"os"
@@ -24,7 +26,7 @@ import (
 	"time"
 )
 
-//随机数种子
+// 随机数种子
 var rnd = rand.New(rand.NewSource(time.Now().UnixNano()))
 
 func GetRandString(size int) string {
@@ -61,13 +63,13 @@ func StringsToJSON(str string) string {
 	return jsons
 }
 
-//序列化
+// 序列化
 func ToString(v interface{}) string {
 	data, _ := json.Marshal(v)
 	return string(data)
 }
 
-//md5加密
+// md5加密
 func MD5(data string) string {
 	m := md5.Sum([]byte(data))
 	return hex.EncodeToString(m[:])
@@ -95,7 +97,7 @@ func GetToday(format string) string {
 	return today
 }
 
-//获取今天剩余秒数
+// 获取今天剩余秒数
 func GetTodayLastSecond() time.Duration {
 	today := GetToday(FormatDate) + " 23:59:59"
 	end, _ := time.ParseInLocation(FormatDateTime, today, time.Local)
@@ -117,7 +119,7 @@ func GetBrithDate(idcard string) string {
 	return GetToday(FormatDate)
 }
 
-//处理性别
+// 处理性别
 func WhichSexByIdcard(idcard string) string {
 	var sexs = [2]string{"女", "男"}
 	length := len(idcard)
@@ -131,7 +133,7 @@ func WhichSexByIdcard(idcard string) string {
 	return "男"
 }
 
-//截取小数点后几位
+// 截取小数点后几位
 func SubFloatToString(f float64, m int) string {
 	n := strconv.FormatFloat(f, 'f', -1, 64)
 	if n == "" {
@@ -150,20 +152,20 @@ func SubFloatToString(f float64, m int) string {
 	return newn[0] + "." + newn[1][:m]
 }
 
-//截取小数点后几位
+// 截取小数点后几位
 func SubFloatToFloat(f float64, m int) float64 {
 	newn := SubFloatToString(f, m)
 	newf, _ := strconv.ParseFloat(newn, 64)
 	return newf
 }
 
-//截取小数点后几位
+// 截取小数点后几位
 func SubFloatToFloatStr(f float64, m int) string {
 	newn := SubFloatToString(f, m)
 	return newn
 }
 
-//获取相差时间-年
+// 获取相差时间-年
 func GetYearDiffer(start_time, end_time string) int {
 	t1, _ := time.ParseInLocation("2006-01-02", start_time, time.Local)
 	t2, _ := time.ParseInLocation("2006-01-02", end_time, time.Local)
@@ -174,7 +176,7 @@ func GetYearDiffer(start_time, end_time string) int {
 	return age
 }
 
-//获取相差时间-秒
+// 获取相差时间-秒
 func GetSecondDifferByTime(start_time, end_time time.Time) int64 {
 	diff := end_time.Unix() - start_time.Unix()
 	return diff
@@ -201,7 +203,7 @@ func StrListToString(strList []string) (str string) {
 	return ""
 }
 
-//Token
+// Token
 func GetToken() string {
 	randStr := GetRandString(64)
 	token := MD5(randStr + Md5Key)
@@ -209,18 +211,18 @@ func GetToken() string {
 	return strings.ToUpper(token + GetRandString(tokenLen))
 }
 
-//数据没有记录
+// 数据没有记录
 func ErrNoRow() string {
 	return "<QuerySeter> no row found"
 }
 
-//判断文件是否存在
+// 判断文件是否存在
 func FileIsExist(filePath string) bool {
 	_, err := os.Stat(filePath)
 	return err == nil || os.IsExist(err)
 }
 
-//获取图片扩展名
+// 获取图片扩展名
 func GetImgExt(file string) (ext string, err error) {
 	var headerByte []byte
 	headerByte = make([]byte, 8)
@@ -263,7 +265,7 @@ func GetImgExt(file string) (ext string, err error) {
 	return ext, nil
 }
 
-//保存图片
+// 保存图片
 func SaveImage(path string, img image.Image) (err error) {
 	//需要保持的文件
 	imgfile, err := os.Create(path)
@@ -273,7 +275,7 @@ func SaveImage(path string, img image.Image) (err error) {
 	return err
 }
 
-//下载图片
+// 下载图片
 func DownloadImage(imgUrl string) (filePath string, err error) {
 	imgPath := "./static/imgs/"
 	fileName := path.Base(imgUrl)
@@ -299,7 +301,7 @@ func DownloadImage(imgUrl string) (filePath string, err error) {
 	return
 }
 
-//保存base64数据为文件
+// 保存base64数据为文件
 func SaveBase64ToFile(content, path string) error {
 	data, err := base64.StdEncoding.DecodeString(content)
 	if err != nil {
@@ -429,7 +431,7 @@ func GetWilsonScore(p, n float64) float64 {
 	return toFixed(((p+1.9208)/(p+n)-1.96*math.Sqrt(p*n/(p+n)+0.9604)/(p+n))/(1+3.8416/(p+n)), 2)
 }
 
-//将中文数字转化成数字，比如 第三百四十五章，返回第345章 不支持一亿及以上
+// 将中文数字转化成数字，比如 第三百四十五章，返回第345章 不支持一亿及以上
 func ChangeWordsToNum(str string) (numStr string) {
 	words := ([]rune)(str)
 	num := 0
@@ -600,12 +602,12 @@ func GetMonthStartAndEnd(myYear string, myMonth string) (startDate, endDate stri
 	return t1, t2
 }
 
-//移除字符串中的空格
+// 移除字符串中的空格
 func TrimStr(str string) (str2 string) {
 	return strings.Replace(str, " ", "", -1)
 }
 
-//字符串转换为time
+// 字符串转换为time
 func StrTimeToTime(strTime string) time.Time {
 	timeLayout := "2006-01-02 15:04:05"  //转化所需模板
 	loc, _ := time.LoadLocation("Local") //重要：获取时区
@@ -613,7 +615,7 @@ func StrTimeToTime(strTime string) time.Time {
 	return resultTime
 }
 
-//字符串类型时间转周几
+// 字符串类型时间转周几
 func StrDateTimeToWeek(strTime string) string {
 	var WeekDayMap = map[string]string{
 		"Monday":    "周一",
@@ -630,7 +632,7 @@ func StrDateTimeToWeek(strTime string) string {
 	return WeekDayMap[staweek_int]
 }
 
-//时间格式转年月日字符串
+// 时间格式转年月日字符串
 func TimeToStrYmd(time2 time.Time) string {
 	var Ymd string
 	year := time2.Year()
@@ -640,7 +642,7 @@ func TimeToStrYmd(time2 time.Time) string {
 	return Ymd
 }
 
-//时间格式去掉时分秒
+// 时间格式去掉时分秒
 func TimeRemoveHms(strTime string) string {
 	var Ymd string
 	var resultTime = StrTimeToTime(strTime)
@@ -651,7 +653,7 @@ func TimeRemoveHms(strTime string) string {
 	return Ymd
 }
 
-//文章上一次编辑时间
+// 文章上一次编辑时间
 func ArticleLastTime(strTime string) string {
 	var newTime string
 	stamp, _ := time.ParseInLocation("2006-01-02 15:04:05", strTime, time.Local)
@@ -672,7 +674,7 @@ func ArticleLastTime(strTime string) string {
 	return newTime
 }
 
-//人民币小写转大写
+// 人民币小写转大写
 func ConvertNumToCny(num float64) (str string, err error) {
 	strNum := strconv.FormatFloat(num*100, 'f', 0, 64)
 	sliceUnit := []string{"仟", "佰", "拾", "亿", "仟", "佰", "拾", "万", "仟", "佰", "拾", "元", "角", "分"}
@@ -923,3 +925,42 @@ func GetOrmInReplace(num int) string {
 	}
 	return strings.Join(template, ",")
 }
+
+// InArrayByInt php中的in_array（判断Int类型的切片中是否存在该int值）
+func InArrayByInt(idIntList []int, searchId int) (has bool) {
+	for _, id := range idIntList {
+		if id == searchId {
+			has = true
+			return
+		}
+	}
+	return
+}
+
+// InArrayByStr php中的in_array（判断String类型的切片中是否存在该string值）
+func InArrayByStr(idStrList []string, searchId string) (has bool) {
+	for _, id := range idStrList {
+		if id == searchId {
+			has = true
+			return
+		}
+	}
+	return
+}
+
+// RangeRand 取区间随机数
+func RangeRand(min, max int64) int64 {
+	if min > max {
+		return max
+	}
+	if min < 0 {
+		f64Min := math.Abs(float64(min))
+		i64Min := int64(f64Min)
+		result, _ := cryRand.Int(cryRand.Reader, big.NewInt(max+1+i64Min))
+
+		return result.Int64() - i64Min
+	} else {
+		result, _ := cryRand.Int(cryRand.Reader, big.NewInt(max-min+1))
+		return min + result.Int64()
+	}
+}