test 统计局数据

services/national_data/classify.go

@@ -0,0 +1,142 @@
+package national_data
+
+import (
+	"encoding/json"
+	"fmt"
+	"hongze/hongze_data_crawler/models"
+	"hongze/hongze_data_crawler/services/alarm_msg"
+	"hongze/hongze_data_crawler/utils"
+	"net/url"
+	"time"
+)
+
+// QuotaClassifyTreeResp 指标分类树响应
+type QuotaClassifyTreeResp struct {
+	Id       string `description:"分类ID(字符串)"`
+	IsParent bool   `description:"是否为父级"`
+	Name     string `description:"分类名称"`
+	Pid      string `description:"父级分类ID"`
+	Dbcode   string `description:"源-dbocde"`
+	Wdcode   string `description:"源-wdcode"`
+}
+
+// SyncQuotaClassifyTree 同步指标分类树
+func SyncQuotaClassifyTree() (err error) {
+	defer func() {
+		if err != nil {
+			utils.FileLog.Error("统计局-同步指标分类树失败, ErrMsg: %s", err.Error())
+			go alarm_msg.SendAlarmMsg(fmt.Sprintf("统计局-同步指标分类树失败, ErrMsg: %s", err.Error()), 3)
+		}
+	}()
+
+	// 含指标维度
+	dbCodes := []string{
+		"hgyd", "hgjd", "hgnd", // 月度/季度/年度
+		"fsyd", "fsjd", "fsnd", "csyd", "csnd", "gatyd", "gatnd", // 分省月度/分省季度/分省年度/城市月度价格/城市年度数据/港澳台月度/港澳台年度
+		"gjyd", "gjydsdj", "gjydsc", "gjnd", // 主要国家月度/三大经济体月度/国际市场月度商品/国家年度
+	}
+	// 含地区维度
+	regCodes := []string{
+		"fsyd", "fsjd", "fsnd", "csyd", "csnd", "gatyd", "gatnd", // 分省月度/分省季度/分省年度/城市月度价格/城市年度数据/港澳台月度/港澳台年度
+		"gjyd", "gjydsdj", "gjydsc", "gjnd", // 主要国家月度/三大经济体月度/国际市场月度商品/国家年度
+	}
+
+	for _, code := range dbCodes {
+		fmt.Println("开始同步DbCode: ", code)
+
+		// 指标维度
+		items := make([]*models.BaseFromNationalStatisticsClassify, 0)
+		resp, e := curlAndFormatQuotaClassify("zb", code, "zb", items)
+		if e != nil {
+			err = fmt.Errorf("递归指标分类失败, Err: %s", e.Error())
+			return
+		}
+		items = resp
+
+		// 地区维度
+		if utils.InArrayByStr(regCodes, code) {
+			regResp, e := curlAndFormatQuotaClassify("reg", code, "reg", items)
+			if e != nil {
+				err = fmt.Errorf("递归地区分类失败, Err: %s", e.Error())
+				return
+			}
+			items = append(items, regResp...)
+		}
+
+		// 去重-code+维度code+ID
+		classifyMap := make(map[string]bool)
+		classifyOB := new(models.BaseFromNationalStatisticsClassify)
+		classifyPars := make([]interface{}, 0)
+		classifies, e := classifyOB.GetItemsByCondition("", classifyPars, []string{}, "")
+		if e != nil {
+			err = fmt.Errorf("获取指标分类列表失败, Err: %s", e.Error())
+			return
+		}
+		for _, c := range classifies {
+			classifyMap[fmt.Sprintf("%s%s%s", code, c.Wdcode, c.Id)] = true
+		}
+
+		finalList := make([]*models.BaseFromNationalStatisticsClassify, 0)
+		for _, v := range items {
+			if classifyMap[fmt.Sprintf("%s%s%s", code, v.Wdcode, v.Id)] {
+				continue
+			}
+			finalList = append(finalList, v)
+		}
+		if e = classifyOB.CreateMulti(items); e != nil {
+			err = fmt.Errorf("批量新增指标分类失败, Err: %s", e.Error())
+			return
+		}
+		fmt.Println("结束同步DbCode: ", code)
+	}
+	return
+}
+
+// curlAndFormatQuotaClassify 递归请求分类树
+func curlAndFormatQuotaClassify(id, dbcode, wdcode string, items []*models.BaseFromNationalStatisticsClassify) (resp []*models.BaseFromNationalStatisticsClassify, err error) {
+	f := url.Values{}
+	f.Add("id", id)
+	f.Add("dbcode", dbcode)
+	f.Add("wdcode", wdcode)
+	f.Add("m", "getTree")
+	r, e := NationalHttpPost(NationalStatisticsBaseReqUrl, f.Encode())
+	if e != nil {
+		err = fmt.Errorf("请求指标分类树失败, Err: %s", e.Error())
+		return
+	}
+	list := make([]*QuotaClassifyTreeResp, 0)
+	if e = json.Unmarshal(r, &list); e != nil {
+		err = fmt.Errorf("解析响应数据失败, Err: %s", e.Error())
+		return
+	}
+
+	nowTime := time.Now().Local()
+	for _, v := range list {
+		isParent := 0
+		if v.IsParent {
+			isParent = 1
+		}
+		fmt.Printf("Dbcode: %s, ClassifyName: %s, Id: %s\n", v.Dbcode, v.Name, v.Id)
+		items = append(items, &models.BaseFromNationalStatisticsClassify{
+			ClassifyName: v.Name,
+			Id:           v.Id,
+			Dbcode:       v.Dbcode,
+			Wdcode:       v.Wdcode,
+			Pid:          v.Pid,
+			IsParent:     isParent,
+			CreateTime:   nowTime,
+			ModifyTime:   nowTime,
+		})
+
+		// 向下递归
+		if isParent == 1 {
+			time.Sleep(5 * time.Second) // 缓缓...毕竟接口是人家的...
+			items, e = curlAndFormatQuotaClassify(v.Id, v.Dbcode, v.Wdcode, items)
+			if e != nil {
+				err = fmt.Errorf("递归请求分类树失败, Err: %s", e.Error())
+				return
+			}
+		}
+	}
+	return items, nil
+}

services/national_data/common.go

@@ -0,0 +1,368 @@
+package national_data
+
+import (
+	"crypto/tls"
+	"encoding/json"
+	"fmt"
+	"hongze/hongze_data_crawler/utils"
+	"io/ioutil"
+	"net/http"
+	"net/url"
+	"strings"
+	"time"
+)
+
+const (
+	NationalStatisticsBaseReqUrl = "https://data.stats.gov.cn/easyquery.htm"
+)
+
+func NationalHttpPost(reqUrl, payload string) (result []byte, err error) {
+	time.Sleep(2200 * time.Millisecond) // 目前来看这个速度是不会中断的...就是慢...
+	tr := &http.Transport{
+		TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
+	}
+	client := &http.Client{
+		Transport: tr,
+	}
+	req, err := http.NewRequest("POST", reqUrl, strings.NewReader(payload))
+	if err != nil {
+		return
+	}
+	req.Header.Add("Accept", "text/plain, */*; q=0.01")
+	req.Header.Add("Accept-Encoding", "tgzip, deflate, br")
+	req.Header.Add("Accept-Language", "zh-CN,zh;q=0.9")
+	req.Header.Add("Connection", "keep-alive")
+	req.Header.Add("Content-Length", "37")
+	req.Header.Add("Content-Type", "application/x-www-form-urlencoded")
+	req.Header.Add("Cookie", "wzws_sessionid=gDExNS4xOTQuMTAyLjEyN6BkERzUgmZjNWVlMYFiOWNiZDg=; JSESSIONID=UOri2Cu3f3c-Y3rPgXWJ04E8pfbeyAUGG-s7zJ7Tt0JhlEiLi0EU!412929168; u=5")
+	req.Header.Add("Host", "data.stats.gov.cn")
+	req.Header.Add("Origin", "https://data.stats.gov.cn")
+	req.Header.Set("Referer", "https://data.stats.gov.cn/easyquery.htm?cn=A01")
+	req.Header.Set("sec-ch-ua", "\"Not_A Brand\";v=\"99\", \"Google Chrome\";v=\"109\", \"Chromium\";v=\"109\"")
+	req.Header.Set("sec-ch-ua-mobile", "?0")
+	req.Header.Set("sec-ch-ua-platform", "\"Windows\"")
+	req.Header.Set("Sec-Fetch-Dest", "empty")
+	req.Header.Set("Sec-Fetch-Mode", "cors")
+	req.Header.Set("Sec-Fetch-Site", "same-origin")
+	req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/109.0.0.0 Safari/537.36")
+	req.Header.Set("X-Requested-With", "XMLHttpRequest")
+	res, err := client.Do(req)
+	if err != nil {
+		return
+	}
+	defer res.Body.Close()
+	body, err := ioutil.ReadAll(res.Body)
+	if err != nil {
+		return
+	}
+	result = body
+	return
+}
+
+func NationalGet(reqUrl, payload string) (err error) {
+	tr := &http.Transport{
+		TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
+	}
+	client := &http.Client{
+		Transport: tr,
+	}
+	req, err := http.NewRequest("GET", reqUrl, strings.NewReader(payload))
+	if err != nil {
+		return
+	}
+	res, err := client.Do(req)
+	if err != nil {
+		return
+	}
+	defer res.Body.Close()
+	_, err = ioutil.ReadAll(res.Body)
+	if err != nil {
+		return
+	}
+
+	Cookie := res.Header.Get("Cookie")
+	fmt.Println(Cookie)
+	rcookie := req.Header.Get("Cookie")
+	fmt.Println("rcookie")
+	fmt.Println(rcookie)
+	//fmt.Println("body:" + string(body))
+	cookiesArr := res.Cookies()
+	fmt.Println("cookiesArrLen:", len(cookiesArr))
+	for k, v := range cookiesArr {
+		fmt.Println(k, v)
+	}
+	return
+}
+
+// DataApiReq 数据接口请求体
+type DataApiReq struct {
+	Method    string `description:"方法: QueryData-查询数据; getOtherWds-获取其他维度" json:"method"`
+	DbCode    string `description:"数据库编码" json:"dbcode"`
+	RowCode   string `description:"行-维度: zb; sj; reg" json:"rowcode"`
+	ColCode   string `description:"列-维度: zb; sj; reg" json:"colcode"`
+	WdsList   []Wds  `description:"维度列表" json:"wdsList"`
+	DfwdsList []Wds  `description:"df不知道啥意思...反正也是维度相关的" json:"dfwdsList"`
+}
+
+// Wds 维度
+type Wds struct {
+	WdCode    string `description:"维度: zb-指标; sj-时间; reg-地区" json:"wdcode"`
+	ValueCode string `description:"维度编码" json:"valuecode"`
+}
+
+// CommonDataApiRequest 数据接口请求
+func CommonDataApiRequest(req DataApiReq) (resp QuotaListDataResp, err error) {
+	if req.DbCode == "" {
+		return
+	}
+	if req.Method == "" {
+		req.Method = "QueryData"
+	}
+	if req.RowCode == "" {
+		req.RowCode = "zb"
+	}
+	if req.ColCode == "" {
+		req.ColCode = "sj"
+	}
+
+	// 构建查询
+	f := url.Values{}
+	f.Add("m", req.Method)
+	f.Add("dbcode", req.DbCode)
+	f.Add("rowcode", req.RowCode)
+	f.Add("colcode", req.ColCode)
+	wds := `[]`
+	if len(req.WdsList) > 0 {
+		wdsByte, e := json.Marshal(req.WdsList)
+		if e != nil {
+			err = fmt.Errorf("wds json marshal err: %s", e.Error())
+			return
+		}
+		wds = string(wdsByte)
+	}
+	dfwds := `[]`
+	if len(req.DfwdsList) > 0 {
+		dfwdsByte, e := json.Marshal(req.DfwdsList)
+		if e != nil {
+			err = fmt.Errorf("dfwds json marshal err: %s", e.Error())
+			return
+		}
+		dfwds = string(dfwdsByte)
+	}
+	f.Add("wds", wds)
+	f.Add("dfwds", dfwds)
+	f.Add("k1", fmt.Sprint(time.Now().UnixNano()/1e6))
+	f.Add("h", "1")
+
+	// 响应
+	r, e := NationalHttpPost(NationalStatisticsBaseReqUrl, f.Encode())
+	if e != nil {
+		err = fmt.Errorf("http request err: %s", e.Error())
+		return
+	}
+	utils.FileLog.Info("result: %s", string(r))
+	if e = json.Unmarshal(r, &resp); e != nil {
+		err = fmt.Errorf("resp unmarshal err: %s", e.Error())
+		return
+	}
+	if resp.ReturnCode != 200 {
+		err = fmt.Errorf("resp code err: %d", resp.ReturnCode)
+		return
+	}
+	return
+}
+
+// QuotaListDataResp 指标数据列表响应体
+type QuotaListDataResp struct {
+	ReturnCode int `description:"状态码" json:"returncode"`
+	ReturnData struct {
+		DataNodes []QuotaDataNode `json:"datanodes"`
+		WdNodes   []QuotaWdNode   `json:"wdnodes"`
+	}
+}
+
+// QuotaDataNode 指标数据节点
+type QuotaDataNode struct {
+	Code string `description:"编码"`
+	Data struct {
+		Data    float64 `description:"指标值"`
+		HasData bool    `description:"是否有值" json:"hasdata"`
+		StrData string  `description:"指标值(字符串)" json:"strdata"`
+	}
+	Wds []Wds
+}
+
+// QuotaWdNode 维度节点
+type QuotaWdNode struct {
+	WdCode string `description:"示例: zb; sj; reg;" json:"wdcode"`
+	WdName string `description:"示例: 指标; 时间; 地区" json:"wdname"`
+	Nodes  []QuotaWdNodeData
+}
+
+// QuotaWdNodeData 维度节点数据
+type QuotaWdNodeData struct {
+	Code     string `description:"指标编码"`
+	Name     string `description:"指标名称"`
+	Unit     string `description:"单位"`
+	SortCode int    `description:"编码排序" json:"sortcode"`
+}
+
+// OtherWdResp 其他维度信息响应体
+type OtherWdResp struct {
+	ReturnCode int           `description:"状态码" json:"returncode"`
+	ReturnData []OtherWdData `description:"响应数据" json:"returndata"`
+}
+
+// OtherWdData 其他维度数据
+type OtherWdData struct {
+	IsSj   bool           `description:"是否为时间" json:"issj"`
+	WdCode string         `description:"维度编码" json:"wdcode"`
+	WdName string         `description:"维度名称" json:"wdname"`
+	Nodes  []OtherWdNodes `description:"维度数据" json:"nodes"`
+}
+
+type OtherWdNodes struct {
+	Code string `description:"编码" json:"code"`
+	Name string `description:"名称" json:"name"`
+	Sort string `description:"排序" json:"sort"`
+}
+
+// formatMonth2YearDateCode 将日期code转为对应日期
+func formatMonth2YearDateCode(dateCode string) (date time.Time, err error) {
+	if dateCode == "" {
+		return
+	}
+	// 根据日期code长度进行区分, 格式为三种: 月度-200601; 季度-2006A; 年度-2006
+	switch len([]rune(dateCode)) {
+	case 6:
+		t, e := time.ParseInLocation("200601", dateCode, time.Local)
+		if e != nil {
+			err = fmt.Errorf("月度指标日期转换失败, Err: %s", e.Error())
+			return
+		}
+		date = t
+		break
+	case 5:
+		// 季度ABCD转换成对应日期
+		dateSuffixMap := map[string]string{
+			"A": "03-31",
+			"B": "06-30",
+			"C": "09-30",
+			"D": "12-31",
+		}
+		dateCode = strings.ToUpper(dateCode)
+		quarterTab := dateCode[4:]
+		dateStr := fmt.Sprintf("%s-%s", dateCode[:4], dateSuffixMap[quarterTab])
+		t, e := time.ParseInLocation(utils.FormatDate, dateStr, time.Local)
+		if e != nil {
+			err = fmt.Errorf("季度指标日期转换失败, Err: %s", e.Error())
+			return
+		}
+		date = t
+		break
+	case 4:
+		dateStr := fmt.Sprintf("%s-%s", dateCode, "12-31")
+		t, e := time.ParseInLocation(utils.FormatDate, dateStr, time.Local)
+		if e != nil {
+			err = fmt.Errorf("年度指标日期转换失败, Err: %s", e.Error())
+			return
+		}
+		date = t
+		break
+	default:
+		err = fmt.Errorf("日期code格式有误, code: %s", dateCode)
+		return
+	}
+	return
+}
+
+// GetOtherWd 获取Db下其他维度信息
+func GetOtherWd(dbCode, rowCode, colCode string) (wdList []OtherWdData, err error) {
+	if dbCode == "" {
+		return
+	}
+	if rowCode == "" {
+		rowCode = "zb"
+	}
+	if colCode == "" {
+		colCode = "sj"
+	}
+
+	// 构建查询
+	f := url.Values{}
+	f.Add("m", "getOtherWds")
+	f.Add("dbcode", dbCode)
+	f.Add("rowcode", rowCode)
+	f.Add("colcode", colCode)
+	f.Add("wds", `[]`)
+	f.Add("k1", fmt.Sprint(time.Now().UnixNano()/1e6))
+	f.Add("h", "1")
+	r, e := NationalHttpPost(NationalStatisticsBaseReqUrl, f.Encode())
+	if e != nil {
+		err = fmt.Errorf("请求其他维度信息失败, Err: %s", e.Error())
+		return
+	}
+	utils.FileLog.Info("GetOtherWdInfo Result: %s", string(r))
+
+	// 响应
+	resp := new(OtherWdResp)
+	if e = json.Unmarshal(r, &resp); e != nil {
+		err = fmt.Errorf("其他维度信息Unmarshal Err: %s", e.Error())
+		return
+	}
+	if resp == nil {
+		err = fmt.Errorf("其他维度信息请求结果为空")
+		return
+	}
+	if resp.ReturnCode != 200 {
+		err = fmt.Errorf("其他维度信息请求有误, Code: %d", resp.ReturnCode)
+		return
+	}
+	wdList = resp.ReturnData
+	return
+}
+
+func ApiTest() (err error) {
+	//f := url.Values{}
+	//f.Add("m", "QueryData")
+	//f.Add("dbcode", "fsyd")
+	//f.Add("rowcode", "zb")
+	//f.Add("colcode", "sj")
+	//f.Add("wds", `[{"wdcode":"reg","valuecode":"000"}]`)
+	//f.Add("dfwds", `[{"wdcode":"zb","valuecode":"A01"}]`)
+	f := url.Values{}
+	f.Add("m", "QueryData")
+	f.Add("dbcode", "fsyd")
+	f.Add("rowcode", "zb")
+	f.Add("colcode", "sj")
+	//f.Add("wds", `[{"wdcode":"reg","valuecode":"110000"}]`)
+	f.Add("wds", `[]`)
+	f.Add("dfwds", `[{"wdcode":"zb","valuecode":"A010101"}]`)
+	f.Add("k1", fmt.Sprint(time.Now().UnixNano()/1e6))
+	f.Add("h", "1")
+
+	//f := url.Values{}
+	//f.Add("m", "QueryData")
+	//f.Add("dbcode", "gatyd")
+	//f.Add("rowcode", "sj")
+	//f.Add("colcode", "reg")
+	//f.Add("wds", `[{"wdcode":"zb","valuecode":"A010A"}]`)
+	//f.Add("dfwds", `[{"wdcode":"sj","valuecode":"LAST36"}]`)
+	//f := url.Values{}
+	//f.Add("m", "QueryData")
+	//f.Add("dbcode", "fsyd")
+	//f.Add("rowcode", "zb")
+	//f.Add("colcode", "sj")
+	//f.Add("wds", `[{"wdcode":"reg","valuecode":"000"}]`)
+	//f.Add("dfwds", `[{"wdcode":"zb","valuecode":"A01"}]`)
+	//f.Add("k1", fmt.Sprint(time.Now().UnixNano()/1e6))
+	//f.Add("h", "1")
+
+	r, e := NationalHttpPost(NationalStatisticsBaseReqUrl, f.Encode())
+	if e != nil {
+		fmt.Println("请求失败, Err: ", e.Error())
+		return
+	}
+	utils.FileLog.Info("result: %s", string(r))
+	return
+}

services/national_data/national_data.go

@@ -1,360 +1,43 @@
 package national_data
 
 import (
-	"crypto/tls"
-	"encoding/json"
+	"context"
 	"fmt"
 	"hongze/hongze_data_crawler/models"
 	"hongze/hongze_data_crawler/services/alarm_msg"
 	"hongze/hongze_data_crawler/utils"
-	"io/ioutil"
-	"net/http"
-	"net/url"
 	"strings"
 	"time"
 )
 
-const (
-	NationalStatisticsBaseReqUrl = "https://data.stats.gov.cn/easyquery.htm"
-)
+// RefreshNationalData 刷新统计局数据
+func RefreshNationalData(cont context.Context) (err error) {
+	utils.FileLog.Info("开始刷新统计局数据")
 
-func NationalHttpPost(reqUrl, payload string) (result []byte, err error) {
-	time.Sleep(5 * time.Second) // 目前来看这个速度是不会中断的...就是慢...
-	tr := &http.Transport{
-		TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
-	}
-	client := &http.Client{
-		Transport: tr,
-	}
-	req, err := http.NewRequest("POST", reqUrl, strings.NewReader(payload))
-	if err != nil {
-		return
-	}
-	req.Header.Add("Accept", "text/plain, */*; q=0.01")
-	req.Header.Add("Accept-Encoding", "tgzip, deflate, br")
-	req.Header.Add("Accept-Language", "zh-CN,zh;q=0.9")
-	req.Header.Add("Connection", "keep-alive")
-	req.Header.Add("Content-Length", "37")
-	req.Header.Add("Content-Type", "application/x-www-form-urlencoded")
-	req.Header.Add("Cookie", "wzws_sessionid=gDExNS4xOTQuMTAyLjEyN6BkERzUgmZjNWVlMYFiOWNiZDg=; JSESSIONID=UOri2Cu3f3c-Y3rPgXWJ04E8pfbeyAUGG-s7zJ7Tt0JhlEiLi0EU!412929168; u=5")
-	req.Header.Add("Host", "data.stats.gov.cn")
-	req.Header.Add("Origin", "https://data.stats.gov.cn")
-	req.Header.Set("Referer", "https://data.stats.gov.cn/easyquery.htm?cn=A01")
-	req.Header.Set("sec-ch-ua", "\"Not_A Brand\";v=\"99\", \"Google Chrome\";v=\"109\", \"Chromium\";v=\"109\"")
-	req.Header.Set("sec-ch-ua-mobile", "?0")
-	req.Header.Set("sec-ch-ua-platform", "\"Windows\"")
-	req.Header.Set("Sec-Fetch-Dest", "empty")
-	req.Header.Set("Sec-Fetch-Mode", "cors")
-	req.Header.Set("Sec-Fetch-Site", "same-origin")
-	req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/109.0.0.0 Safari/537.36")
-	req.Header.Set("X-Requested-With", "XMLHttpRequest")
-	res, err := client.Do(req)
-	if err != nil {
-		return
-	}
-	defer res.Body.Close()
-	body, err := ioutil.ReadAll(res.Body)
-	if err != nil {
-		return
-	}
-	result = body
-	return
-}
+	_ = SyncXDateYQuotaDb()
 
-func NationalGet(reqUrl, payload string) (err error) {
-	tr := &http.Transport{
-		TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
-	}
-	client := &http.Client{
-		Transport: tr,
-	}
-	req, err := http.NewRequest("GET", reqUrl, strings.NewReader(payload))
-	if err != nil {
-		return
-	}
-	res, err := client.Do(req)
-	if err != nil {
-		return
-	}
-	defer res.Body.Close()
-	_, err = ioutil.ReadAll(res.Body)
-	if err != nil {
-		return
-	}
-
-	Cookie := res.Header.Get("Cookie")
-	fmt.Println(Cookie)
-	rcookie := req.Header.Get("Cookie")
-	fmt.Println("rcookie")
-	fmt.Println(rcookie)
-	//fmt.Println("body:" + string(body))
-	cookiesArr := res.Cookies()
-	fmt.Println("cookiesArrLen:", len(cookiesArr))
-	for k, v := range cookiesArr {
-		fmt.Println(k, v)
-	}
-	return
-}
-
-// QuotaClassifyTreeResp 指标分类树响应
-type QuotaClassifyTreeResp struct {
-	Id       string `description:"分类ID(字符串)"`
-	IsParent bool   `description:"是否为父级"`
-	Name     string `description:"分类名称"`
-	Pid      string `description:"父级分类ID"`
-	Dbcode   string `description:"源-dbocde"`
-	Wdcode   string `description:"源-wdcode"`
-}
-
-// SyncQuotaClassifyTree 同步指标分类树
-func SyncQuotaClassifyTree() (err error) {
-	defer func() {
-		if err != nil {
-			utils.FileLog.Error("统计局-同步指标分类树失败, ErrMsg: %s", err.Error())
-			go alarm_msg.SendAlarmMsg(fmt.Sprintf("统计局-同步指标分类树失败, ErrMsg: %s", err.Error()), 3)
-		}
-	}()
-
-	// 含指标维度
-	dbCodes := []string{
-		"hgyd", "hgjd", "hgnd", // 月度/季度/年度
-		"fsyd", "fsjd", "fsnd", "csyd", "csnd", "gatyd", "gatnd", // 分省月度/分省季度/分省年度/城市月度价格/城市年度数据/港澳台月度/港澳台年度
-		"gjyd", "gjydsdj", "gjydsc", "gjnd", // 主要国家月度/三大经济体月度/国际市场月度商品/国家年度
-	}
-	// 含地区维度
-	regCodes := []string{
-		"fsyd", "fsjd", "fsnd", "csyd", "csnd", "gatyd", "gatnd", // 分省月度/分省季度/分省年度/城市月度价格/城市年度数据/港澳台月度/港澳台年度
-		"gjyd", "gjydsdj", "gjydsc", "gjnd", // 主要国家月度/三大经济体月度/国际市场月度商品/国家年度
-	}
+	_ = SyncXDateYQuotaZRegDb()
 
-	for _, code := range dbCodes {
-		fmt.Println("开始同步DbCode: ", code)
+	_ = SyncXRegYDateZQuotaDb()
 
-		// 指标维度
-		items := make([]*models.BaseFromNationalStatisticsClassify, 0)
-		resp, e := curlAndFormatQuotaClassify("zb", code, "zb", items)
-		if e != nil {
-			err = fmt.Errorf("递归指标分类失败, Err: %s", e.Error())
-			return
-		}
-		items = resp
-
-		// 地区维度
-		if utils.InArrayByStr(regCodes, code) {
-			regResp, e := curlAndFormatQuotaClassify("reg", code, "reg", items)
-			if e != nil {
-				err = fmt.Errorf("递归地区分类失败, Err: %s", e.Error())
-				return
-			}
-			items = append(items, regResp...)
-		}
-
-		// 去重-code+维度code+ID
-		classifyMap := make(map[string]bool)
-		classifyOB := new(models.BaseFromNationalStatisticsClassify)
-		classifyPars := make([]interface{}, 0)
-		classifies, e := classifyOB.GetItemsByCondition("", classifyPars, []string{}, "")
-		if e != nil {
-			err = fmt.Errorf("获取指标分类列表失败, Err: %s", e.Error())
-			return
-		}
-		for _, c := range classifies {
-			classifyMap[fmt.Sprintf("%s%s%s", code, c.Wdcode, c.Id)] = true
-		}
-
-		finalList := make([]*models.BaseFromNationalStatisticsClassify, 0)
-		for _, v := range items {
-			if classifyMap[fmt.Sprintf("%s%s%s", code, v.Wdcode, v.Id)] {
-				continue
-			}
-			finalList = append(finalList, v)
-		}
-		if e = classifyOB.CreateMulti(items); e != nil {
-			err = fmt.Errorf("批量新增指标分类失败, Err: %s", e.Error())
-			return
-		}
-		fmt.Println("结束同步DbCode: ", code)
-	}
+	utils.FileLog.Info("统计局数据刷新成功")
 	return
 }
 
-// curlAndFormatQuotaClassify 递归请求分类树
-func curlAndFormatQuotaClassify(id, dbcode, wdcode string, items []*models.BaseFromNationalStatisticsClassify) (resp []*models.BaseFromNationalStatisticsClassify, err error) {
-	f := url.Values{}
-	f.Add("id", id)
-	f.Add("dbcode", dbcode)
-	f.Add("wdcode", wdcode)
-	f.Add("m", "getTree")
-	r, e := NationalHttpPost(NationalStatisticsBaseReqUrl, f.Encode())
-	if e != nil {
-		err = fmt.Errorf("请求指标分类树失败, Err: %s", e.Error())
-		return
-	}
-	list := make([]*QuotaClassifyTreeResp, 0)
-	if e = json.Unmarshal(r, &list); e != nil {
-		err = fmt.Errorf("解析响应数据失败, Err: %s", e.Error())
-		return
-	}
-
-	nowTime := time.Now().Local()
-	for _, v := range list {
-		isParent := 0
-		if v.IsParent {
-			isParent = 1
-		}
-		fmt.Printf("Dbcode: %s, ClassifyName: %s, Id: %s\n", v.Dbcode, v.Name, v.Id)
-		items = append(items, &models.BaseFromNationalStatisticsClassify{
-			ClassifyName: v.Name,
-			Id:           v.Id,
-			Dbcode:       v.Dbcode,
-			Wdcode:       v.Wdcode,
-			Pid:          v.Pid,
-			IsParent:     isParent,
-			CreateTime:   nowTime,
-			ModifyTime:   nowTime,
-		})
-
-		// 向下递归
-		if isParent == 1 {
-			time.Sleep(5 * time.Second) // 缓缓...毕竟接口是人家的...
-			items, e = curlAndFormatQuotaClassify(v.Id, v.Dbcode, v.Wdcode, items)
-			if e != nil {
-				err = fmt.Errorf("递归请求分类树失败, Err: %s", e.Error())
-				return
-			}
-		}
-	}
-	return items, nil
-}
-
-// DataApiReq 数据接口请求体
-type DataApiReq struct {
-	Method    string `description:"方法: QueryData-查询数据; getOtherWds-获取其他维度" json:"method"`
-	DbCode    string `description:"数据库编码" json:"dbcode"`
-	RowCode   string `description:"行-维度: zb; sj; reg" json:"rowcode"`
-	ColCode   string `description:"列-维度: zb; sj; reg" json:"colcode"`
-	WdsList   []Wds  `description:"维度列表" json:"wdsList"`
-	DfwdsList []Wds  `description:"df不知道啥意思...反正也是维度相关的" json:"dfwdsList"`
-}
-
-// Wds 维度
-type Wds struct {
-	WdCode    string `description:"维度: zb-指标; sj-时间; reg-地区" json:"wdcode"`
-	ValueCode string `description:"维度编码" json:"valuecode"`
-}
-
-// CommonDataApiRequest 数据接口请求
-func CommonDataApiRequest(req DataApiReq) (resp QuotaListDataResp, err error) {
-	if req.DbCode == "" {
-		return
-	}
-	if req.Method == "" {
-		req.Method = "QueryData"
-	}
-	if req.RowCode == "" {
-		req.RowCode = "zb"
-	}
-	if req.ColCode == "" {
-		req.ColCode = "sj"
-	}
-
-	// 构建查询
-	f := url.Values{}
-	f.Add("m", req.Method)
-	f.Add("dbcode", req.DbCode)
-	f.Add("rowcode", req.RowCode)
-	f.Add("colcode", req.ColCode)
-	wds := `[]`
-	if len(req.WdsList) > 0 {
-		wdsByte, e := json.Marshal(req.WdsList)
-		if e != nil {
-			err = fmt.Errorf("wds json marshal err: %s", e.Error())
-			return
-		}
-		wds = string(wdsByte)
-	}
-	dfwds := `[]`
-	if len(req.DfwdsList) > 0 {
-		dfwdsByte, e := json.Marshal(req.DfwdsList)
-		if e != nil {
-			err = fmt.Errorf("dfwds json marshal err: %s", e.Error())
-			return
-		}
-		dfwds = string(dfwdsByte)
-	}
-	f.Add("wds", wds)
-	f.Add("dfwds", dfwds)
-	f.Add("k1", fmt.Sprint(time.Now().UnixNano()/1e6))
-	f.Add("h", "1")
-
-	// 响应
-	r, e := NationalHttpPost(NationalStatisticsBaseReqUrl, f.Encode())
-	if e != nil {
-		err = fmt.Errorf("http request err: %s", e.Error())
-		return
-	}
-	utils.FileLog.Info("result: %s", string(r))
-	if e = json.Unmarshal(r, &resp); e != nil {
-		err = fmt.Errorf("resp unmarshal err: %s", e.Error())
-		return
-	}
-	return
-}
-
-// QuotaListDataResp 指标数据列表响应体
-type QuotaListDataResp struct {
-	ReturnCode int `description:"状态码" json:"returncode"`
-	ReturnData struct {
-		DataNodes []QuotaDataNode `json:"datanodes"`
-		WdNodes   []QuotaWdNode   `json:"wdnodes"`
-	}
-}
-
-// QuotaDataNode 指标数据节点
-type QuotaDataNode struct {
-	Code string `description:"编码"`
-	Data struct {
-		Data    float64 `description:"指标值"`
-		HasData bool    `description:"是否有值" json:"hasdata"`
-		StrData string  `description:"指标值(字符串)" json:"strdata"`
-	}
-	Wds []Wds
-}
-
-//// QuotaDataWds 指标数据对应的维度信息
-//type QuotaDataWds struct {
-//	ValueCode string `json:"valuecode"`
-//	WdCode    string `json:"wdcode"`
-//}
-
-// QuotaWdNode 维度节点
-type QuotaWdNode struct {
-	WdCode string `description:"示例: zb; sj; reg;" json:"wdcode"`
-	WdName string `description:"示例: 指标; 时间; 地区" json:"wdname"`
-	Nodes  []QuotaWdNodeData
-}
-
-// QuotaWdNodeData 维度节点数据
-type QuotaWdNodeData struct {
-	Code     string `description:"指标编码"`
-	Name     string `description:"指标名称"`
-	Unit     string `description:"单位"`
-	SortCode int    `description:"编码排序" json:"sortcode"`
-}
-
-// SyncMonth2YearDbQuota 同步月度/季度/年度数据指标
-func SyncMonth2YearDbQuota() (err error) {
+// SyncXDateYQuotaDb 同步两维度X轴-日期, Y轴-指标数据库(月度/季度/年度数据指标)
+func SyncXDateYQuotaDb() (err error) {
 	defer func() {
 		if err != nil {
-			utils.FileLog.Error("统计局-同步月度/季度/年度数据指标, ErrMsg: %s", err.Error())
-			go alarm_msg.SendAlarmMsg(fmt.Sprintf("统计局-同步月度/季度/年度数据指标, ErrMsg: %s", err.Error()), 3)
+			utils.FileLog.Error("统计局-同步月度/季度/年度数据失败, ErrMsg: %s", err.Error())
+			go alarm_msg.SendAlarmMsg(fmt.Sprintf("统计局-同步月度/季度/年度数据失败, ErrMsg: %s", err.Error()), 3)
+			return
 		}
+		utils.FileLog.Info("统计局-同步月度/季度/年度数据成功")
 	}()
 
 	// 查询无父级的指标分类
 	classifyOB := new(models.BaseFromNationalStatisticsClassify)
 	classifyCond := ` AND is_parent = 0 AND dbcode IN ('hgyd', 'hgjd', 'hgnd')`
-	//classifyCond := ` AND is_parent = 0 AND dbcode IN ('hgnd')`
 	classifyPars := make([]interface{}, 0)
 	classifyOrder := ` base_from_national_statistics_classify_id ASC`
 	classifyList, e := classifyOB.GetItemsByCondition(classifyCond, classifyPars, []string{}, classifyOrder)
@@ -362,24 +45,25 @@ func SyncMonth2YearDbQuota() (err error) {
 		err = fmt.Errorf("获取指标分类列表失败, Err: %s", e.Error())
 		return
 	}
-	fmt.Println("分类长度: ", len(classifyList))
+	utils.FileLog.Info("分类长度: %d\n", len(classifyList))
 
 	// 同步指标和数据
 	for _, c := range classifyList {
-		fmt.Printf("开始同步分类: %s-%s-%s\n", c.ClassifyName, c.Dbcode, c.Id)
-		if e = SyncMonth2YearIndexAndDataByClassify(c.BaseFromNationalStatisticsClassifyId, c.Dbcode, c.Id); e != nil {
-			err = fmt.Errorf("同步指标数据失败, Err: %s", e.Error())
+		if e = SyncXDateYQuotaData(c.BaseFromNationalStatisticsClassifyId, c.Dbcode, c.Id); e != nil {
+			err = fmt.Errorf("同步指标数据失败, DbCode: %s, ClassifyId: %d, Err: %s", c.Dbcode, c.BaseFromNationalStatisticsClassifyId, e.Error())
 			return
 		}
-		fmt.Printf("结束同步分类: %s-%s-%s\n", c.ClassifyName, c.Dbcode, c.Id)
-		// TODO:只测试一个
-		break
 	}
 	return
 }
 
-// SyncMonth2YearIndexAndDataByClassify 同步月度/季度/年度数据-分类下的指标
-func SyncMonth2YearIndexAndDataByClassify(classifyId int, dbCode, classifyCode string) (err error) {
+// SyncXDateYQuotaData 同步两维度X轴-日期, Y轴-指标的数据
+func SyncXDateYQuotaData(classifyId int, dbCode, classifyCode string) (err error) {
+	defer func() {
+		if err != nil {
+			utils.FileLog.Error("同步数据库DbCode: %s, 分类ClassifyId: %d失败, ErrMsg: %s", dbCode, classifyId, err.Error())
+		}
+	}()
 	// yd-月度 jd-季度 nd-年度
 	frequency := ""
 	timeParam := ""
@@ -398,8 +82,7 @@ func SyncMonth2YearIndexAndDataByClassify(classifyId int, dbCode, classifyCode s
 
 	var dataReq DataApiReq
 	dataReq.DbCode = dbCode
-	var dfwdsList []Wds
-	dfwdsList = append(dfwdsList, Wds{
+	dataReq.DfwdsList = append(dataReq.DfwdsList, Wds{
 		WdCode:    "zb",
 		ValueCode: classifyCode,
 	}, Wds{
@@ -528,113 +211,51 @@ func SyncMonth2YearIndexAndDataByClassify(classifyId int, dbCode, classifyCode s
 	return
 }
 
-// formatMonth2YearDateCode 将日期code转为对应日期
-func formatMonth2YearDateCode(dateCode string) (date time.Time, err error) {
-	if dateCode == "" {
-		return
-	}
-	// 根据日期code长度进行区分, 格式为三种: 月度-200601; 季度-2006A; 年度-2006
-	switch len([]rune(dateCode)) {
-	case 6:
-		t, e := time.ParseInLocation("200601", dateCode, time.Local)
-		if e != nil {
-			err = fmt.Errorf("月度指标日期转换失败, Err: %s", e.Error())
+// SyncXDateYQuotaZRegDb 同步三维度X轴-日期, Y轴-指标, Z轴-地区的数据库(分省月季年度、主要城市月年度、国际市场月度商品价格)
+func SyncXDateYQuotaZRegDb() (err error) {
+	defer func() {
+		if err != nil {
+			utils.FileLog.Error("统计局-同步分省月季年度、主要城市月年度、国际市场月度商品价格失败, ErrMsg: %s", err.Error())
+			go alarm_msg.SendAlarmMsg(fmt.Sprintf("统计局-同步分省月季年度、主要城市月年度、国际市场月度商品价格失败, ErrMsg: %s", err.Error()), 3)
 			return
 		}
-		date = t
-		break
-	case 5:
-		// 季度ABCD转换成对应日期
-		dateSuffixMap := map[string]string{
-			"A": "03-31",
-			"B": "06-30",
-			"C": "09-30",
-			"D": "12-31",
-		}
-		dateCode = strings.ToUpper(dateCode)
-		quarterTab := dateCode[4:]
-		dateStr := fmt.Sprintf("%s-%s", dateCode[:4], dateSuffixMap[quarterTab])
-		t, e := time.ParseInLocation(utils.FormatDate, dateStr, time.Local)
+		utils.FileLog.Info("统计局-同步分省月季年度、主要城市月年度、国际市场月度商品价格成功")
+	}()
+
+	// 需要同步的数据库
+	dbCodes := []string{"fsyd", "fsjd", "fsnd", "csyd", "csnd", "gjydsc"}
+	for _, d := range dbCodes {
+		classifyOB := new(models.BaseFromNationalStatisticsClassify)
+		// 注意此处只需要同步分类中为指标的即可, 分类为地区的数据在指标中均有包含
+		classifyCond := ` AND is_parent = 0 AND wdcode = 'zb' AND dbcode = ?`
+		classifyPars := make([]interface{}, 0)
+		classifyPars = append(classifyPars, d)
+		classifyOrder := ` base_from_national_statistics_classify_id ASC`
+		classifyList, e := classifyOB.GetItemsByCondition(classifyCond, classifyPars, []string{}, classifyOrder)
 		if e != nil {
-			err = fmt.Errorf("季度指标日期转换失败, Err: %s", e.Error())
+			err = fmt.Errorf("获取%s分类列表失败, Err: %s", d, e.Error())
 			return
 		}
-		date = t
-		break
-	case 4:
-		dateStr := fmt.Sprintf("%s-%s", dateCode, "12-31")
-		t, e := time.ParseInLocation(utils.FormatDate, dateStr, time.Local)
-		if e != nil {
-			err = fmt.Errorf("年度指标日期转换失败, Err: %s", e.Error())
+		utils.FileLog.Info("%s分类长度: %d\n", d, len(classifyList))
+
+		// 查询其他维度-地区
+		wdList, e := GetOtherWd(d, "", "")
+		var regList []OtherWdNodes
+		for _, wd := range wdList {
+			if wd.WdCode == "reg" {
+				regList = wd.Nodes
+				break
+			}
+		}
+		if len(regList) == 0 {
+			err = fmt.Errorf("其他维度为空, DbCode: %s", d)
 			return
 		}
-		date = t
-		break
-	default:
-		err = fmt.Errorf("日期code格式有误, code: %s", dateCode)
-		return
-	}
-	return
-}
-
-// OtherWdResp 其他维度信息响应体
-type OtherWdResp struct {
-	ReturnCode int `description:"状态码" json:"returncode"`
-	ReturnData []struct {
-		IsSj   bool           `description:"是否为时间" json:"issj"`
-		WdCode string         `description:"维度编码" json:"wdcode"`
-		WdName string         `description:"维度名称" json:"wdname"`
-		Nodes  []OtherWdNodes `description:"维度数据" json:"nodes"`
-	} `description:"响应数据" json:"returndata"`
-}
-
-type OtherWdNodes struct {
-	Code string `description:"编码" json:"code"`
-	Name string `description:"名称" json:"name"`
-	Sort string `description:"排序" json:"sort"`
-}
-
-// GetDbOtherWdInfo 获取Db下其他维度信息
-func GetDbOtherWdInfo(dbCode, wdCode string) (nodes []OtherWdNodes, err error) {
-	defer func() {
-		fmt.Println("维度信息 end")
-	}()
-	fmt.Println("维度信息 start")
-
-	// 默认获取地区维度
-	if wdCode == "" {
-		wdCode = "reg"
-	}
-	// 构建查询
-	f := url.Values{}
-	f.Add("m", "getOtherWds")
-	f.Add("dbcode", dbCode)
-	f.Add("rowcode", "zb")
-	f.Add("colcode", "sj")
-	f.Add("wds", `[]`)
-	f.Add("k1", fmt.Sprint(time.Now().UnixNano()/1e6))
-	f.Add("h", "1")
-	r, e := NationalHttpPost(NationalStatisticsBaseReqUrl, f.Encode())
-	if e != nil {
-		err = fmt.Errorf("请求其他维度信息失败, Err: %s", e.Error())
-		return
-	}
-	utils.FileLog.Info("GetOtherWdInfo Result: %s", string(r))
 
-	// 响应
-	resp := new(OtherWdResp)
-	if e = json.Unmarshal(r, &resp); e != nil {
-		err = fmt.Errorf("其他维度信息Unmarshal Err: %s", e.Error())
-		return
-	}
-	if resp == nil {
-		err = fmt.Errorf("其他维度信息请求结果为空")
-		return
-	}
-	if resp.ReturnCode == 200 {
-		for _, d := range resp.ReturnData {
-			if d.WdCode == wdCode {
-				nodes = d.Nodes
+		// 同步指标和数据
+		for _, c := range classifyList {
+			if e = SyncXDateYQuotaZRegData(c.BaseFromNationalStatisticsClassifyId, c.Dbcode, c.Id, regList); e != nil {
+				err = fmt.Errorf("同步指标数据失败, DbCode: %s, ClassifyId: %d, Err: %s", c.Dbcode, c.BaseFromNationalStatisticsClassifyId, e.Error())
 				return
 			}
 		}
@@ -642,70 +263,15 @@ func GetDbOtherWdInfo(dbCode, wdCode string) (nodes []OtherWdNodes, err error) {
 	return
 }
 
-// SyncRegQuota TODO:同步地区指标
-func SyncRegQuota() (err error) {
+// SyncXDateYQuotaZRegData 同步三维度X轴-日期, Y轴-指标, Z轴-地区的数据
+func SyncXDateYQuotaZRegData(classifyId int, dbCode, classifyCode string, regList []OtherWdNodes) (err error) {
 	defer func() {
 		if err != nil {
-			utils.FileLog.Error("统计局-同步地区指标失败, ErrMsg: %s", err.Error())
-			go alarm_msg.SendAlarmMsg(fmt.Sprintf("统计局-同步地区指标失败, ErrMsg: %s", err.Error()), 3)
-		}
-	}()
-
-	// 查询无父级的指标分类
-	//"fsyd", "fsjd", "fsnd", "csyd", "csnd", "gatyd", "gatnd"
-	classifyOB := new(models.BaseFromNationalStatisticsClassify)
-	// 注意此处只需要同步分类中为指标的即可, 分类为地区的数据在指标中均有包含
-	//classifyCond := ` AND is_parent = 0 AND wdcode = 'zb' AND dbcode IN ('fsyd', 'fsjd', 'fsnd', 'csyd', 'csnd')`
-	classifyCond := ` AND is_parent = 0 AND wdcode = 'zb' AND dbcode IN ('fsjd')`
-	classifyPars := make([]interface{}, 0)
-	classifyOrder := ` base_from_national_statistics_classify_id ASC`
-	classifyList, e := classifyOB.GetItemsByCondition(classifyCond, classifyPars, []string{}, classifyOrder)
-	if e != nil {
-		err = fmt.Errorf("获取指标分类列表失败, Err: %s", e.Error())
-		return
-	}
-	fmt.Println("分类长度: ", len(classifyList))
-
-	// 其他维度
-	dbCodes := make([]string, 0)
-	for _, c := range classifyList {
-		if !utils.InArrayByStr(dbCodes, c.Dbcode) {
-			dbCodes = append(dbCodes, c.Dbcode)
-		}
-	}
-	fmt.Println("DbCode长度: ", len(dbCodes))
-
-	dbWdMap := make(map[string][]OtherWdNodes)
-	for _, dc := range dbCodes {
-		otherWd, e := GetDbOtherWdInfo(dc, "reg")
-		if e != nil {
-			err = fmt.Errorf("获取其他维度失败, DbCode: %s, Err: %s", dc, e.Error())
-			return
-		}
-		dbWdMap[dc] = otherWd
-	}
-
-	// 同步指标和数据
-	for _, c := range classifyList {
-		regList := dbWdMap[c.Dbcode]
-		if e = SyncRegIndexAndDataByClassify(c.BaseFromNationalStatisticsClassifyId, c.Dbcode, c.Id, regList); e != nil {
-			err = fmt.Errorf("同步分类下的指标和数据失败, Err: %s", e.Error())
-			return
+			utils.FileLog.Error("同步数据库DbCode: %s, 分类ClassifyId: %d失败, ErrMsg: %s", dbCode, classifyId, err.Error())
 		}
-		// TODO:测试一个
-		break
-	}
-	return
-}
-
-// SyncRegIndexAndDataByClassify 同步地区数据-分类下的指标
-func SyncRegIndexAndDataByClassify(classifyId int, dbCode, classifyCode string, regList []OtherWdNodes) (err error) {
-	defer func() {
-		fmt.Println("同步地区数据-分类指标 end")
 	}()
-	fmt.Println("同步地区数据-分类指标 start")
 
-	// yd-月度 jd-季度 nd-年度
+	// 根据DbCode判断频度和查询的时间区间
 	frequency := ""
 	timeParam := ""
 	if strings.Contains(dbCode, "yd") {
@@ -723,27 +289,22 @@ func SyncRegIndexAndDataByClassify(classifyId int, dbCode, classifyCode string,
 
 	// 遍历地区维度, 查询指标和数据
 	for _, reg := range regList {
-		fmt.Printf("同步地区数据-分类指标-%s start\n", reg.Name)
-
-		// 构建查询
-		f := url.Values{}
-		f.Add("m", "QueryData")
-		f.Add("dbcode", dbCode)
-		f.Add("rowcode", "zb")
-		f.Add("colcode", "sj")
-		f.Add("wds", fmt.Sprintf(`[{"wdcode":"reg","valuecode":"%s"}]`, reg.Code))
-		f.Add("dfwds", fmt.Sprintf(`[{"wdcode":"zb","valuecode":"%s"},{"wdcode":"sj","valuecode":"%s"}]`, classifyCode, timeParam))
-		f.Add("k1", fmt.Sprint(time.Now().UnixNano()/1e6))
-		f.Add("h", "1")
-		r, e := NationalHttpPost(NationalStatisticsBaseReqUrl, f.Encode())
+		var dataReq DataApiReq
+		dataReq.DbCode = dbCode
+		dataReq.WdsList = append(dataReq.WdsList, Wds{
+			WdCode:    "reg",
+			ValueCode: reg.Code,
+		})
+		dataReq.DfwdsList = append(dataReq.DfwdsList, Wds{
+			WdCode:    "zb",
+			ValueCode: classifyCode,
+		}, Wds{
+			WdCode:    "sj",
+			ValueCode: timeParam,
+		})
+		resp, e := CommonDataApiRequest(dataReq)
 		if e != nil {
-			err = fmt.Errorf("请求分类下的指标失败, Err: %s", e.Error())
-			return
-		}
-
-		resp := new(QuotaListDataResp)
-		if e = json.Unmarshal(r, &resp); e != nil {
-			fmt.Println("Unmarshal Err: ", e.Error())
+			err = fmt.Errorf("获取分类下的指标数据失败, Err: %s", e.Error())
 			return
 		}
 
@@ -863,67 +424,57 @@ func SyncRegIndexAndDataByClassify(classifyId int, dbCode, classifyCode string,
 				return
 			}
 		}
-		fmt.Printf("同步地区数据-分类指标-%s end\n", reg.Name)
-		// TODO:测试一个
-		break
 	}
 	return
 }
 
-// SyncOtherQuota TODO:同步港澳台、国际数据指标
-func SyncOtherQuota() (err error) {
+// SyncXRegYDateZQuotaDb 同步三维度X轴-地区, Y轴-日期的数据库(港澳台、国际数据指标)
+func SyncXRegYDateZQuotaDb() (err error) {
 	defer func() {
 		if err != nil {
 			utils.FileLog.Error("统计局-同步港澳台、国际数据指标失败, ErrMsg: %s", err.Error())
 			go alarm_msg.SendAlarmMsg(fmt.Sprintf("统计局-同步港澳台、国际数据指标失败, ErrMsg: %s", err.Error()), 3)
+			return
 		}
+		utils.FileLog.Info("统计局-同步港澳台、国际数据指标成功")
 	}()
 
-	// 查询无父级的指标分类
-	classifyOB := new(models.BaseFromNationalStatisticsClassify)
-	// 注意此处只需要同步分类中为指标的即可, 分类为地区的数据在指标中均有包含
-	//classifyCond := ` AND is_parent = 0 AND wdcode = 'zb' AND dbcode IN ('gatyd', 'gatnd', 'gjyd', 'gjydsdj', 'gjydsc', 'gjnd')`
-	classifyCond := ` AND is_parent = 0 AND wdcode = 'zb' AND dbcode IN ('gatyd', 'gatnd')`
-	classifyPars := make([]interface{}, 0)
-	classifyOrder := ` base_from_national_statistics_classify_id ASC`
-	classifyList, e := classifyOB.GetItemsByCondition(classifyCond, classifyPars, []string{}, classifyOrder)
-	if e != nil {
-		err = fmt.Errorf("获取指标分类列表失败, Err: %s", e.Error())
-		return
-	}
-	fmt.Println("分类长度: ", len(classifyList))
-
-	// 同步指标和数据
-	for _, c := range classifyList {
-		if e = SyncOtherIndexAndDataByClassify(c.BaseFromNationalStatisticsClassifyId, c.Dbcode, c.Id); e != nil {
-			err = fmt.Errorf("同步分类下的指标和数据失败, Err: %s", e.Error())
+	// 需要同步的数据库
+	dbCodes := []string{"gatyd", "gatnd", "gjyd", "gjydsdj", "gjnd"}
+	for _, d := range dbCodes {
+		classifyOB := new(models.BaseFromNationalStatisticsClassify)
+		// 注意此处只需要同步分类中为指标的即可, 分类为地区的数据在指标中均有包含
+		classifyCond := ` AND is_parent = 0 AND wdcode = 'zb' AND dbcode = ?`
+		classifyPars := make([]interface{}, 0)
+		classifyPars = append(classifyPars, d)
+		classifyOrder := ` base_from_national_statistics_classify_id ASC`
+		classifyList, e := classifyOB.GetItemsByCondition(classifyCond, classifyPars, []string{}, classifyOrder)
+		if e != nil {
+			err = fmt.Errorf("获取%s分类列表失败, Err: %s", d, e.Error())
 			return
 		}
-		// TODO:测试一个
-		break
+		utils.FileLog.Info("%s分类长度: %d\n", d, len(classifyList))
+
+		// 同步指标和数据
+		for _, c := range classifyList {
+			if e = SyncXRegYDateZQuotaDbData(c.BaseFromNationalStatisticsClassifyId, c.Dbcode, c.Id); e != nil {
+				err = fmt.Errorf("同步指标数据失败, DbCode: %s, ClassifyId: %d, Err: %s", c.Dbcode, c.BaseFromNationalStatisticsClassifyId, e.Error())
+				return
+			}
+		}
 	}
 	return
 }
 
-// SyncOtherIndexAndDataByClassify 同步港澳台、国际数据指标-分类下的指标
-func SyncOtherIndexAndDataByClassify(classifyId int, dbCode, classifyCode string) (err error) {
+// SyncXRegYDateZQuotaDbData 同步三维度X轴-地区, Y轴-日期, Z轴-指标的数据
+func SyncXRegYDateZQuotaDbData(classifyId int, dbCode, classifyCode string) (err error) {
 	defer func() {
-		fmt.Println("同步港澳台、国际数据指标-分类指标 end")
+		if err != nil {
+			utils.FileLog.Error("同步数据库DbCode: %s, 分类ClassifyId: %d失败, ErrMsg: %s", dbCode, classifyId, err.Error())
+		}
 	}()
-	fmt.Println("同步港澳台、国际数据指标-分类指标 start")
-
-	// 先执行一次默认的, 取出其中的地区维度, 再遍历地区维度, 进行指标的搜索
-
-	// m: QueryData
-	//dbcode: gatyd
-	//rowcode: sj
-	//colcode: reg
-	//wds: [{"wdcode":"zb","valuecode":"A010A"}]
-	//dfwds: []
-	//k1: 1679994758636
-	//h: 1
 
-	// yd-月度 jd-季度 nd-年度
+	// 根据DbCode判断频度和查询的时间区间
 	frequency := ""
 	timeParam := ""
 	if strings.Contains(dbCode, "yd") {
@@ -939,29 +490,63 @@ func SyncOtherIndexAndDataByClassify(classifyId int, dbCode, classifyCode string
 		frequency = "年度"
 	}
 
-	// 遍历地区维度, 查询指标和数据
-	for _, reg := range regList {
-		fmt.Printf("同步地区数据-分类指标-%s start\n", reg.Name)
-
-		// 构建查询
-		f := url.Values{}
-		f.Add("m", "QueryData")
-		f.Add("dbcode", dbCode)
-		f.Add("rowcode", "zb")
-		f.Add("colcode", "sj")
-		f.Add("wds", fmt.Sprintf(`[{"wdcode":"reg","valuecode":"%s"}]`, reg.Code))
-		f.Add("dfwds", fmt.Sprintf(`[{"wdcode":"zb","valuecode":"%s"},{"wdcode":"sj","valuecode":"%s"}]`, classifyCode, timeParam))
-		f.Add("k1", fmt.Sprint(time.Now().UnixNano()/1e6))
-		f.Add("h", "1")
-		r, e := NationalHttpPost(NationalStatisticsBaseReqUrl, f.Encode())
+	// 先以指标作为行进行默认查询, 取出其中的指标作为当前分类的指标
+	//f := url.Values{}
+	//f.Add("m", "QueryData")
+	//f.Add("dbcode", "gatyd")
+	//f.Add("rowcode", "zb")
+	//f.Add("colcode", "reg")
+	//f.Add("wds", `[{"wdcode":"sj","valuecode":"LAST36"}]`)
+	//f.Add("dfwds", `[{"wdcode":"zb","valuecode":"A01"}]`)
+	var defaultReq DataApiReq
+	defaultReq.DbCode = dbCode
+	defaultReq.RowCode = "zb"
+	defaultReq.ColCode = "reg"
+	defaultReq.WdsList = append(defaultReq.WdsList, Wds{
+		WdCode:    "sj",
+		ValueCode: timeParam,
+	})
+	defaultReq.DfwdsList = append(defaultReq.DfwdsList, Wds{
+		WdCode:    "zb",
+		ValueCode: classifyCode,
+	})
+	defaultResult, e := CommonDataApiRequest(defaultReq)
+	if e != nil {
+		err = fmt.Errorf("默认查询数据失败, Err: %s", e.Error())
+		return
+	}
+	var quotaWds []QuotaWdNodeData
+	for _, n := range defaultResult.ReturnData.WdNodes {
+		if n.WdCode == "zb" {
+			quotaWds = n.Nodes
+			break
+		}
+	}
+
+	// 遍历指标维度
+	for _, quota := range quotaWds {
+		//f := url.Values{}
+		//f.Add("m", "QueryData")
+		//f.Add("dbcode", "gatyd")
+		//f.Add("rowcode", "sj")
+		//f.Add("colcode", "reg")
+		//f.Add("wds", `[{"wdcode":"zb","valuecode":"A010A"}]`)
+		//f.Add("dfwds", `[{"wdcode":"sj","valuecode":"LAST36"}]`)
+		var dataReq DataApiReq
+		dataReq.DbCode = dbCode
+		dataReq.RowCode = "sj"
+		dataReq.ColCode = "reg"
+		dataReq.WdsList = append(defaultReq.WdsList, Wds{
+			WdCode:    "zb",
+			ValueCode: quota.Code,
+		})
+		dataReq.DfwdsList = append(defaultReq.DfwdsList, Wds{
+			WdCode:    "sj",
+			ValueCode: timeParam,
+		})
+		resp, e := CommonDataApiRequest(dataReq)
 		if e != nil {
-			err = fmt.Errorf("请求分类下的指标失败, Err: %s", e.Error())
-			return
-		}
-
-		resp := new(QuotaListDataResp)
-		if e = json.Unmarshal(r, &resp); e != nil {
-			fmt.Println("Unmarshal Err: ", e.Error())
+			err = fmt.Errorf("查询数据失败, Err: %s", e.Error())
 			return
 		}
 
@@ -974,14 +559,19 @@ func SyncOtherIndexAndDataByClassify(classifyId int, dbCode, classifyCode string
 
 		// 取出指标(Y轴), 日期(X轴)
 		wdNodes := resp.ReturnData.WdNodes
-		var quotaNodes, dateNodes []QuotaWdNodeData
+		//var quotaNodes, dateNodes, regNodes []QuotaWdNodeData
+		var dateNodes, regNodes []QuotaWdNodeData
 		for _, w := range wdNodes {
-			if w.WdCode == "zb" {
-				quotaNodes = w.Nodes
-				continue
-			}
-			if w.WdCode == "sj" {
+			switch w.WdCode {
+			case "zb":
+				//quotaNodes = w.Nodes
+				break
+			case "sj":
 				dateNodes = w.Nodes
+				break
+			case "reg":
+				regNodes = w.Nodes
+				break
 			}
 		}
 
@@ -1014,17 +604,17 @@ func SyncOtherIndexAndDataByClassify(classifyId int, dbCode, classifyCode string
 		// 遍历XY轴
 		indexDataList := make([]*models.SaveNationalStatisticsIndexAndDataReq, 0)
 		indexDataMap := make(map[string][]*models.BaseFromNationalStatisticsData)
-		for _, q := range quotaNodes {
-			// dbcode+指标code+地区code
-			indexCode := fmt.Sprintf("%s%s%s", dbCode, q.Code, reg.Code)
 
-			// 指标
+		// 遍历X轴-地区
+		for _, reg := range regNodes {
+			// 指标: dbcode+指标code+地区code
+			indexCode := fmt.Sprintf("%s%s%s", dbCode, quota.Code, reg.Code)
 			r := new(models.SaveNationalStatisticsIndexAndDataReq)
 			r.Index = &models.BaseFromNationalStatisticsIndex{
 				BaseFromNationalStatisticsClassifyId: classifyId,
 				Dbcode:                               dbCode,
 				IndexCode:                            indexCode,
-				IndexName:                            q.Name,
+				IndexName:                            quota.Name,
 				Frequency:                            frequency,
 				Reg:                                  reg.Name,
 				CreateTime:                           time.Now().Local(),
@@ -1034,10 +624,9 @@ func SyncOtherIndexAndDataByClassify(classifyId int, dbCode, classifyCode string
 				r.IndexExist = true
 			}
 
-			// 数据
-			// zb.A01010201_reg.110000_sj.201608
+			// 遍历Y轴-日期
 			for _, d := range dateNodes {
-				k := fmt.Sprintf("%s.%s_%s.%s_%s.%s", "zb", q.Code, "reg", reg.Code, "sj", d.Code)
+				k := fmt.Sprintf("%s.%s_%s.%s_%s.%s", "zb", quota.Code, "reg", reg.Code, "sj", d.Code)
 				v := dataMap[k]
 				if !v.Data.HasData {
 					continue
@@ -1081,54 +670,6 @@ func SyncOtherIndexAndDataByClassify(classifyId int, dbCode, classifyCode string
 				return
 			}
 		}
-		fmt.Printf("同步地区数据-分类指标-%s end\n", reg.Name)
-		// TODO:测试一个
-		break
-	}
-	return
-}
-
-func ApiTest() (err error) {
-	defer func() {
-		if err != nil {
-			fmt.Println(err.Error())
-		}
-	}()
-
-	dbCode := "gatyd"
-	classifyCode := "A0107"
-	wdCode := "121"
-	timeParam := "LAST36"
-
-	// m: QueryData
-	//dbcode: gatyd
-	//rowcode: zb
-	//colcode: reg
-	//wds: [{"wdcode":"sj","valuecode":"202302"}]
-	//dfwds: []
-	//k1: 1679992128420
-	//h: 1
-
-	f := url.Values{}
-	f.Add("m", "QueryData")
-	f.Add("dbcode", dbCode)
-	f.Add("rowcode", "zb")
-	f.Add("colcode", "sj")
-	f.Add("wds", fmt.Sprintf(`[{"wdcode":"reg","valuecode":"%s"}]`, wdCode))
-	f.Add("dfwds", fmt.Sprintf(`[{"wdcode":"zb","valuecode":"%s"},{"wdcode":"sj","valuecode":"%s"}]`, classifyCode, timeParam))
-	f.Add("k1", fmt.Sprint(time.Now().UnixNano()/1e6))
-	f.Add("h", "1")
-	r, e := NationalHttpPost(NationalStatisticsBaseReqUrl, f.Encode())
-	if e != nil {
-		err = fmt.Errorf("请求分类下的指标失败, Err: %s", e.Error())
-		return
-	}
-	utils.FileLog.Info("result: %s", string(r))
-
-	resp := new(QuotaListDataResp)
-	if e = json.Unmarshal(r, &resp); e != nil {
-		fmt.Println("Unmarshal Err: ", e.Error())
-		return
 	}
 	return
 }

services/task.go

@@ -5,22 +5,23 @@ import (
 	"fmt"
 	"github.com/beego/beego/v2/task"
 	"hongze/hongze_data_crawler/services/national_data"
-	"time"
 )
 
-func Task123() {
+func Task() {
 	fmt.Println("start crawler")
 	refreshData := task.NewTask("refreshData", "0 0,30 16-20 * * *", RefreshData)
 	refreshEic := task.NewTask("refreshData", "0 0 2,6 * * *", RefreshEic)
 	refreshCoal := task.NewTask("refreshData", "0 0,30 17-23 * * *", RefreshCoal)
 	refreshVisitors := task.NewTask("RefreshChangesVisitorsCovid", "0 30 2-22/10 * * *", RefreshChangesVisitorsCovid)
 	syncEiaSteoData := task.NewTask("SyncEiaSteoData", "0 0 22 * * *", SyncEiaSteoData)
+	refreshNationalData := task.NewTask("RefreshNationalData", "0 15 2 10 * *", national_data.RefreshNationalData)
 
 	task.AddTask("数据爬取", refreshData)
 	task.AddTask("欧洲天然气爬取", refreshEic)
 	task.AddTask("中国煤炭网爬取", refreshCoal)
 	task.AddTask("谷歌出行指数爬取", refreshVisitors)
-	task.AddTask("eia steo报告", syncEiaSteoData) //每天22点爬一次
+	task.AddTask("eia steo报告", syncEiaSteoData)  //每天22点爬一次
+	task.AddTask("统计局数据爬取", refreshNationalData) // 每月10号2:15执行一次
 	task.StartTask()
 	//FileCoalJsm()
 	//FileCoalFirm()
@@ -60,11 +61,10 @@ func RefreshChangesVisitorsCovid(cont context.Context) (err error) {
 	return
 }
 
-func Task() {
+func Task11() {
 	fmt.Println("start")
 
-	time.Sleep(2 * time.Second)
-	national_data.ApiTest()
+	//_ = national_data.SyncXDateYQuotaZRegDb()
 	//_ = national_data.SyncQuotaClassifyTree()
 	//_ = national_data.SyncQuotaDataFromDbCodeAndId()