temp commit

models/base_from_national_statistics_index.go

@@ -15,6 +15,7 @@ type BaseFromNationalStatisticsIndex struct {
 	IndexCode                            string    `description:"指标编码"`
 	IndexName                            string    `description:"指标名称"`
 	Frequency                            string    `description:"频度"`
+	Reg                                  string    `description:"地区"`
 	StartDate                            time.Time `description:"开始日期"`
 	EndDate                              time.Time `description:"结束日期"`
 	CreateTime                           time.Time `description:"创建时间"`

services/national_data/national_data.go

@@ -19,6 +19,7 @@ const (
 )
 
 func NationalHttpPost(reqUrl, payload string) (result []byte, err error) {
+	time.Sleep(5 * time.Second) // 目前来看这个速度是不会中断的...就是慢...
 	tr := &http.Transport{
 		TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
 	}
@@ -114,14 +115,22 @@ func SyncQuotaClassifyTree() (err error) {
 		}
 	}()
 
+	// 含指标维度
 	dbCodes := []string{
 		"hgyd", "hgjd", "hgnd", // 月度/季度/年度
 		"fsyd", "fsjd", "fsnd", "csyd", "csnd", "gatyd", "gatnd", // 分省月度/分省季度/分省年度/城市月度价格/城市年度数据/港澳台月度/港澳台年度
 		"gjyd", "gjydsdj", "gjydsc", "gjnd", // 主要国家月度/三大经济体月度/国际市场月度商品/国家年度
 	}
+	// 含地区维度
+	regCodes := []string{
+		"fsyd", "fsjd", "fsnd", "csyd", "csnd", "gatyd", "gatnd", // 分省月度/分省季度/分省年度/城市月度价格/城市年度数据/港澳台月度/港澳台年度
+		"gjyd", "gjydsdj", "gjydsc", "gjnd", // 主要国家月度/三大经济体月度/国际市场月度商品/国家年度
+	}
 
 	for _, code := range dbCodes {
 		fmt.Println("开始同步DbCode: ", code)
+
+		// 指标维度
 		items := make([]*models.BaseFromNationalStatisticsClassify, 0)
 		resp, e := curlAndFormatQuotaClassify("zb", code, "zb", items)
 		if e != nil {
@@ -130,7 +139,17 @@ func SyncQuotaClassifyTree() (err error) {
 		}
 		items = resp
 
-		// 去重
+		// 地区维度
+		if utils.InArrayByStr(regCodes, code) {
+			regResp, e := curlAndFormatQuotaClassify("reg", code, "reg", items)
+			if e != nil {
+				err = fmt.Errorf("递归地区分类失败, Err: %s", e.Error())
+				return
+			}
+			items = append(items, regResp...)
+		}
+
+		// 去重-code+维度code+ID
 		classifyMap := make(map[string]bool)
 		classifyOB := new(models.BaseFromNationalStatisticsClassify)
 		classifyPars := make([]interface{}, 0)
@@ -140,12 +159,12 @@ func SyncQuotaClassifyTree() (err error) {
 			return
 		}
 		for _, c := range classifies {
-			classifyMap[fmt.Sprintf("%s%s", code, c.Id)] = true
+			classifyMap[fmt.Sprintf("%s%s%s", code, c.Wdcode, c.Id)] = true
 		}
 
 		finalList := make([]*models.BaseFromNationalStatisticsClassify, 0)
 		for _, v := range items {
-			if classifyMap[fmt.Sprintf("%s%s", code, v.Id)] {
+			if classifyMap[fmt.Sprintf("%s%s%s", code, v.Wdcode, v.Id)] {
 				continue
 			}
 			finalList = append(finalList, v)
@@ -183,6 +202,7 @@ func curlAndFormatQuotaClassify(id, dbcode, wdcode string, items []*models.BaseF
 		if v.IsParent {
 			isParent = 1
 		}
+		fmt.Printf("Dbcode: %s, ClassifyName: %s, Id: %s\n", v.Dbcode, v.Name, v.Id)
 		items = append(items, &models.BaseFromNationalStatisticsClassify{
 			ClassifyName: v.Name,
 			Id:           v.Id,
@@ -196,7 +216,7 @@ func curlAndFormatQuotaClassify(id, dbcode, wdcode string, items []*models.BaseF
 
 		// 向下递归
 		if isParent == 1 {
-			time.Sleep(1 * time.Second) // 缓缓...毕竟接口是人家的...
+			time.Sleep(5 * time.Second) // 缓缓...毕竟接口是人家的...
 			items, e = curlAndFormatQuotaClassify(v.Id, v.Dbcode, v.Wdcode, items)
 			if e != nil {
 				err = fmt.Errorf("递归请求分类树失败, Err: %s", e.Error())
@@ -207,6 +227,80 @@ func curlAndFormatQuotaClassify(id, dbcode, wdcode string, items []*models.BaseF
 	return items, nil
 }
 
+// DataApiReq 数据接口请求体
+type DataApiReq struct {
+	Method    string `description:"方法: QueryData-查询数据; getOtherWds-获取其他维度" json:"method"`
+	DbCode    string `description:"数据库编码" json:"dbcode"`
+	RowCode   string `description:"行-维度: zb; sj; reg" json:"rowcode"`
+	ColCode   string `description:"列-维度: zb; sj; reg" json:"colcode"`
+	WdsList   []Wds  `description:"维度列表" json:"wdsList"`
+	DfwdsList []Wds  `description:"df不知道啥意思...反正也是维度相关的" json:"dfwdsList"`
+}
+
+// Wds 维度
+type Wds struct {
+	WdCode    string `description:"维度: zb-指标; sj-时间; reg-地区" json:"wdcode"`
+	ValueCode string `description:"维度编码" json:"valuecode"`
+}
+
+// CommonDataApiRequest 数据接口请求
+func CommonDataApiRequest(req DataApiReq) (resp QuotaListDataResp, err error) {
+	if req.DbCode == "" {
+		return
+	}
+	if req.Method == "" {
+		req.Method = "QueryData"
+	}
+	if req.RowCode == "" {
+		req.RowCode = "zb"
+	}
+	if req.ColCode == "" {
+		req.ColCode = "sj"
+	}
+
+	// 构建查询
+	f := url.Values{}
+	f.Add("m", req.Method)
+	f.Add("dbcode", req.DbCode)
+	f.Add("rowcode", req.RowCode)
+	f.Add("colcode", req.ColCode)
+	wds := `[]`
+	if len(req.WdsList) > 0 {
+		wdsByte, e := json.Marshal(req.WdsList)
+		if e != nil {
+			err = fmt.Errorf("wds json marshal err: %s", e.Error())
+			return
+		}
+		wds = string(wdsByte)
+	}
+	dfwds := `[]`
+	if len(req.DfwdsList) > 0 {
+		dfwdsByte, e := json.Marshal(req.DfwdsList)
+		if e != nil {
+			err = fmt.Errorf("dfwds json marshal err: %s", e.Error())
+			return
+		}
+		dfwds = string(dfwdsByte)
+	}
+	f.Add("wds", wds)
+	f.Add("dfwds", dfwds)
+	f.Add("k1", fmt.Sprint(time.Now().UnixNano()/1e6))
+	f.Add("h", "1")
+
+	// 响应
+	r, e := NationalHttpPost(NationalStatisticsBaseReqUrl, f.Encode())
+	if e != nil {
+		err = fmt.Errorf("http request err: %s", e.Error())
+		return
+	}
+	utils.FileLog.Info("result: %s", string(r))
+	if e = json.Unmarshal(r, &resp); e != nil {
+		err = fmt.Errorf("resp unmarshal err: %s", e.Error())
+		return
+	}
+	return
+}
+
 // QuotaListDataResp 指标数据列表响应体
 type QuotaListDataResp struct {
 	ReturnCode int `description:"状态码" json:"returncode"`
@@ -224,14 +318,14 @@ type QuotaDataNode struct {
 		HasData bool    `description:"是否有值" json:"hasdata"`
 		StrData string  `description:"指标值(字符串)" json:"strdata"`
 	}
-	Wds []QuotaDataWds
+	Wds []Wds
 }
 
-// QuotaDataWds 指标数据对应的维度信息
-type QuotaDataWds struct {
-	ValueCode string `json:"valuecode"`
-	WdCode    string `json:"wdcode"`
-}
+//// QuotaDataWds 指标数据对应的维度信息
+//type QuotaDataWds struct {
+//	ValueCode string `json:"valuecode"`
+//	WdCode    string `json:"wdcode"`
+//}
 
 // QuotaWdNode 维度节点
 type QuotaWdNode struct {
@@ -248,39 +342,44 @@ type QuotaWdNodeData struct {
 	SortCode int    `description:"编码排序" json:"sortcode"`
 }
 
-// SyncQuotaDataFromDbCodeAndId 同步指标值
-func SyncQuotaDataFromDbCodeAndId() (err error) {
+// SyncMonth2YearDbQuota 同步月度/季度/年度数据指标
+func SyncMonth2YearDbQuota() (err error) {
 	defer func() {
 		if err != nil {
-			utils.FileLog.Error("统计局-同步数据失败, ErrMsg: %s", err.Error())
-			go alarm_msg.SendAlarmMsg(fmt.Sprintf("统计局-同步数据失败, ErrMsg: %s", err.Error()), 3)
+			utils.FileLog.Error("统计局-同步月度/季度/年度数据指标, ErrMsg: %s", err.Error())
+			go alarm_msg.SendAlarmMsg(fmt.Sprintf("统计局-同步月度/季度/年度数据指标, ErrMsg: %s", err.Error()), 3)
 		}
 	}()
 
 	// 查询无父级的指标分类
 	classifyOB := new(models.BaseFromNationalStatisticsClassify)
-	classifyCond := ` AND is_parent = 0`
+	classifyCond := ` AND is_parent = 0 AND dbcode IN ('hgyd', 'hgjd', 'hgnd')`
+	//classifyCond := ` AND is_parent = 0 AND dbcode IN ('hgnd')`
 	classifyPars := make([]interface{}, 0)
-	classifyOrder := ` ORDER BY base_from_national_statistics_classify_id ASC`
+	classifyOrder := ` base_from_national_statistics_classify_id ASC`
 	classifyList, e := classifyOB.GetItemsByCondition(classifyCond, classifyPars, []string{}, classifyOrder)
 	if e != nil {
 		err = fmt.Errorf("获取指标分类列表失败, Err: %s", e.Error())
 		return
 	}
+	fmt.Println("分类长度: ", len(classifyList))
 
 	// 同步指标和数据
 	for _, c := range classifyList {
-		time.Sleep(time.Second)
-		if e = SyncIndexAndDataByClassify(c.BaseFromNationalStatisticsClassifyId, c.Dbcode, c.Id); e != nil {
-			err = fmt.Errorf("同步分类下的指标和数据失败, Err: %s", e.Error())
+		fmt.Printf("开始同步分类: %s-%s-%s\n", c.ClassifyName, c.Dbcode, c.Id)
+		if e = SyncMonth2YearIndexAndDataByClassify(c.BaseFromNationalStatisticsClassifyId, c.Dbcode, c.Id); e != nil {
+			err = fmt.Errorf("同步指标数据失败, Err: %s", e.Error())
 			return
 		}
+		fmt.Printf("结束同步分类: %s-%s-%s\n", c.ClassifyName, c.Dbcode, c.Id)
+		// TODO:只测试一个
+		break
 	}
 	return
 }
 
-// SyncIndexAndDataByClassify 同步分类下的指标和数据
-func SyncIndexAndDataByClassify(classifyId int, dbCode, classifyCode string) (err error) {
+// SyncMonth2YearIndexAndDataByClassify 同步月度/季度/年度数据-分类下的指标
+func SyncMonth2YearIndexAndDataByClassify(classifyId int, dbCode, classifyCode string) (err error) {
 	// yd-月度 jd-季度 nd-年度
 	frequency := ""
 	timeParam := ""
@@ -297,25 +396,19 @@ func SyncIndexAndDataByClassify(classifyId int, dbCode, classifyCode string) (er
 		frequency = "年度"
 	}
 
-	// 构建查询
-	f := url.Values{}
-	f.Add("m", "QueryData")
-	f.Add("dbcode", dbCode)
-	f.Add("rowcode", "zb")
-	f.Add("colcode", "sj")
-	f.Add("wds", "[]")
-	f.Add("dfwds", fmt.Sprintf(`[{"wdcode":"zb","valuecode":"%s"},{"wdcode":"sj","valuecode":"%s"}]`, classifyCode, timeParam))
-	f.Add("k1", fmt.Sprint(time.Now().UnixNano()/1e6))
-	f.Add("h", "1")
-	r, e := NationalHttpPost(NationalStatisticsBaseReqUrl, f.Encode())
+	var dataReq DataApiReq
+	dataReq.DbCode = dbCode
+	var dfwdsList []Wds
+	dfwdsList = append(dfwdsList, Wds{
+		WdCode:    "zb",
+		ValueCode: classifyCode,
+	}, Wds{
+		WdCode:    "sj",
+		ValueCode: timeParam,
+	})
+	resp, e := CommonDataApiRequest(dataReq)
 	if e != nil {
-		err = fmt.Errorf("请求分类下的指标失败, Err: %s", e.Error())
-		return
-	}
-
-	resp := new(QuotaListDataResp)
-	if e = json.Unmarshal(r, &resp); e != nil {
-		fmt.Println("Unmarshal Err: ", e.Error())
+		err = fmt.Errorf("请求数据接口失败, Err: %s", e.Error())
 		return
 	}
 
@@ -395,9 +488,9 @@ func SyncIndexAndDataByClassify(classifyId int, dbCode, classifyCode string) (er
 			}
 
 			// 日期去重
-			t, e := time.ParseInLocation("200601", d.Code, time.Local)
+			t, e := formatMonth2YearDateCode(d.Code)
 			if e != nil {
-				err = fmt.Errorf("指标日期转换失败, Err: %s", e.Error())
+				err = fmt.Errorf("格式化日期code失败, Err: %s", e.Error())
 				return
 			}
 			existKey := fmt.Sprintf("%s%s", indexCode, t.Format(utils.FormatDate))
@@ -435,6 +528,566 @@ func SyncIndexAndDataByClassify(classifyId int, dbCode, classifyCode string) (er
 	return
 }
 
+// formatMonth2YearDateCode 将日期code转为对应日期
+func formatMonth2YearDateCode(dateCode string) (date time.Time, err error) {
+	if dateCode == "" {
+		return
+	}
+	// 根据日期code长度进行区分, 格式为三种: 月度-200601; 季度-2006A; 年度-2006
+	switch len([]rune(dateCode)) {
+	case 6:
+		t, e := time.ParseInLocation("200601", dateCode, time.Local)
+		if e != nil {
+			err = fmt.Errorf("月度指标日期转换失败, Err: %s", e.Error())
+			return
+		}
+		date = t
+		break
+	case 5:
+		// 季度ABCD转换成对应日期
+		dateSuffixMap := map[string]string{
+			"A": "03-31",
+			"B": "06-30",
+			"C": "09-30",
+			"D": "12-31",
+		}
+		dateCode = strings.ToUpper(dateCode)
+		quarterTab := dateCode[4:]
+		dateStr := fmt.Sprintf("%s-%s", dateCode[:4], dateSuffixMap[quarterTab])
+		t, e := time.ParseInLocation(utils.FormatDate, dateStr, time.Local)
+		if e != nil {
+			err = fmt.Errorf("季度指标日期转换失败, Err: %s", e.Error())
+			return
+		}
+		date = t
+		break
+	case 4:
+		dateStr := fmt.Sprintf("%s-%s", dateCode, "12-31")
+		t, e := time.ParseInLocation(utils.FormatDate, dateStr, time.Local)
+		if e != nil {
+			err = fmt.Errorf("年度指标日期转换失败, Err: %s", e.Error())
+			return
+		}
+		date = t
+		break
+	default:
+		err = fmt.Errorf("日期code格式有误, code: %s", dateCode)
+		return
+	}
+	return
+}
+
+// OtherWdResp 其他维度信息响应体
+type OtherWdResp struct {
+	ReturnCode int `description:"状态码" json:"returncode"`
+	ReturnData []struct {
+		IsSj   bool           `description:"是否为时间" json:"issj"`
+		WdCode string         `description:"维度编码" json:"wdcode"`
+		WdName string         `description:"维度名称" json:"wdname"`
+		Nodes  []OtherWdNodes `description:"维度数据" json:"nodes"`
+	} `description:"响应数据" json:"returndata"`
+}
+
+type OtherWdNodes struct {
+	Code string `description:"编码" json:"code"`
+	Name string `description:"名称" json:"name"`
+	Sort string `description:"排序" json:"sort"`
+}
+
+// GetDbOtherWdInfo 获取Db下其他维度信息
+func GetDbOtherWdInfo(dbCode, wdCode string) (nodes []OtherWdNodes, err error) {
+	defer func() {
+		fmt.Println("维度信息 end")
+	}()
+	fmt.Println("维度信息 start")
+
+	// 默认获取地区维度
+	if wdCode == "" {
+		wdCode = "reg"
+	}
+	// 构建查询
+	f := url.Values{}
+	f.Add("m", "getOtherWds")
+	f.Add("dbcode", dbCode)
+	f.Add("rowcode", "zb")
+	f.Add("colcode", "sj")
+	f.Add("wds", `[]`)
+	f.Add("k1", fmt.Sprint(time.Now().UnixNano()/1e6))
+	f.Add("h", "1")
+	r, e := NationalHttpPost(NationalStatisticsBaseReqUrl, f.Encode())
+	if e != nil {
+		err = fmt.Errorf("请求其他维度信息失败, Err: %s", e.Error())
+		return
+	}
+	utils.FileLog.Info("GetOtherWdInfo Result: %s", string(r))
+
+	// 响应
+	resp := new(OtherWdResp)
+	if e = json.Unmarshal(r, &resp); e != nil {
+		err = fmt.Errorf("其他维度信息Unmarshal Err: %s", e.Error())
+		return
+	}
+	if resp == nil {
+		err = fmt.Errorf("其他维度信息请求结果为空")
+		return
+	}
+	if resp.ReturnCode == 200 {
+		for _, d := range resp.ReturnData {
+			if d.WdCode == wdCode {
+				nodes = d.Nodes
+				return
+			}
+		}
+	}
+	return
+}
+
+// SyncRegQuota TODO:同步地区指标
+func SyncRegQuota() (err error) {
+	defer func() {
+		if err != nil {
+			utils.FileLog.Error("统计局-同步地区指标失败, ErrMsg: %s", err.Error())
+			go alarm_msg.SendAlarmMsg(fmt.Sprintf("统计局-同步地区指标失败, ErrMsg: %s", err.Error()), 3)
+		}
+	}()
+
+	// 查询无父级的指标分类
+	//"fsyd", "fsjd", "fsnd", "csyd", "csnd", "gatyd", "gatnd"
+	classifyOB := new(models.BaseFromNationalStatisticsClassify)
+	// 注意此处只需要同步分类中为指标的即可, 分类为地区的数据在指标中均有包含
+	//classifyCond := ` AND is_parent = 0 AND wdcode = 'zb' AND dbcode IN ('fsyd', 'fsjd', 'fsnd', 'csyd', 'csnd')`
+	classifyCond := ` AND is_parent = 0 AND wdcode = 'zb' AND dbcode IN ('fsjd')`
+	classifyPars := make([]interface{}, 0)
+	classifyOrder := ` base_from_national_statistics_classify_id ASC`
+	classifyList, e := classifyOB.GetItemsByCondition(classifyCond, classifyPars, []string{}, classifyOrder)
+	if e != nil {
+		err = fmt.Errorf("获取指标分类列表失败, Err: %s", e.Error())
+		return
+	}
+	fmt.Println("分类长度: ", len(classifyList))
+
+	// 其他维度
+	dbCodes := make([]string, 0)
+	for _, c := range classifyList {
+		if !utils.InArrayByStr(dbCodes, c.Dbcode) {
+			dbCodes = append(dbCodes, c.Dbcode)
+		}
+	}
+	fmt.Println("DbCode长度: ", len(dbCodes))
+
+	dbWdMap := make(map[string][]OtherWdNodes)
+	for _, dc := range dbCodes {
+		otherWd, e := GetDbOtherWdInfo(dc, "reg")
+		if e != nil {
+			err = fmt.Errorf("获取其他维度失败, DbCode: %s, Err: %s", dc, e.Error())
+			return
+		}
+		dbWdMap[dc] = otherWd
+	}
+
+	// 同步指标和数据
+	for _, c := range classifyList {
+		regList := dbWdMap[c.Dbcode]
+		if e = SyncRegIndexAndDataByClassify(c.BaseFromNationalStatisticsClassifyId, c.Dbcode, c.Id, regList); e != nil {
+			err = fmt.Errorf("同步分类下的指标和数据失败, Err: %s", e.Error())
+			return
+		}
+		// TODO:测试一个
+		break
+	}
+	return
+}
+
+// SyncRegIndexAndDataByClassify 同步地区数据-分类下的指标
+func SyncRegIndexAndDataByClassify(classifyId int, dbCode, classifyCode string, regList []OtherWdNodes) (err error) {
+	defer func() {
+		fmt.Println("同步地区数据-分类指标 end")
+	}()
+	fmt.Println("同步地区数据-分类指标 start")
+
+	// yd-月度 jd-季度 nd-年度
+	frequency := ""
+	timeParam := ""
+	if strings.Contains(dbCode, "yd") {
+		timeParam = "LAST36" // 最近36个月
+		frequency = "月度"
+	}
+	if strings.Contains(dbCode, "jd") {
+		timeParam = "LAST18" // 最近18个季度
+		frequency = "季度"
+	}
+	if strings.Contains(dbCode, "nd") {
+		timeParam = "LAST20" // 最近20年
+		frequency = "年度"
+	}
+
+	// 遍历地区维度, 查询指标和数据
+	for _, reg := range regList {
+		fmt.Printf("同步地区数据-分类指标-%s start\n", reg.Name)
+
+		// 构建查询
+		f := url.Values{}
+		f.Add("m", "QueryData")
+		f.Add("dbcode", dbCode)
+		f.Add("rowcode", "zb")
+		f.Add("colcode", "sj")
+		f.Add("wds", fmt.Sprintf(`[{"wdcode":"reg","valuecode":"%s"}]`, reg.Code))
+		f.Add("dfwds", fmt.Sprintf(`[{"wdcode":"zb","valuecode":"%s"},{"wdcode":"sj","valuecode":"%s"}]`, classifyCode, timeParam))
+		f.Add("k1", fmt.Sprint(time.Now().UnixNano()/1e6))
+		f.Add("h", "1")
+		r, e := NationalHttpPost(NationalStatisticsBaseReqUrl, f.Encode())
+		if e != nil {
+			err = fmt.Errorf("请求分类下的指标失败, Err: %s", e.Error())
+			return
+		}
+
+		resp := new(QuotaListDataResp)
+		if e = json.Unmarshal(r, &resp); e != nil {
+			fmt.Println("Unmarshal Err: ", e.Error())
+			return
+		}
+
+		// 数据集
+		dataNodes := resp.ReturnData.DataNodes
+		dataMap := make(map[string]QuotaDataNode)
+		for _, d := range dataNodes {
+			dataMap[d.Code] = d
+		}
+
+		// 取出指标(Y轴), 日期(X轴)
+		wdNodes := resp.ReturnData.WdNodes
+		var quotaNodes, dateNodes []QuotaWdNodeData
+		for _, w := range wdNodes {
+			if w.WdCode == "zb" {
+				quotaNodes = w.Nodes
+				continue
+			}
+			if w.WdCode == "sj" {
+				dateNodes = w.Nodes
+			}
+		}
+
+		// 指标编码去重, 指标编码+日期数据去重
+		indexOB := new(models.BaseFromNationalStatisticsIndex)
+		indexCond := ``
+		indexPars := make([]interface{}, 0)
+		indexList, e := indexOB.GetItemsByCondition(indexCond, indexPars, []string{"index_code"}, "")
+		if e != nil {
+			err = fmt.Errorf("获取指标列表失败, Err: %s", e.Error())
+			return
+		}
+		indexExistMap := make(map[string]bool)
+		for _, v := range indexList {
+			indexExistMap[v.IndexCode] = true
+		}
+		dataOB := new(models.BaseFromNationalStatisticsData)
+		dataCond := ``
+		dataPars := make([]interface{}, 0)
+		dataList, e := dataOB.GetItemsByCondition(dataCond, dataPars, []string{"index_code", "data_time"}, "")
+		if e != nil {
+			err = fmt.Errorf("获取指标数据列表失败, Err: %s", e.Error())
+			return
+		}
+		dataExistMap := make(map[string]bool)
+		for _, v := range dataList {
+			dataExistMap[fmt.Sprintf("%s%s", v.IndexCode, v.DataTime.Format(utils.FormatDate))] = true
+		}
+
+		// 遍历XY轴
+		indexDataList := make([]*models.SaveNationalStatisticsIndexAndDataReq, 0)
+		indexDataMap := make(map[string][]*models.BaseFromNationalStatisticsData)
+		for _, q := range quotaNodes {
+			// dbcode+指标code+地区code
+			indexCode := fmt.Sprintf("%s%s%s", dbCode, q.Code, reg.Code)
+
+			// 指标
+			r := new(models.SaveNationalStatisticsIndexAndDataReq)
+			r.Index = &models.BaseFromNationalStatisticsIndex{
+				BaseFromNationalStatisticsClassifyId: classifyId,
+				Dbcode:                               dbCode,
+				IndexCode:                            indexCode,
+				IndexName:                            q.Name,
+				Frequency:                            frequency,
+				Reg:                                  reg.Name,
+				CreateTime:                           time.Now().Local(),
+				ModifyTime:                           time.Now().Local(),
+			}
+			if indexExistMap[indexCode] {
+				r.IndexExist = true
+			}
+
+			// 数据
+			// zb.A01010201_reg.110000_sj.201608
+			for _, d := range dateNodes {
+				k := fmt.Sprintf("%s.%s_%s.%s_%s.%s", "zb", q.Code, "reg", reg.Code, "sj", d.Code)
+				v := dataMap[k]
+				if !v.Data.HasData {
+					continue
+				}
+
+				// 日期去重
+				t, e := formatMonth2YearDateCode(d.Code)
+				if e != nil {
+					err = fmt.Errorf("格式化日期code失败, Err: %s", e.Error())
+					return
+				}
+				existKey := fmt.Sprintf("%s%s", indexCode, t.Format(utils.FormatDate))
+				if dataExistMap[existKey] {
+					continue
+				}
+
+				// 数据map
+				if indexDataMap[indexCode] == nil {
+					indexDataMap[indexCode] = make([]*models.BaseFromNationalStatisticsData, 0)
+				}
+				indexDataMap[indexCode] = append(indexDataMap[indexCode], &models.BaseFromNationalStatisticsData{
+					IndexCode:  indexCode,
+					DataTime:   t,
+					Value:      v.Data.Data,
+					CreateTime: time.Now().Local(),
+					ModifyTime: time.Now().Local(),
+				})
+			}
+			indexDataList = append(indexDataList, r)
+		}
+
+		// 保存指标
+		for _, v := range indexDataList {
+			ds := indexDataMap[v.Index.IndexCode]
+			if ds == nil || (ds != nil && len(ds) == 0) {
+				continue
+			}
+			v.DataList = ds
+			if e := models.SaveNationalStatisticsIndexAndData(v); e != nil {
+				err = fmt.Errorf("保存指标和数据失败, Err: %s", e.Error())
+				return
+			}
+		}
+		fmt.Printf("同步地区数据-分类指标-%s end\n", reg.Name)
+		// TODO:测试一个
+		break
+	}
+	return
+}
+
+// SyncOtherQuota TODO:同步港澳台、国际数据指标
+func SyncOtherQuota() (err error) {
+	defer func() {
+		if err != nil {
+			utils.FileLog.Error("统计局-同步港澳台、国际数据指标失败, ErrMsg: %s", err.Error())
+			go alarm_msg.SendAlarmMsg(fmt.Sprintf("统计局-同步港澳台、国际数据指标失败, ErrMsg: %s", err.Error()), 3)
+		}
+	}()
+
+	// 查询无父级的指标分类
+	classifyOB := new(models.BaseFromNationalStatisticsClassify)
+	// 注意此处只需要同步分类中为指标的即可, 分类为地区的数据在指标中均有包含
+	//classifyCond := ` AND is_parent = 0 AND wdcode = 'zb' AND dbcode IN ('gatyd', 'gatnd', 'gjyd', 'gjydsdj', 'gjydsc', 'gjnd')`
+	classifyCond := ` AND is_parent = 0 AND wdcode = 'zb' AND dbcode IN ('gatyd', 'gatnd')`
+	classifyPars := make([]interface{}, 0)
+	classifyOrder := ` base_from_national_statistics_classify_id ASC`
+	classifyList, e := classifyOB.GetItemsByCondition(classifyCond, classifyPars, []string{}, classifyOrder)
+	if e != nil {
+		err = fmt.Errorf("获取指标分类列表失败, Err: %s", e.Error())
+		return
+	}
+	fmt.Println("分类长度: ", len(classifyList))
+
+	// 同步指标和数据
+	for _, c := range classifyList {
+		if e = SyncOtherIndexAndDataByClassify(c.BaseFromNationalStatisticsClassifyId, c.Dbcode, c.Id); e != nil {
+			err = fmt.Errorf("同步分类下的指标和数据失败, Err: %s", e.Error())
+			return
+		}
+		// TODO:测试一个
+		break
+	}
+	return
+}
+
+// SyncOtherIndexAndDataByClassify 同步港澳台、国际数据指标-分类下的指标
+func SyncOtherIndexAndDataByClassify(classifyId int, dbCode, classifyCode string) (err error) {
+	defer func() {
+		fmt.Println("同步港澳台、国际数据指标-分类指标 end")
+	}()
+	fmt.Println("同步港澳台、国际数据指标-分类指标 start")
+
+	// 先执行一次默认的, 取出其中的地区维度, 再遍历地区维度, 进行指标的搜索
+
+	// m: QueryData
+	//dbcode: gatyd
+	//rowcode: sj
+	//colcode: reg
+	//wds: [{"wdcode":"zb","valuecode":"A010A"}]
+	//dfwds: []
+	//k1: 1679994758636
+	//h: 1
+
+	// yd-月度 jd-季度 nd-年度
+	frequency := ""
+	timeParam := ""
+	if strings.Contains(dbCode, "yd") {
+		timeParam = "LAST36" // 最近36个月
+		frequency = "月度"
+	}
+	if strings.Contains(dbCode, "jd") {
+		timeParam = "LAST18" // 最近18个季度
+		frequency = "季度"
+	}
+	if strings.Contains(dbCode, "nd") {
+		timeParam = "LAST20" // 最近20年
+		frequency = "年度"
+	}
+
+	// 遍历地区维度, 查询指标和数据
+	for _, reg := range regList {
+		fmt.Printf("同步地区数据-分类指标-%s start\n", reg.Name)
+
+		// 构建查询
+		f := url.Values{}
+		f.Add("m", "QueryData")
+		f.Add("dbcode", dbCode)
+		f.Add("rowcode", "zb")
+		f.Add("colcode", "sj")
+		f.Add("wds", fmt.Sprintf(`[{"wdcode":"reg","valuecode":"%s"}]`, reg.Code))
+		f.Add("dfwds", fmt.Sprintf(`[{"wdcode":"zb","valuecode":"%s"},{"wdcode":"sj","valuecode":"%s"}]`, classifyCode, timeParam))
+		f.Add("k1", fmt.Sprint(time.Now().UnixNano()/1e6))
+		f.Add("h", "1")
+		r, e := NationalHttpPost(NationalStatisticsBaseReqUrl, f.Encode())
+		if e != nil {
+			err = fmt.Errorf("请求分类下的指标失败, Err: %s", e.Error())
+			return
+		}
+
+		resp := new(QuotaListDataResp)
+		if e = json.Unmarshal(r, &resp); e != nil {
+			fmt.Println("Unmarshal Err: ", e.Error())
+			return
+		}
+
+		// 数据集
+		dataNodes := resp.ReturnData.DataNodes
+		dataMap := make(map[string]QuotaDataNode)
+		for _, d := range dataNodes {
+			dataMap[d.Code] = d
+		}
+
+		// 取出指标(Y轴), 日期(X轴)
+		wdNodes := resp.ReturnData.WdNodes
+		var quotaNodes, dateNodes []QuotaWdNodeData
+		for _, w := range wdNodes {
+			if w.WdCode == "zb" {
+				quotaNodes = w.Nodes
+				continue
+			}
+			if w.WdCode == "sj" {
+				dateNodes = w.Nodes
+			}
+		}
+
+		// 指标编码去重, 指标编码+日期数据去重
+		indexOB := new(models.BaseFromNationalStatisticsIndex)
+		indexCond := ``
+		indexPars := make([]interface{}, 0)
+		indexList, e := indexOB.GetItemsByCondition(indexCond, indexPars, []string{"index_code"}, "")
+		if e != nil {
+			err = fmt.Errorf("获取指标列表失败, Err: %s", e.Error())
+			return
+		}
+		indexExistMap := make(map[string]bool)
+		for _, v := range indexList {
+			indexExistMap[v.IndexCode] = true
+		}
+		dataOB := new(models.BaseFromNationalStatisticsData)
+		dataCond := ``
+		dataPars := make([]interface{}, 0)
+		dataList, e := dataOB.GetItemsByCondition(dataCond, dataPars, []string{"index_code", "data_time"}, "")
+		if e != nil {
+			err = fmt.Errorf("获取指标数据列表失败, Err: %s", e.Error())
+			return
+		}
+		dataExistMap := make(map[string]bool)
+		for _, v := range dataList {
+			dataExistMap[fmt.Sprintf("%s%s", v.IndexCode, v.DataTime.Format(utils.FormatDate))] = true
+		}
+
+		// 遍历XY轴
+		indexDataList := make([]*models.SaveNationalStatisticsIndexAndDataReq, 0)
+		indexDataMap := make(map[string][]*models.BaseFromNationalStatisticsData)
+		for _, q := range quotaNodes {
+			// dbcode+指标code+地区code
+			indexCode := fmt.Sprintf("%s%s%s", dbCode, q.Code, reg.Code)
+
+			// 指标
+			r := new(models.SaveNationalStatisticsIndexAndDataReq)
+			r.Index = &models.BaseFromNationalStatisticsIndex{
+				BaseFromNationalStatisticsClassifyId: classifyId,
+				Dbcode:                               dbCode,
+				IndexCode:                            indexCode,
+				IndexName:                            q.Name,
+				Frequency:                            frequency,
+				Reg:                                  reg.Name,
+				CreateTime:                           time.Now().Local(),
+				ModifyTime:                           time.Now().Local(),
+			}
+			if indexExistMap[indexCode] {
+				r.IndexExist = true
+			}
+
+			// 数据
+			// zb.A01010201_reg.110000_sj.201608
+			for _, d := range dateNodes {
+				k := fmt.Sprintf("%s.%s_%s.%s_%s.%s", "zb", q.Code, "reg", reg.Code, "sj", d.Code)
+				v := dataMap[k]
+				if !v.Data.HasData {
+					continue
+				}
+
+				// 日期去重
+				t, e := formatMonth2YearDateCode(d.Code)
+				if e != nil {
+					err = fmt.Errorf("格式化日期code失败, Err: %s", e.Error())
+					return
+				}
+				existKey := fmt.Sprintf("%s%s", indexCode, t.Format(utils.FormatDate))
+				if dataExistMap[existKey] {
+					continue
+				}
+
+				// 数据map
+				if indexDataMap[indexCode] == nil {
+					indexDataMap[indexCode] = make([]*models.BaseFromNationalStatisticsData, 0)
+				}
+				indexDataMap[indexCode] = append(indexDataMap[indexCode], &models.BaseFromNationalStatisticsData{
+					IndexCode:  indexCode,
+					DataTime:   t,
+					Value:      v.Data.Data,
+					CreateTime: time.Now().Local(),
+					ModifyTime: time.Now().Local(),
+				})
+			}
+			indexDataList = append(indexDataList, r)
+		}
+
+		// 保存指标
+		for _, v := range indexDataList {
+			ds := indexDataMap[v.Index.IndexCode]
+			if ds == nil || (ds != nil && len(ds) == 0) {
+				continue
+			}
+			v.DataList = ds
+			if e := models.SaveNationalStatisticsIndexAndData(v); e != nil {
+				err = fmt.Errorf("保存指标和数据失败, Err: %s", e.Error())
+				return
+			}
+		}
+		fmt.Printf("同步地区数据-分类指标-%s end\n", reg.Name)
+		// TODO:测试一个
+		break
+	}
+	return
+}
+
 func ApiTest() (err error) {
 	defer func() {
 		if err != nil {
@@ -442,31 +1095,40 @@ func ApiTest() (err error) {
 		}
 	}()
 
-	dbCode := "hgyd"
-	classifyCode := "A010101"
+	dbCode := "gatyd"
+	classifyCode := "A0107"
+	wdCode := "121"
 	timeParam := "LAST36"
 
+	// m: QueryData
+	//dbcode: gatyd
+	//rowcode: zb
+	//colcode: reg
+	//wds: [{"wdcode":"sj","valuecode":"202302"}]
+	//dfwds: []
+	//k1: 1679992128420
+	//h: 1
+
 	f := url.Values{}
 	f.Add("m", "QueryData")
 	f.Add("dbcode", dbCode)
 	f.Add("rowcode", "zb")
 	f.Add("colcode", "sj")
-	f.Add("wds", "[]")
+	f.Add("wds", fmt.Sprintf(`[{"wdcode":"reg","valuecode":"%s"}]`, wdCode))
 	f.Add("dfwds", fmt.Sprintf(`[{"wdcode":"zb","valuecode":"%s"},{"wdcode":"sj","valuecode":"%s"}]`, classifyCode, timeParam))
 	f.Add("k1", fmt.Sprint(time.Now().UnixNano()/1e6))
-	//f.Add("h", "1")
+	f.Add("h", "1")
 	r, e := NationalHttpPost(NationalStatisticsBaseReqUrl, f.Encode())
 	if e != nil {
 		err = fmt.Errorf("请求分类下的指标失败, Err: %s", e.Error())
 		return
 	}
+	utils.FileLog.Info("result: %s", string(r))
 
 	resp := new(QuotaListDataResp)
 	if e = json.Unmarshal(r, &resp); e != nil {
 		fmt.Println("Unmarshal Err: ", e.Error())
 		return
 	}
-	result, _ := json.Marshal(resp)
-	utils.FileLog.Info("result: ", string(result))
 	return
 }

services/task.go

@@ -5,6 +5,7 @@ import (
 	"fmt"
 	"github.com/beego/beego/v2/task"
 	"hongze/hongze_data_crawler/services/national_data"
+	"time"
 )
 
 func Task123() {
@@ -62,6 +63,7 @@ func RefreshChangesVisitorsCovid(cont context.Context) (err error) {
 func Task() {
 	fmt.Println("start")
 
+	time.Sleep(2 * time.Second)
 	national_data.ApiTest()
 	//_ = national_data.SyncQuotaClassifyTree()
 	//_ = national_data.SyncQuotaDataFromDbCodeAndId()

utils/common.go

@@ -24,7 +24,7 @@ import (
 	"time"
 )
 
-//随机数种子
+// 随机数种子
 var rnd = rand.New(rand.NewSource(time.Now().UnixNano()))
 
 func GetRandString(size int) string {
@@ -61,13 +61,13 @@ func StringsToJSON(str string) string {
 	return jsons
 }
 
-//序列化
+// 序列化
 func ToString(v interface{}) string {
 	data, _ := json.Marshal(v)
 	return string(data)
 }
 
-//md5加密
+// md5加密
 func MD5(data string) string {
 	m := md5.Sum([]byte(data))
 	return hex.EncodeToString(m[:])
@@ -95,7 +95,7 @@ func GetToday(format string) string {
 	return today
 }
 
-//获取今天剩余秒数
+// 获取今天剩余秒数
 func GetTodayLastSecond() time.Duration {
 	today := GetToday(FormatDate) + " 23:59:59"
 	end, _ := time.ParseInLocation(FormatDateTime, today, time.Local)
@@ -117,7 +117,7 @@ func GetBrithDate(idcard string) string {
 	return GetToday(FormatDate)
 }
 
-//处理性别
+// 处理性别
 func WhichSexByIdcard(idcard string) string {
 	var sexs = [2]string{"女", "男"}
 	length := len(idcard)
@@ -131,7 +131,7 @@ func WhichSexByIdcard(idcard string) string {
 	return "男"
 }
 
-//截取小数点后几位
+// 截取小数点后几位
 func SubFloatToString(f float64, m int) string {
 	n := strconv.FormatFloat(f, 'f', -1, 64)
 	if n == "" {
@@ -150,20 +150,20 @@ func SubFloatToString(f float64, m int) string {
 	return newn[0] + "." + newn[1][:m]
 }
 
-//截取小数点后几位
+// 截取小数点后几位
 func SubFloatToFloat(f float64, m int) float64 {
 	newn := SubFloatToString(f, m)
 	newf, _ := strconv.ParseFloat(newn, 64)
 	return newf
 }
 
-//截取小数点后几位
+// 截取小数点后几位
 func SubFloatToFloatStr(f float64, m int) string {
 	newn := SubFloatToString(f, m)
 	return newn
 }
 
-//获取相差时间-年
+// 获取相差时间-年
 func GetYearDiffer(start_time, end_time string) int {
 	t1, _ := time.ParseInLocation("2006-01-02", start_time, time.Local)
 	t2, _ := time.ParseInLocation("2006-01-02", end_time, time.Local)
@@ -174,7 +174,7 @@ func GetYearDiffer(start_time, end_time string) int {
 	return age
 }
 
-//获取相差时间-秒
+// 获取相差时间-秒
 func GetSecondDifferByTime(start_time, end_time time.Time) int64 {
 	diff := end_time.Unix() - start_time.Unix()
 	return diff
@@ -201,7 +201,7 @@ func StrListToString(strList []string) (str string) {
 	return ""
 }
 
-//Token
+// Token
 func GetToken() string {
 	randStr := GetRandString(64)
 	token := MD5(randStr + Md5Key)
@@ -209,18 +209,18 @@ func GetToken() string {
 	return strings.ToUpper(token + GetRandString(tokenLen))
 }
 
-//数据没有记录
+// 数据没有记录
 func ErrNoRow() string {
 	return "<QuerySeter> no row found"
 }
 
-//判断文件是否存在
+// 判断文件是否存在
 func FileIsExist(filePath string) bool {
 	_, err := os.Stat(filePath)
 	return err == nil || os.IsExist(err)
 }
 
-//获取图片扩展名
+// 获取图片扩展名
 func GetImgExt(file string) (ext string, err error) {
 	var headerByte []byte
 	headerByte = make([]byte, 8)
@@ -263,7 +263,7 @@ func GetImgExt(file string) (ext string, err error) {
 	return ext, nil
 }
 
-//保存图片
+// 保存图片
 func SaveImage(path string, img image.Image) (err error) {
 	//需要保持的文件
 	imgfile, err := os.Create(path)
@@ -273,7 +273,7 @@ func SaveImage(path string, img image.Image) (err error) {
 	return err
 }
 
-//下载图片
+// 下载图片
 func DownloadImage(imgUrl string) (filePath string, err error) {
 	imgPath := "./static/imgs/"
 	fileName := path.Base(imgUrl)
@@ -299,7 +299,7 @@ func DownloadImage(imgUrl string) (filePath string, err error) {
 	return
 }
 
-//保存base64数据为文件
+// 保存base64数据为文件
 func SaveBase64ToFile(content, path string) error {
 	data, err := base64.StdEncoding.DecodeString(content)
 	if err != nil {
@@ -429,7 +429,7 @@ func GetWilsonScore(p, n float64) float64 {
 	return toFixed(((p+1.9208)/(p+n)-1.96*math.Sqrt(p*n/(p+n)+0.9604)/(p+n))/(1+3.8416/(p+n)), 2)
 }
 
-//将中文数字转化成数字，比如 第三百四十五章，返回第345章 不支持一亿及以上
+// 将中文数字转化成数字，比如 第三百四十五章，返回第345章 不支持一亿及以上
 func ChangeWordsToNum(str string) (numStr string) {
 	words := ([]rune)(str)
 	num := 0
@@ -600,12 +600,12 @@ func GetMonthStartAndEnd(myYear string, myMonth string) (startDate, endDate stri
 	return t1, t2
 }
 
-//移除字符串中的空格
+// 移除字符串中的空格
 func TrimStr(str string) (str2 string) {
 	return strings.Replace(str, " ", "", -1)
 }
 
-//字符串转换为time
+// 字符串转换为time
 func StrTimeToTime(strTime string) time.Time {
 	timeLayout := "2006-01-02 15:04:05"  //转化所需模板
 	loc, _ := time.LoadLocation("Local") //重要：获取时区
@@ -613,7 +613,7 @@ func StrTimeToTime(strTime string) time.Time {
 	return resultTime
 }
 
-//字符串类型时间转周几
+// 字符串类型时间转周几
 func StrDateTimeToWeek(strTime string) string {
 	var WeekDayMap = map[string]string{
 		"Monday":    "周一",
@@ -630,7 +630,7 @@ func StrDateTimeToWeek(strTime string) string {
 	return WeekDayMap[staweek_int]
 }
 
-//时间格式转年月日字符串
+// 时间格式转年月日字符串
 func TimeToStrYmd(time2 time.Time) string {
 	var Ymd string
 	year := time2.Year()
@@ -640,7 +640,7 @@ func TimeToStrYmd(time2 time.Time) string {
 	return Ymd
 }
 
-//时间格式去掉时分秒
+// 时间格式去掉时分秒
 func TimeRemoveHms(strTime string) string {
 	var Ymd string
 	var resultTime = StrTimeToTime(strTime)
@@ -651,7 +651,7 @@ func TimeRemoveHms(strTime string) string {
 	return Ymd
 }
 
-//文章上一次编辑时间
+// 文章上一次编辑时间
 func ArticleLastTime(strTime string) string {
 	var newTime string
 	stamp, _ := time.ParseInLocation("2006-01-02 15:04:05", strTime, time.Local)
@@ -672,7 +672,7 @@ func ArticleLastTime(strTime string) string {
 	return newTime
 }
 
-//人民币小写转大写
+// 人民币小写转大写
 func ConvertNumToCny(num float64) (str string, err error) {
 	strNum := strconv.FormatFloat(num*100, 'f', 0, 64)
 	sliceUnit := []string{"仟", "佰", "拾", "亿", "仟", "佰", "拾", "万", "仟", "佰", "拾", "元", "角", "分"}
@@ -923,3 +923,25 @@ func GetOrmInReplace(num int) string {
 	}
 	return strings.Join(template, ",")
 }
+
+// InArrayByInt php中的in_array（判断Int类型的切片中是否存在该int值）
+func InArrayByInt(idIntList []int, searchId int) (has bool) {
+	for _, id := range idIntList {
+		if id == searchId {
+			has = true
+			return
+		}
+	}
+	return
+}
+
+// InArrayByStr php中的in_array（判断String类型的切片中是否存在该string值）
+func InArrayByStr(idStrList []string, searchId string) (has bool) {
+	for _, id := range idStrList {
+		if id == searchId {
+			has = true
+			return
+		}
+	}
+	return
+}