指标分类; 月季年数据

models/base_from_national_statistics_data.go

@@ -0,0 +1,93 @@
+package models
+
+import (
+	"fmt"
+	"github.com/beego/beego/v2/client/orm"
+	"strings"
+	"time"
+)
+
+// BaseFromNationalStatisticsData 国统局-指标
+type BaseFromNationalStatisticsData struct {
+	BaseFromNationalStatisticsDataId  int       `orm:"column(base_from_national_statistics_data_id);pk"`
+	BaseFromNationalStatisticsIndexId int       `description:"指标ID"`
+	IndexCode                         string    `description:"指标编码"`
+	DataTime                          time.Time `description:"数据日期"`
+	Value                             float64   `description:"数据值"`
+	CreateTime                        time.Time `description:"创建时间"`
+	ModifyTime                        time.Time `description:"更新时间"`
+}
+
+func (m *BaseFromNationalStatisticsData) TableName() string {
+	return "base_from_national_statistics_data"
+}
+
+func (m *BaseFromNationalStatisticsData) Create() (err error) {
+	o := orm.NewOrmUsingDB("data")
+	id, err := o.Insert(m)
+	if err != nil {
+		return
+	}
+	m.BaseFromNationalStatisticsDataId = int(id)
+	return
+}
+
+func (m *BaseFromNationalStatisticsData) CreateMulti(items []*BaseFromNationalStatisticsData) (err error) {
+	if len(items) == 0 {
+		return
+	}
+	o := orm.NewOrmUsingDB("data")
+	_, err = o.InsertMulti(len(items), items)
+	return
+}
+
+func (m *BaseFromNationalStatisticsData) Update(cols []string) (err error) {
+	o := orm.NewOrmUsingDB("data")
+	_, err = o.Update(m, cols...)
+	return
+}
+
+func (m *BaseFromNationalStatisticsData) Del() (err error) {
+	o := orm.NewOrmUsingDB("data")
+	sql := `DELETE FROM base_from_national_statistics_data WHERE base_from_national_statistics_data_id = ? LIMIT 1`
+	_, err = o.Raw(sql, m.BaseFromNationalStatisticsDataId).Exec()
+	return
+}
+
+func (m *BaseFromNationalStatisticsData) GetItemById(id int) (err error) {
+	o := orm.NewOrmUsingDB("data")
+	sql := `SELECT * FROM base_from_national_statistics_data WHERE base_from_national_statistics_data_id = ? LIMIT 1`
+	err = o.Raw(sql, id).QueryRow(&m)
+	return
+}
+
+func (m *BaseFromNationalStatisticsData) GetItemByCondition(condition string, pars []interface{}) (err error) {
+	o := orm.NewOrmUsingDB("data")
+	sql := `SELECT * FROM base_from_national_statistics_data WHERE 1=1 `
+	sql += condition
+	sql += ` LIMIT 1`
+	err = o.Raw(sql, pars).QueryRow(&m)
+	return
+}
+
+func (m *BaseFromNationalStatisticsData) GetCountByCondition(condition string, pars []interface{}) (count int, err error) {
+	o := orm.NewOrmUsingDB("data")
+	sql := fmt.Sprintf(`SELECT COUNT(1) FROM %s WHERE 1=1 %s`, m.TableName(), condition)
+	err = o.Raw(sql, pars).QueryRow(&count)
+	return
+}
+
+func (m *BaseFromNationalStatisticsData) GetItemsByCondition(condition string, pars []interface{}, fieldArr []string, orderRule string) (items []*BaseFromNationalStatisticsData, err error) {
+	o := orm.NewOrmUsingDB("data")
+	fields := strings.Join(fieldArr, ",")
+	if len(fieldArr) == 0 {
+		fields = `*`
+	}
+	order := `ORDER BY create_time DESC`
+	if orderRule != "" {
+		order = ` ORDER BY ` + orderRule
+	}
+	sql := fmt.Sprintf(`SELECT %s FROM %s WHERE 1=1 %s %s`, fields, m.TableName(), condition, order)
+	_, err = o.Raw(sql, pars).QueryRows(&items)
+	return
+}

models/base_from_national_statistics_index.go

@@ -0,0 +1,198 @@
+package models
+
+import (
+	"fmt"
+	"github.com/beego/beego/v2/client/orm"
+	"strings"
+	"time"
+)
+
+// BaseFromNationalStatisticsIndex 国统局-指标
+type BaseFromNationalStatisticsIndex struct {
+	BaseFromNationalStatisticsIndexId    int       `orm:"column(base_from_national_statistics_index_id);pk"`
+	BaseFromNationalStatisticsClassifyId int       `description:"指标分类ID"`
+	Dbcode                               string    `description:"dbcode"`
+	IndexCode                            string    `description:"指标编码"`
+	IndexName                            string    `description:"指标名称"`
+	Frequency                            string    `description:"频度"`
+	StartDate                            time.Time `description:"开始日期"`
+	EndDate                              time.Time `description:"结束日期"`
+	CreateTime                           time.Time `description:"创建时间"`
+	ModifyTime                           time.Time `description:"更新时间"`
+}
+
+func (m *BaseFromNationalStatisticsIndex) TableName() string {
+	return "base_from_national_statistics_index"
+}
+
+func (m *BaseFromNationalStatisticsIndex) Create() (err error) {
+	o := orm.NewOrmUsingDB("data")
+	id, err := o.Insert(m)
+	if err != nil {
+		return
+	}
+	m.BaseFromNationalStatisticsIndexId = int(id)
+	return
+}
+
+func (m *BaseFromNationalStatisticsIndex) CreateMulti(items []*BaseFromNationalStatisticsIndex) (err error) {
+	if len(items) == 0 {
+		return
+	}
+	o := orm.NewOrmUsingDB("data")
+	_, err = o.InsertMulti(len(items), items)
+	return
+}
+
+func (m *BaseFromNationalStatisticsIndex) Update(cols []string) (err error) {
+	o := orm.NewOrmUsingDB("data")
+	_, err = o.Update(m, cols...)
+	return
+}
+
+func (m *BaseFromNationalStatisticsIndex) Del() (err error) {
+	o := orm.NewOrmUsingDB("data")
+	sql := `DELETE FROM base_from_national_statistics_index WHERE base_from_national_statistics_index_id = ? LIMIT 1`
+	_, err = o.Raw(sql, m.BaseFromNationalStatisticsIndexId).Exec()
+	return
+}
+
+func (m *BaseFromNationalStatisticsIndex) GetItemById(id int) (err error) {
+	o := orm.NewOrmUsingDB("data")
+	sql := `SELECT * FROM base_from_national_statistics_index WHERE base_from_national_statistics_index_id = ? LIMIT 1`
+	err = o.Raw(sql, id).QueryRow(&m)
+	return
+}
+
+func (m *BaseFromNationalStatisticsIndex) GetItemByCondition(condition string, pars []interface{}) (err error) {
+	o := orm.NewOrmUsingDB("data")
+	sql := `SELECT * FROM base_from_national_statistics_index WHERE 1=1 `
+	sql += condition
+	sql += ` LIMIT 1`
+	err = o.Raw(sql, pars).QueryRow(&m)
+	return
+}
+
+func (m *BaseFromNationalStatisticsIndex) GetCountByCondition(condition string, pars []interface{}) (count int, err error) {
+	o := orm.NewOrmUsingDB("data")
+	sql := fmt.Sprintf(`SELECT COUNT(1) FROM %s WHERE 1=1 %s`, m.TableName(), condition)
+	err = o.Raw(sql, pars).QueryRow(&count)
+	return
+}
+
+func (m *BaseFromNationalStatisticsIndex) GetItemsByCondition(condition string, pars []interface{}, fieldArr []string, orderRule string) (items []*BaseFromNationalStatisticsIndex, err error) {
+	o := orm.NewOrmUsingDB("data")
+	fields := strings.Join(fieldArr, ",")
+	if len(fieldArr) == 0 {
+		fields = `*`
+	}
+	order := `ORDER BY create_time DESC`
+	if orderRule != "" {
+		order = ` ORDER BY ` + orderRule
+	}
+	sql := fmt.Sprintf(`SELECT %s FROM %s WHERE 1=1 %s %s`, fields, m.TableName(), condition, order)
+	_, err = o.Raw(sql, pars).QueryRows(&items)
+	return
+}
+
+// SaveNationalStatisticsIndexAndDataReq 保存指标和数据请求体
+type SaveNationalStatisticsIndexAndDataReq struct {
+	Index      *BaseFromNationalStatisticsIndex  `description:"指标信息"`
+	IndexExist bool                              `description:"指标是否存在"`
+	DataList   []*BaseFromNationalStatisticsData `description:"新增的指标数据"`
+}
+
+// SaveNationalStatisticsIndexAndData 保存指标和值
+func SaveNationalStatisticsIndexAndData(req *SaveNationalStatisticsIndexAndDataReq) (err error) {
+	if req.Index == nil {
+		return
+	}
+	o := orm.NewOrmUsingDB("data")
+	tx, err := o.Begin()
+	if err != nil {
+		return
+	}
+	defer func() {
+		if err != nil {
+			_ = tx.Rollback()
+			return
+		}
+		_ = tx.Commit()
+	}()
+
+	if !req.IndexExist {
+		id, e := tx.Insert(req.Index)
+		if e != nil {
+			return e
+		}
+		req.Index.BaseFromNationalStatisticsIndexId = int(id)
+	}
+	indexId := req.Index.BaseFromNationalStatisticsIndexId
+	if req.DataList != nil && len(req.DataList) > 0 {
+		for _, d := range req.DataList {
+			d.BaseFromNationalStatisticsIndexId = indexId
+		}
+		_, e := tx.InsertMulti(len(req.DataList), req.DataList)
+		if e != nil {
+			return e
+		}
+	}
+	return
+}
+
+// BatchSaveNationalStatisticsIndexAndData 批量保存指标和值
+func BatchSaveNationalStatisticsIndexAndData(indexArr []*BaseFromNationalStatisticsIndex, indexDataMap map[string][]*BaseFromNationalStatisticsData) (err error) {
+	o := orm.NewOrmUsingDB("data")
+	tx, err := o.Begin()
+	if err != nil {
+		return
+	}
+	defer func() {
+		if err != nil {
+			_ = tx.Rollback()
+			return
+		}
+		_ = tx.Commit()
+	}()
+
+	// 指标
+	for _, v := range indexArr {
+		id, e := tx.Insert(v)
+		if e != nil {
+			return e
+		}
+		indexId := int(id)
+
+		// 数据
+		dataList := indexDataMap[v.IndexCode]
+		if dataList != nil && len(dataList) > 0 {
+			for _, d := range dataList {
+				d.BaseFromNationalStatisticsIndexId = indexId
+			}
+			_, e = tx.InsertMulti(len(dataList), dataList)
+			if e != nil {
+				return e
+			}
+		}
+	}
+	return
+}
+
+// UpdateNationalStatisticsIndexStartEndDate 更新指标开始结束日期
+func UpdateNationalStatisticsIndexStartEndDate() (err error) {
+	o := orm.NewOrmUsingDB("data")
+	sql := `UPDATE base_from_national_statistics_index AS a
+			JOIN (
+				SELECT
+					index_code,
+					MIN(data_time) AS min_time,
+					MAX(data_time) AS max_time
+				FROM
+					base_from_national_statistics_data
+				GROUP BY
+					index_code
+			) AS b ON a.index_code = b.index_code
+			SET a.start_date = b.min_time, a.end_date = b.max_time`
+	_, err = o.Raw(sql).Exec()
+	return
+}

models/db.go

@@ -50,5 +50,7 @@ func init() {
 		new(BaseFromEiaSteoData),
 		new(BaseFromEiaSteoClassify),
 		new(BaseFromNationalStatisticsClassify),
+		new(BaseFromNationalStatisticsIndex),
+		new(BaseFromNationalStatisticsData),
 	)
 }

services/national_data/national_data.go

@@ -5,6 +5,8 @@ import (
 	"encoding/json"
 	"fmt"
 	"hongze/hongze_data_crawler/models"
+	"hongze/hongze_data_crawler/services/alarm_msg"
+	"hongze/hongze_data_crawler/utils"
 	"io/ioutil"
 	"net/http"
 	"net/url"
@@ -104,48 +106,55 @@ type QuotaClassifyTreeResp struct {
 }
 
 // SyncQuotaClassifyTree 同步指标分类树
-// dbcode: hgyd-月度; hgjd-季度; ...
 func SyncQuotaClassifyTree() (err error) {
 	defer func() {
 		if err != nil {
-			fmt.Println(err.Error())
+			utils.FileLog.Error("统计局-同步指标分类树失败, ErrMsg: %s", err.Error())
+			go alarm_msg.SendAlarmMsg(fmt.Sprintf("统计局-同步指标分类树失败, ErrMsg: %s", err.Error()), 3)
 		}
 	}()
 
-	items := make([]*models.BaseFromNationalStatisticsClassify, 0)
-	initId := "zb"
-	initDbcode := "hgyd"
-	initWdcode := "zb"
-	resp, e := curlAndFormatQuotaClassify(initId, initDbcode, initWdcode, items)
-	if e != nil {
-		err = fmt.Errorf("递归指标分类失败, Err: %s", e.Error())
-		return
+	dbCodes := []string{
+		"hgyd", "hgjd", "hgnd", // 月度/季度/年度
+		"fsyd", "fsjd", "fsnd", "csyd", "csnd", "gatyd", "gatnd", // 分省月度/分省季度/分省年度/城市月度价格/城市年度数据/港澳台月度/港澳台年度
+		"gjyd", "gjydsdj", "gjydsc", "gjnd", // 主要国家月度/三大经济体月度/国际市场月度商品/国家年度
 	}
-	items = resp
 
-	// 去重
-	classifyMap := make(map[string]bool)
-	classifyOB := new(models.BaseFromNationalStatisticsClassify)
-	classifyPars := make([]interface{}, 0)
-	classifies, e := classifyOB.GetItemsByCondition("", classifyPars, []string{}, "")
-	if e != nil {
-		err = fmt.Errorf("获取指标分类列表失败, Err: %s", e.Error())
-		return
-	}
-	for _, c := range classifies {
-		classifyMap[c.Id] = true
-	}
+	for _, code := range dbCodes {
+		fmt.Println("开始同步DbCode: ", code)
+		items := make([]*models.BaseFromNationalStatisticsClassify, 0)
+		resp, e := curlAndFormatQuotaClassify("zb", code, "zb", items)
+		if e != nil {
+			err = fmt.Errorf("递归指标分类失败, Err: %s", e.Error())
+			return
+		}
+		items = resp
 
-	finalList := make([]*models.BaseFromNationalStatisticsClassify, 0)
-	for _, v := range items {
-		if classifyMap[v.Id] {
-			continue
+		// 去重
+		classifyMap := make(map[string]bool)
+		classifyOB := new(models.BaseFromNationalStatisticsClassify)
+		classifyPars := make([]interface{}, 0)
+		classifies, e := classifyOB.GetItemsByCondition("", classifyPars, []string{}, "")
+		if e != nil {
+			err = fmt.Errorf("获取指标分类列表失败, Err: %s", e.Error())
+			return
 		}
-		finalList = append(finalList, v)
-	}
-	if e = classifyOB.CreateMulti(items); e != nil {
-		err = fmt.Errorf("批量新增指标分类失败, Err: %s", e.Error())
-		return
+		for _, c := range classifies {
+			classifyMap[fmt.Sprintf("%s%s", code, c.Id)] = true
+		}
+
+		finalList := make([]*models.BaseFromNationalStatisticsClassify, 0)
+		for _, v := range items {
+			if classifyMap[fmt.Sprintf("%s%s", code, v.Id)] {
+				continue
+			}
+			finalList = append(finalList, v)
+		}
+		if e = classifyOB.CreateMulti(items); e != nil {
+			err = fmt.Errorf("批量新增指标分类失败, Err: %s", e.Error())
+			return
+		}
+		fmt.Println("结束同步DbCode: ", code)
 	}
 	return
 }
@@ -198,14 +207,16 @@ func curlAndFormatQuotaClassify(id, dbcode, wdcode string, items []*models.BaseF
 	return items, nil
 }
 
+// QuotaListDataResp 指标数据列表响应体
 type QuotaListDataResp struct {
 	ReturnCode int `description:"状态码" json:"returncode"`
 	ReturnData struct {
 		DataNodes []QuotaDataNode `json:"datanodes"`
-		WdNodes   struct{}        `json:"wdnodes"`
+		WdNodes   []QuotaWdNode   `json:"wdnodes"`
 	}
 }
 
+// QuotaDataNode 指标数据节点
 type QuotaDataNode struct {
 	Code string `description:"编码"`
 	Data struct {
@@ -216,51 +227,246 @@ type QuotaDataNode struct {
 	Wds []QuotaDataWds
 }
 
+// QuotaDataWds 指标数据对应的维度信息
 type QuotaDataWds struct {
 	ValueCode string `json:"valuecode"`
 	WdCode    string `json:"wdcode"`
 }
 
+// QuotaWdNode 维度节点
+type QuotaWdNode struct {
+	WdCode string `description:"示例: zb; sj; reg;" json:"wdcode"`
+	WdName string `description:"示例: 指标; 时间; 地区" json:"wdname"`
+	Nodes  []QuotaWdNodeData
+}
+
+// QuotaWdNodeData 维度节点数据
+type QuotaWdNodeData struct {
+	Code     string `description:"指标编码"`
+	Name     string `description:"指标名称"`
+	Unit     string `description:"单位"`
+	SortCode int    `description:"编码排序" json:"sortcode"`
+}
+
 // SyncQuotaDataFromDbCodeAndId 同步指标值
 func SyncQuotaDataFromDbCodeAndId() (err error) {
+	defer func() {
+		if err != nil {
+			utils.FileLog.Error("统计局-同步数据失败, ErrMsg: %s", err.Error())
+			go alarm_msg.SendAlarmMsg(fmt.Sprintf("统计局-同步数据失败, ErrMsg: %s", err.Error()), 3)
+		}
+	}()
+
 	// 查询无父级的指标分类
-	//classifyOB := new(models.BaseFromNationalStatisticsClassify)
-	//classifyCond := ` AND is_parent = 0`
-	//classifyPars := make([]interface{}, 0)
-	//classifyOrder := ` ORDER BY base_from_national_statistics_classify_id ASC`
-	//classifies, e := classifyOB.GetItemsByCondition(classifyCond, classifyPars, []string{}, classifyOrder)
-	//if e != nil {
-	//	err = fmt.Errorf("获取指标分类列表失败, Err: %s", e.Error())
-	//	return
-	//}
-
-	//	m: QueryData
-	//	dbcode: hgyd
-	//	rowcode: zb
-	//	colcode: sj
-	//	wds: []
-	//	dfwds: [{"wdcode":"zb","valuecode":"A010101"}]
-	//	k1: 1678872233342
-	//	h: 1
-	time.Sleep(2 * time.Second)
+	classifyOB := new(models.BaseFromNationalStatisticsClassify)
+	classifyCond := ` AND is_parent = 0`
+	classifyPars := make([]interface{}, 0)
+	classifyOrder := ` ORDER BY base_from_national_statistics_classify_id ASC`
+	classifyList, e := classifyOB.GetItemsByCondition(classifyCond, classifyPars, []string{}, classifyOrder)
+	if e != nil {
+		err = fmt.Errorf("获取指标分类列表失败, Err: %s", e.Error())
+		return
+	}
+
+	// 同步指标和数据
+	for _, c := range classifyList {
+		time.Sleep(time.Second)
+		if e = SyncIndexAndDataByClassify(c.BaseFromNationalStatisticsClassifyId, c.Dbcode, c.Id); e != nil {
+			err = fmt.Errorf("同步分类下的指标和数据失败, Err: %s", e.Error())
+			return
+		}
+	}
+	return
+}
+
+// SyncIndexAndDataByClassify 同步分类下的指标和数据
+func SyncIndexAndDataByClassify(classifyId int, dbCode, classifyCode string) (err error) {
+	// yd-月度 jd-季度 nd-年度
+	frequency := ""
+	timeParam := ""
+	if strings.Contains(dbCode, "yd") {
+		timeParam = "LAST36" // 最近36个月
+		frequency = "月度"
+	}
+	if strings.Contains(dbCode, "jd") {
+		timeParam = "LAST18" // 最近18个季度
+		frequency = "季度"
+	}
+	if strings.Contains(dbCode, "nd") {
+		timeParam = "LAST20" // 最近20年
+		frequency = "年度"
+	}
 
 	// 构建查询
 	f := url.Values{}
 	f.Add("m", "QueryData")
-	f.Add("dbcode", "hgyd")
+	f.Add("dbcode", dbCode)
 	f.Add("rowcode", "zb")
 	f.Add("colcode", "sj")
 	f.Add("wds", "[]")
-	f.Add("dfwds", `[{"wdcode":"zb","valuecode":"A010101"}]`)
+	f.Add("dfwds", fmt.Sprintf(`[{"wdcode":"zb","valuecode":"%s"},{"wdcode":"sj","valuecode":"%s"}]`, classifyCode, timeParam))
 	f.Add("k1", fmt.Sprint(time.Now().UnixNano()/1e6))
 	f.Add("h", "1")
 	r, e := NationalHttpPost(NationalStatisticsBaseReqUrl, f.Encode())
-	fmt.Println("err: ", e)
 	if e != nil {
-		fmt.Println("获取失败, Err: ", e.Error())
-		//err = fmt.Errorf("请求指标分类树失败, Err: %s", e.Error())
+		err = fmt.Errorf("请求分类下的指标失败, Err: %s", e.Error())
+		return
+	}
+
+	resp := new(QuotaListDataResp)
+	if e = json.Unmarshal(r, &resp); e != nil {
+		fmt.Println("Unmarshal Err: ", e.Error())
+		return
+	}
+
+	// 数据集
+	dataNodes := resp.ReturnData.DataNodes
+	dataMap := make(map[string]QuotaDataNode)
+	for _, d := range dataNodes {
+		dataMap[d.Code] = d
+	}
+
+	// 取出指标(Y轴), 日期(X轴)
+	wdNodes := resp.ReturnData.WdNodes
+	var quotaNodes, dateNodes []QuotaWdNodeData
+	for _, w := range wdNodes {
+		if w.WdCode == "zb" {
+			quotaNodes = w.Nodes
+			continue
+		}
+		if w.WdCode == "sj" {
+			dateNodes = w.Nodes
+		}
+	}
+
+	// 指标编码去重, 指标编码+日期数据去重
+	indexOB := new(models.BaseFromNationalStatisticsIndex)
+	indexCond := ``
+	indexPars := make([]interface{}, 0)
+	indexList, e := indexOB.GetItemsByCondition(indexCond, indexPars, []string{"index_code"}, "")
+	if e != nil {
+		err = fmt.Errorf("获取指标列表失败, Err: %s", e.Error())
+		return
+	}
+	indexExistMap := make(map[string]bool)
+	for _, v := range indexList {
+		indexExistMap[v.IndexCode] = true
+	}
+	dataOB := new(models.BaseFromNationalStatisticsData)
+	dataCond := ``
+	dataPars := make([]interface{}, 0)
+	dataList, e := dataOB.GetItemsByCondition(dataCond, dataPars, []string{"index_code", "data_time"}, "")
+	if e != nil {
+		err = fmt.Errorf("获取指标数据列表失败, Err: %s", e.Error())
+		return
+	}
+	dataExistMap := make(map[string]bool)
+	for _, v := range dataList {
+		dataExistMap[fmt.Sprintf("%s%s", v.IndexCode, v.DataTime.Format(utils.FormatDate))] = true
+	}
+
+	// 遍历XY轴
+	indexDataList := make([]*models.SaveNationalStatisticsIndexAndDataReq, 0)
+	indexDataMap := make(map[string][]*models.BaseFromNationalStatisticsData)
+	for _, q := range quotaNodes {
+		indexCode := fmt.Sprintf("%s%s", dbCode, q.Code)
+
+		// 指标
+		r := new(models.SaveNationalStatisticsIndexAndDataReq)
+		r.Index = &models.BaseFromNationalStatisticsIndex{
+			BaseFromNationalStatisticsClassifyId: classifyId,
+			Dbcode:                               dbCode,
+			IndexCode:                            indexCode,
+			IndexName:                            q.Name,
+			Frequency:                            frequency,
+			CreateTime:                           time.Now().Local(),
+			ModifyTime:                           time.Now().Local(),
+		}
+		if indexExistMap[indexCode] {
+			r.IndexExist = true
+		}
+
+		// 数据
+		for _, d := range dateNodes {
+			k := fmt.Sprintf("%s.%s_%s.%s", "zb", q.Code, "sj", d.Code)
+			v := dataMap[k]
+			if !v.Data.HasData {
+				continue
+			}
+
+			// 日期去重
+			t, e := time.ParseInLocation("200601", d.Code, time.Local)
+			if e != nil {
+				err = fmt.Errorf("指标日期转换失败, Err: %s", e.Error())
+				return
+			}
+			existKey := fmt.Sprintf("%s%s", indexCode, t.Format(utils.FormatDate))
+			if dataExistMap[existKey] {
+				continue
+			}
+
+			// 数据map
+			if indexDataMap[indexCode] == nil {
+				indexDataMap[indexCode] = make([]*models.BaseFromNationalStatisticsData, 0)
+			}
+			indexDataMap[indexCode] = append(indexDataMap[indexCode], &models.BaseFromNationalStatisticsData{
+				IndexCode:  indexCode,
+				DataTime:   t,
+				Value:      v.Data.Data,
+				CreateTime: time.Now().Local(),
+				ModifyTime: time.Now().Local(),
+			})
+		}
+		indexDataList = append(indexDataList, r)
+	}
+
+	// 保存指标
+	for _, v := range indexDataList {
+		ds := indexDataMap[v.Index.IndexCode]
+		if ds == nil || (ds != nil && len(ds) == 0) {
+			continue
+		}
+		v.DataList = ds
+		if e := models.SaveNationalStatisticsIndexAndData(v); e != nil {
+			err = fmt.Errorf("保存指标和数据失败, Err: %s", e.Error())
+			return
+		}
+	}
+	return
+}
+
+func ApiTest() (err error) {
+	defer func() {
+		if err != nil {
+			fmt.Println(err.Error())
+		}
+	}()
+
+	dbCode := "hgyd"
+	classifyCode := "A010101"
+	timeParam := "LAST36"
+
+	f := url.Values{}
+	f.Add("m", "QueryData")
+	f.Add("dbcode", dbCode)
+	f.Add("rowcode", "zb")
+	f.Add("colcode", "sj")
+	f.Add("wds", "[]")
+	f.Add("dfwds", fmt.Sprintf(`[{"wdcode":"zb","valuecode":"%s"},{"wdcode":"sj","valuecode":"%s"}]`, classifyCode, timeParam))
+	f.Add("k1", fmt.Sprint(time.Now().UnixNano()/1e6))
+	//f.Add("h", "1")
+	r, e := NationalHttpPost(NationalStatisticsBaseReqUrl, f.Encode())
+	if e != nil {
+		err = fmt.Errorf("请求分类下的指标失败, Err: %s", e.Error())
+		return
+	}
+
+	resp := new(QuotaListDataResp)
+	if e = json.Unmarshal(r, &resp); e != nil {
+		fmt.Println("Unmarshal Err: ", e.Error())
 		return
 	}
-	fmt.Println("res: ", string(r))
+	result, _ := json.Marshal(resp)
+	utils.FileLog.Info("result: ", string(result))
 	return
 }

services/task.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"fmt"
 	"github.com/beego/beego/v2/task"
+	"hongze/hongze_data_crawler/services/national_data"
 )
 
 func Task123() {
@@ -61,6 +62,7 @@ func RefreshChangesVisitorsCovid(cont context.Context) (err error) {
 func Task() {
 	fmt.Println("start")
 
+	national_data.ApiTest()
 	//_ = national_data.SyncQuotaClassifyTree()
 	//_ = national_data.SyncQuotaDataFromDbCodeAndId()