eta_crawler/services/national_data/classify.go

package national_data

import (
	"encoding/json"
	"eta/eta_crawler/models"
	"eta/eta_crawler/services/alarm_msg"
	"eta/eta_crawler/utils"
	"fmt"
	"net/url"
	"time"
)

// QuotaClassifyTreeResp 指标分类树响应
type QuotaClassifyTreeResp struct {
	Id       string `description:"分类ID(字符串)"`
	IsParent bool   `description:"是否为父级"`
	Name     string `description:"分类名称"`
	Pid      string `description:"父级分类ID"`
	Dbcode   string `description:"源-dbocde"`
	Wdcode   string `description:"源-wdcode"`
}

// SyncQuotaClassifyTree 同步指标分类树
func SyncQuotaClassifyTree() (err error) {
	defer func() {
		if err != nil {
			utils.FileLog.Error("统计局-同步指标分类树失败, ErrMsg: %s", err.Error())
			go alarm_msg.SendAlarmMsg(fmt.Sprintf("统计局-同步指标分类树失败, ErrMsg: %s", err.Error()), 3)
		}
	}()

	// 含指标维度
	dbCodes := []string{
		"hgyd", "hgjd", "hgnd", // 月度/季度/年度
		"fsyd", "fsjd", "fsnd", "csyd", "csnd", "gatyd", "gatnd", // 分省月度/分省季度/分省年度/城市月度价格/城市年度数据/港澳台月度/港澳台年度
		"gjyd", "gjydsdj", "gjydsc", "gjnd", // 主要国家月度/三大经济体月度/国际市场月度商品/国家年度
	}
	// 含地区维度
	regCodes := []string{
		"fsyd", "fsjd", "fsnd", "csyd", "csnd", "gatyd", "gatnd", // 分省月度/分省季度/分省年度/城市月度价格/城市年度数据/港澳台月度/港澳台年度
		"gjyd", "gjydsdj", "gjydsc", "gjnd", // 主要国家月度/三大经济体月度/国际市场月度商品/国家年度
	}

	for _, code := range dbCodes {
		fmt.Println("开始同步DbCode: ", code)

		// 指标维度
		items := make([]*models.BaseFromNationalStatisticsClassify, 0)
		resp, e := curlAndFormatQuotaClassify("zb", code, "zb", items)
		if e != nil {
			err = fmt.Errorf("递归指标分类失败, Err: %s", e.Error())
			return
		}
		items = resp

		// 地区维度
		if utils.InArrayByStr(regCodes, code) {
			regResp, e := curlAndFormatQuotaClassify("reg", code, "reg", items)
			if e != nil {
				err = fmt.Errorf("递归地区分类失败, Err: %s", e.Error())
				return
			}
			items = append(items, regResp...)
		}

		// 去重-code+维度code+ID
		classifyMap := make(map[string]bool)
		classifyOB := new(models.BaseFromNationalStatisticsClassify)
		classifyPars := make([]interface{}, 0)
		classifies, e := classifyOB.GetItemsByCondition("", classifyPars, []string{}, "")
		if e != nil {
			err = fmt.Errorf("获取指标分类列表失败, Err: %s", e.Error())
			return
		}
		for _, c := range classifies {
			classifyMap[fmt.Sprintf("%s%s%s", code, c.Wdcode, c.Id)] = true
		}

		finalList := make([]*models.BaseFromNationalStatisticsClassify, 0)
		for _, v := range items {
			if classifyMap[fmt.Sprintf("%s%s%s", code, v.Wdcode, v.Id)] {
				continue
			}
			finalList = append(finalList, v)
		}
		if e = classifyOB.CreateMulti(items); e != nil {
			err = fmt.Errorf("批量新增指标分类失败, Err: %s", e.Error())
			return
		}
		fmt.Println("结束同步DbCode: ", code)
	}
	return
}

// curlAndFormatQuotaClassify 递归请求分类树
func curlAndFormatQuotaClassify(id, dbcode, wdcode string, items []*models.BaseFromNationalStatisticsClassify) (resp []*models.BaseFromNationalStatisticsClassify, err error) {
	f := url.Values{}
	f.Add("id", id)
	f.Add("dbcode", dbcode)
	f.Add("wdcode", wdcode)
	f.Add("m", "getTree")
	r, e := NationalHttpPost(NationalStatisticsBaseReqUrl, f.Encode())
	if e != nil {
		err = fmt.Errorf("请求指标分类树失败, Err: %s", e.Error())
		return
	}
	list := make([]*QuotaClassifyTreeResp, 0)
	if e = json.Unmarshal(r, &list); e != nil {
		err = fmt.Errorf("解析响应数据失败, Err: %s", e.Error())
		return
	}

	nowTime := time.Now().Local()
	for _, v := range list {
		isParent := 0
		if v.IsParent {
			isParent = 1
		}
		fmt.Printf("Dbcode: %s, ClassifyName: %s, Id: %s\n", v.Dbcode, v.Name, v.Id)
		items = append(items, &models.BaseFromNationalStatisticsClassify{
			ClassifyName: v.Name,
			Id:           v.Id,
			Dbcode:       v.Dbcode,
			Wdcode:       v.Wdcode,
			Pid:          v.Pid,
			IsParent:     isParent,
			CreateTime:   nowTime,
			ModifyTime:   nowTime,
		})

		// 向下递归
		if isParent == 1 {
			time.Sleep(5 * time.Second) // 缓缓...毕竟接口是人家的...
			items, e = curlAndFormatQuotaClassify(v.Id, v.Dbcode, v.Wdcode, items)
			if e != nil {
				err = fmt.Errorf("递归请求分类树失败, Err: %s", e.Error())
				return
			}
		}
	}
	return items, nil
}