package national_data import ( "encoding/json" "eta/eta_crawler/models" "eta/eta_crawler/services/alarm_msg" "eta/eta_crawler/utils" "fmt" "net/url" "time" ) // QuotaClassifyTreeResp 指标分类树响应 type QuotaClassifyTreeResp struct { Id string `description:"分类ID(字符串)"` IsParent bool `description:"是否为父级"` Name string `description:"分类名称"` Pid string `description:"父级分类ID"` Dbcode string `description:"源-dbocde"` Wdcode string `description:"源-wdcode"` } // SyncQuotaClassifyTree 同步指标分类树 func SyncQuotaClassifyTree() (err error) { defer func() { if err != nil { utils.FileLog.Error("统计局-同步指标分类树失败, ErrMsg: %s", err.Error()) go alarm_msg.SendAlarmMsg(fmt.Sprintf("统计局-同步指标分类树失败, ErrMsg: %s", err.Error()), 3) } }() // 含指标维度 dbCodes := []string{ "hgyd", "hgjd", "hgnd", // 月度/季度/年度 "fsyd", "fsjd", "fsnd", "csyd", "csnd", "gatyd", "gatnd", // 分省月度/分省季度/分省年度/城市月度价格/城市年度数据/港澳台月度/港澳台年度 "gjyd", "gjydsdj", "gjydsc", "gjnd", // 主要国家月度/三大经济体月度/国际市场月度商品/国家年度 } // 含地区维度 regCodes := []string{ "fsyd", "fsjd", "fsnd", "csyd", "csnd", "gatyd", "gatnd", // 分省月度/分省季度/分省年度/城市月度价格/城市年度数据/港澳台月度/港澳台年度 "gjyd", "gjydsdj", "gjydsc", "gjnd", // 主要国家月度/三大经济体月度/国际市场月度商品/国家年度 } for _, code := range dbCodes { fmt.Println("开始同步DbCode: ", code) // 指标维度 items := make([]*models.BaseFromNationalStatisticsClassify, 0) resp, e := curlAndFormatQuotaClassify("zb", code, "zb", items) if e != nil { err = fmt.Errorf("递归指标分类失败, Err: %s", e.Error()) return } items = resp // 地区维度 if utils.InArrayByStr(regCodes, code) { regResp, e := curlAndFormatQuotaClassify("reg", code, "reg", items) if e != nil { err = fmt.Errorf("递归地区分类失败, Err: %s", e.Error()) return } items = append(items, regResp...) } // 去重-code+维度code+ID classifyMap := make(map[string]bool) classifyOB := new(models.BaseFromNationalStatisticsClassify) classifyPars := make([]interface{}, 0) classifies, e := classifyOB.GetItemsByCondition("", classifyPars, []string{}, "") if e != nil { err = fmt.Errorf("获取指标分类列表失败, Err: %s", e.Error()) return } for _, c := range classifies { classifyMap[fmt.Sprintf("%s%s%s", code, c.Wdcode, c.Id)] = true } finalList := make([]*models.BaseFromNationalStatisticsClassify, 0) for _, v := range items { if classifyMap[fmt.Sprintf("%s%s%s", code, v.Wdcode, v.Id)] { continue } finalList = append(finalList, v) } if e = classifyOB.CreateMulti(items); e != nil { err = fmt.Errorf("批量新增指标分类失败, Err: %s", e.Error()) return } fmt.Println("结束同步DbCode: ", code) } return } // curlAndFormatQuotaClassify 递归请求分类树 func curlAndFormatQuotaClassify(id, dbcode, wdcode string, items []*models.BaseFromNationalStatisticsClassify) (resp []*models.BaseFromNationalStatisticsClassify, err error) { f := url.Values{} f.Add("id", id) f.Add("dbcode", dbcode) f.Add("wdcode", wdcode) f.Add("m", "getTree") r, e := NationalHttpPost(NationalStatisticsBaseReqUrl, f.Encode()) if e != nil { err = fmt.Errorf("请求指标分类树失败, Err: %s", e.Error()) return } list := make([]*QuotaClassifyTreeResp, 0) if e = json.Unmarshal(r, &list); e != nil { err = fmt.Errorf("解析响应数据失败, Err: %s", e.Error()) return } nowTime := time.Now().Local() for _, v := range list { isParent := 0 if v.IsParent { isParent = 1 } fmt.Printf("Dbcode: %s, ClassifyName: %s, Id: %s\n", v.Dbcode, v.Name, v.Id) items = append(items, &models.BaseFromNationalStatisticsClassify{ ClassifyName: v.Name, Id: v.Id, Dbcode: v.Dbcode, Wdcode: v.Wdcode, Pid: v.Pid, IsParent: isParent, CreateTime: nowTime, ModifyTime: nowTime, }) // 向下递归 if isParent == 1 { time.Sleep(5 * time.Second) // 缓缓...毕竟接口是人家的... items, e = curlAndFormatQuotaClassify(v.Id, v.Dbcode, v.Wdcode, items) if e != nil { err = fmt.Errorf("递归请求分类树失败, Err: %s", e.Error()) return } } } return items, nil }