新增icpi爬虫

services/icpi.go

@@ -27,150 +27,11 @@ func CrawlerIcpi(cont context.Context) (err error) {
 	cookieStr = strings.Trim(cookieStr, ";")
 	fmt.Println(cookieStr)
 	//获取分类
-	//CrawlerIcpiClassify(cookieStr)
+	CrawlerIcpiClassify(cookieStr)
 	//大类
-	//CrawlerIcpiMaxClassifyIndex(cookieStr)
+	CrawlerIcpiMaxClassifyIndex(cookieStr)
 	//中类
 	CrawlerIcpiMiddleClassifyIndex(cookieStr)
-
-	//list := make([]*IcpiItem, 0)
-	//err = json.Unmarshal(body, &list)
-	//if err != nil {
-	//	fmt.Println("CrawlerIcpi json.Unmarshal Err:" + err.Error())
-	//	return
-	//}
-	//
-	//icpiIndexObj := new(models.BaseFromIcpiIndex)
-	//
-	//indexAll, err := icpiIndexObj.GetBaseFromIcpiIndexAll()
-	//if err != nil {
-	//	fmt.Println("GetBaseFromIcpiIndexAll Err:" + err.Error())
-	//	return
-	//}
-	//
-	//indexMap := make(map[string]*models.BaseFromIcpiIndex)
-	//for _, v := range indexAll {
-	//	indexMap[v.IndexCode] = v
-	//}
-	//
-	//limit := 2
-	//for _, v := range list {
-	//	indexCode := strings.ReplaceAll(v.EnglishName, " ", "_")
-	//
-	//	dataList, err := icpiIndexObj.GetBaseFromIcpiData(indexCode, limit)
-	//	if err != nil {
-	//		fmt.Println("GetBaseFromIcpiIndexAll Err:" + err.Error())
-	//		return err
-	//	}
-	//
-	//	dataMap := make(map[string]string)
-	//	for _, dv := range dataList {
-	//		dataMap[dv.DataTime] = dv.Value
-	//	}
-	//
-	//	if indexItem, ok := indexMap[indexCode]; !ok { //新增指标
-	//		icpiIndexObj = new(models.BaseFromIcpiIndex)
-	//		icpiIndexObj.IndexCode = indexCode
-	//		icpiIndexObj.IndexName = v.Name
-	//		icpiIndexObj.Frequency = "日度"
-	//		icpiIndexObj.CreateTime = time.Now()
-	//		icpiIndexObj.ModifyTime = time.Now()
-	//		err = icpiIndexObj.Add()
-	//		if err != nil {
-	//			fmt.Println("icpiIndexObj Add Err:" + err.Error())
-	//			return err
-	//		}
-	//
-	//		//新增指标对应的数据
-	//		dataItem := new(models.BaseFromIcpiData)
-	//		dataItem.IndexCode = indexCode
-	//		dataItem.BaseFromIcpiIndexId = icpiIndexObj.BaseFromIcpiIndexId
-	//		dataItem.DataTime = time.Now().Format(utils.FormatDate)
-	//		dataItem.Value = v.Today
-	//		dataItem.CreateTime = time.Now()
-	//		dataItem.ModifyTime = time.Now()
-	//		err = icpiIndexObj.AddData(dataItem)
-	//		if err != nil {
-	//			fmt.Println("icpiIndexObj AddData Err:" + err.Error())
-	//			return err
-	//		}
-	//
-	//		//新增指标对应的数据
-	//		dataItem = new(models.BaseFromIcpiData)
-	//		dataItem.IndexCode = indexCode
-	//		dataItem.BaseFromIcpiIndexId = icpiIndexObj.BaseFromIcpiIndexId
-	//		dataItem.DataTime = time.Now().AddDate(0, 0, -1).Format(utils.FormatDate)
-	//		dataItem.Value = v.Yesterday
-	//		dataItem.CreateTime = time.Now()
-	//		dataItem.ModifyTime = time.Now()
-	//		err = icpiIndexObj.AddData(dataItem)
-	//		if err != nil {
-	//			fmt.Println("icpiIndexObj AddData Err:" + err.Error())
-	//			return err
-	//		}
-	//	} else {
-	//		today := time.Now().Format(utils.FormatDate)
-	//		if dataVal, ok := dataMap[today]; !ok {
-	//			dataItem := new(models.BaseFromIcpiData)
-	//			dataItem.IndexCode = indexCode
-	//			dataItem.BaseFromIcpiIndexId = indexItem.BaseFromIcpiIndexId
-	//			dataItem.DataTime = today
-	//			dataItem.Value = v.Today
-	//			dataItem.CreateTime = time.Now()
-	//			dataItem.ModifyTime = time.Now()
-	//			err = icpiIndexObj.AddData(dataItem)
-	//			if err != nil {
-	//				fmt.Println("icpiIndexObj AddData Err:" + err.Error())
-	//				return err
-	//			}
-	//		} else {
-	//			if dataVal != v.Today {
-	//				err = icpiIndexObj.ModifyData(indexCode, today, v.Today)
-	//				if err != nil {
-	//					fmt.Println("icpiIndexObj ModifyData Err:" + err.Error())
-	//					return err
-	//				}
-	//			}
-	//		}
-	//
-	//		yesterday := time.Now().AddDate(0, 0, -1).Format(utils.FormatDate)
-	//		if dataVal, ok := dataMap[yesterday]; !ok {
-	//			dataItem := new(models.BaseFromIcpiData)
-	//			dataItem.IndexCode = indexCode
-	//			dataItem.BaseFromIcpiIndexId = indexItem.BaseFromIcpiIndexId
-	//			dataItem.DataTime = yesterday
-	//			dataItem.Value = v.Yesterday
-	//			dataItem.CreateTime = time.Now()
-	//			dataItem.ModifyTime = time.Now()
-	//			err = icpiIndexObj.AddData(dataItem)
-	//			if err != nil {
-	//				fmt.Println("icpiIndexObj AddData Err:" + err.Error())
-	//				return err
-	//			}
-	//		} else {
-	//			if dataVal != v.Today {
-	//				err = icpiIndexObj.ModifyData(indexCode, yesterday, v.Today)
-	//				if err != nil {
-	//					fmt.Println("icpiIndexObj ModifyData Err:" + err.Error())
-	//					return err
-	//				}
-	//			}
-	//		}
-	//	}
-	//
-	//	maxItem, err := icpiIndexObj.GetBaseFromIcpiIndexMaxAndMinInfo(indexCode)
-	//	if err != nil {
-	//		fmt.Println("GetBaseFromIcpiIndexMaxAndMinInfo Err:" + err.Error())
-	//		return err
-	//	}
-	//
-	//	err = icpiIndexObj.ModifyEdbInfoMaxAndMinInfo(indexCode, maxItem)
-	//	if err != nil {
-	//		fmt.Println("ModifyEdbInfoMaxAndMinInfo Err:" + err.Error())
-	//		return err
-	//	}
-
-	//}
 	return err
 }
 
@@ -237,6 +98,9 @@ func CrawlerIcpiClassify(cookieStr string) (err error) {
 	}
 	for _, v := range classifyList {
 		if _, ok := classifyMap[v.Category]; !ok {
+			if v.Category == "总类" {
+				v.Category = "日总类"
+			}
 			classifyItem := new(models.BaseFromIcpiClassify)
 			classifyItem.ClassifyName = v.Category
 			classifyItem.ClassifyNameEn = v.Categoryenglish

services/task.go

@@ -29,7 +29,7 @@ func Task() {
 	syncYearComTrade := task.NewTask("SyncYearComTrade", "0 0 3 1 1 *", SyncYearComTrade)                // 每年一月一号同步
 	syncYearMonthComTrade := task.NewTask("SyncYearMonthComTrade", "0 0 3 1 * *", SyncYearMonthComTrade) // 每月1号同步
 
-	//crawlerIcpi := task.NewTask("refreshData", "0 0,30 16-18 * * *", CrawlerIcpi) //居民消费价格指数
+	crawlerIcpi := task.NewTask("refreshData", "0 0,30 16-23 * * *", CrawlerIcpi) //居民消费价格指数
 
 	// 统计局-分月季年爬
 	//refreshNationalMonthA := task.NewTask("RefreshNationalMonthDbA", "0 15 2 10 * *", national_data.RefreshNationalMonthDbA)
@@ -47,7 +47,7 @@ func Task() {
 	task.AddTask("UN年度数据", syncYearComTrade)      //每年一月一号的3点同步
 	task.AddTask("UN月度数据", syncYearMonthComTrade) //每月1号的3点同步
 
-	//task.AddTask("居民消费价格指数", crawlerIcpi) //每月1号的3点同步
+	task.AddTask("居民消费价格指数", crawlerIcpi) //每月1号的3点同步
 
 	//task.AddTask("统计局数据爬取-月度A", refreshNationalMonthA) // 每月10号2:15执行
 	//task.AddTask("统计局数据爬取-月度B", refreshNationalMonthB) // 每月16号2:15执行