新增消费者指数爬虫

models/base_from_icpi.go

@@ -6,14 +6,15 @@ import (
 )
 
 type BaseFromIcpiIndex struct {
-	BaseFromIcpiIndexId int       `orm:"column(base_from_icpi_index_id);pk"`
-	IndexCode           string    `description:"指标编码"`
-	IndexName           string    `description:"指标名称"`
-	Frequency           string    `description:"频度"`
-	StartDate           time.Time `description:"开始日期"`
-	EndDate             time.Time `description:"结束日期"`
-	CreateTime          time.Time `description:"创建时间"`
-	ModifyTime          time.Time `description:"修改时间"`
+	BaseFromIcpiIndexId    int       `orm:"column(base_from_icpi_index_id);pk"`
+	BaseFromIcpiClassifyId int       `description:"分类id"`
+	IndexCode              string    `description:"指标编码"`
+	IndexName              string    `description:"指标名称"`
+	Frequency              string    `description:"频度"`
+	StartDate              time.Time `description:"开始日期"`
+	EndDate                time.Time `description:"结束日期"`
+	CreateTime             time.Time `description:"创建时间"`
+	ModifyTime             time.Time `description:"修改时间"`
 }
 
 type BaseFromIcpiData struct {
@@ -26,6 +27,15 @@ type BaseFromIcpiData struct {
 	ModifyTime          time.Time `description:"修改时间"`
 }
 
+type BaseFromIcpiClassify struct {
+	BaseFromIcpiClassifyId int       `orm:"column(base_from_icpi_classify_id);pk"`
+	ClassifyName           string    `description:"分类名称"`
+	ClassifyNameEn         string    `description:"英文名称"`
+	ParentId               int       `description:"上级id"`
+	CreateTime             time.Time `description:"创建时间"`
+	ModifyTime             time.Time `description:"修改时间"`
+}
+
 func (obj *BaseFromIcpiIndex) GetBaseFromIcpiIndexAll() (list []*BaseFromIcpiIndex, err error) {
 	o := orm.NewOrmUsingDB("data")
 	sql := `SELECT * FROM base_from_icpi_index`
@@ -43,6 +53,16 @@ func (m *BaseFromIcpiIndex) Add() (err error) {
 	return
 }
 
+func (m *BaseFromIcpiIndex) AddClassify(classify *BaseFromIcpiClassify) (err error) {
+	o := orm.NewOrmUsingDB("data")
+	id, err := o.Insert(classify)
+	if err != nil {
+		return
+	}
+	m.BaseFromIcpiIndexId = int(id)
+	return
+}
+
 func (obj *BaseFromIcpiIndex) GetBaseFromIcpiData(indexCode string, limit int) (list []*BaseFromIcpiData, err error) {
 	o := orm.NewOrmUsingDB("data")
 	sql := `SELECT * FROM base_from_icpi_data WHERE index_code=? ORDER BY data_time DESC LIMIT ?`
@@ -91,3 +111,17 @@ func (m *BaseFromIcpiIndex) ModifyEdbInfoMaxAndMinInfo(indexCode string, item *I
 	_, err = o.Raw(sql, item.MinDate, item.MaxDate, indexCode).Exec()
 	return
 }
+
+type IcpiClassify struct {
+	Id              interface{} `json:"id"`
+	Category        string      `json:"category"`
+	Percentage      interface{} `json:"percentage"`
+	Categoryenglish string      `json:"categoryenglish"`
+}
+
+func (obj *BaseFromIcpiIndex) GetBaseFromIcpiClassifyAll() (list []*BaseFromIcpiClassify, err error) {
+	o := orm.NewOrmUsingDB("data")
+	sql := `SELECT * FROM base_from_icpi_classify `
+	_, err = o.Raw(sql).QueryRows(&list)
+	return
+}

models/db.go

@@ -64,5 +64,7 @@ func init() {
 		new(BaseFromTradeGuangzhouIndex),
 		new(BaseFromTradeGuangzhouData),
 		new(BaseFromTradeGuangzhouContract),
+
+		new(BaseFromIcpiClassify),
 	)
 }

models/trade_guangzhou.go

@@ -55,6 +55,7 @@ type BaseFromTradeGuangzhouIndex struct {
 	EndDate                          string    `description:"结束日期"`
 	CreateTime                       time.Time `description:"创建日期"`
 	ModifyTime                       time.Time `description:"修改日期"`
+	Value                            float64   `description:"数据值"`
 }
 
 type BaseFromTradeGuangzhouData struct {

services/icpi.go

@@ -26,12 +26,160 @@ func CrawlerIcpi(cont context.Context) (err error) {
 
 	cookieStr = strings.Trim(cookieStr, ";")
 	fmt.Println(cookieStr)
-	fmt.Println("end")
+	//获取分类
+	//CrawlerIcpiClassify(cookieStr)
+	//大类
+	//CrawlerIcpiMaxClassifyIndex(cookieStr)
+	//中类
+	CrawlerIcpiMiddleClassifyIndex(cookieStr)
+
+	//list := make([]*IcpiItem, 0)
+	//err = json.Unmarshal(body, &list)
+	//if err != nil {
+	//	fmt.Println("CrawlerIcpi json.Unmarshal Err:" + err.Error())
+	//	return
+	//}
+	//
+	//icpiIndexObj := new(models.BaseFromIcpiIndex)
+	//
+	//indexAll, err := icpiIndexObj.GetBaseFromIcpiIndexAll()
+	//if err != nil {
+	//	fmt.Println("GetBaseFromIcpiIndexAll Err:" + err.Error())
+	//	return
+	//}
+	//
+	//indexMap := make(map[string]*models.BaseFromIcpiIndex)
+	//for _, v := range indexAll {
+	//	indexMap[v.IndexCode] = v
+	//}
+	//
+	//limit := 2
+	//for _, v := range list {
+	//	indexCode := strings.ReplaceAll(v.EnglishName, " ", "_")
+	//
+	//	dataList, err := icpiIndexObj.GetBaseFromIcpiData(indexCode, limit)
+	//	if err != nil {
+	//		fmt.Println("GetBaseFromIcpiIndexAll Err:" + err.Error())
+	//		return err
+	//	}
+	//
+	//	dataMap := make(map[string]string)
+	//	for _, dv := range dataList {
+	//		dataMap[dv.DataTime] = dv.Value
+	//	}
+	//
+	//	if indexItem, ok := indexMap[indexCode]; !ok { //新增指标
+	//		icpiIndexObj = new(models.BaseFromIcpiIndex)
+	//		icpiIndexObj.IndexCode = indexCode
+	//		icpiIndexObj.IndexName = v.Name
+	//		icpiIndexObj.Frequency = "日度"
+	//		icpiIndexObj.CreateTime = time.Now()
+	//		icpiIndexObj.ModifyTime = time.Now()
+	//		err = icpiIndexObj.Add()
+	//		if err != nil {
+	//			fmt.Println("icpiIndexObj Add Err:" + err.Error())
+	//			return err
+	//		}
+	//
+	//		//新增指标对应的数据
+	//		dataItem := new(models.BaseFromIcpiData)
+	//		dataItem.IndexCode = indexCode
+	//		dataItem.BaseFromIcpiIndexId = icpiIndexObj.BaseFromIcpiIndexId
+	//		dataItem.DataTime = time.Now().Format(utils.FormatDate)
+	//		dataItem.Value = v.Today
+	//		dataItem.CreateTime = time.Now()
+	//		dataItem.ModifyTime = time.Now()
+	//		err = icpiIndexObj.AddData(dataItem)
+	//		if err != nil {
+	//			fmt.Println("icpiIndexObj AddData Err:" + err.Error())
+	//			return err
+	//		}
+	//
+	//		//新增指标对应的数据
+	//		dataItem = new(models.BaseFromIcpiData)
+	//		dataItem.IndexCode = indexCode
+	//		dataItem.BaseFromIcpiIndexId = icpiIndexObj.BaseFromIcpiIndexId
+	//		dataItem.DataTime = time.Now().AddDate(0, 0, -1).Format(utils.FormatDate)
+	//		dataItem.Value = v.Yesterday
+	//		dataItem.CreateTime = time.Now()
+	//		dataItem.ModifyTime = time.Now()
+	//		err = icpiIndexObj.AddData(dataItem)
+	//		if err != nil {
+	//			fmt.Println("icpiIndexObj AddData Err:" + err.Error())
+	//			return err
+	//		}
+	//	} else {
+	//		today := time.Now().Format(utils.FormatDate)
+	//		if dataVal, ok := dataMap[today]; !ok {
+	//			dataItem := new(models.BaseFromIcpiData)
+	//			dataItem.IndexCode = indexCode
+	//			dataItem.BaseFromIcpiIndexId = indexItem.BaseFromIcpiIndexId
+	//			dataItem.DataTime = today
+	//			dataItem.Value = v.Today
+	//			dataItem.CreateTime = time.Now()
+	//			dataItem.ModifyTime = time.Now()
+	//			err = icpiIndexObj.AddData(dataItem)
+	//			if err != nil {
+	//				fmt.Println("icpiIndexObj AddData Err:" + err.Error())
+	//				return err
+	//			}
+	//		} else {
+	//			if dataVal != v.Today {
+	//				err = icpiIndexObj.ModifyData(indexCode, today, v.Today)
+	//				if err != nil {
+	//					fmt.Println("icpiIndexObj ModifyData Err:" + err.Error())
+	//					return err
+	//				}
+	//			}
+	//		}
+	//
+	//		yesterday := time.Now().AddDate(0, 0, -1).Format(utils.FormatDate)
+	//		if dataVal, ok := dataMap[yesterday]; !ok {
+	//			dataItem := new(models.BaseFromIcpiData)
+	//			dataItem.IndexCode = indexCode
+	//			dataItem.BaseFromIcpiIndexId = indexItem.BaseFromIcpiIndexId
+	//			dataItem.DataTime = yesterday
+	//			dataItem.Value = v.Yesterday
+	//			dataItem.CreateTime = time.Now()
+	//			dataItem.ModifyTime = time.Now()
+	//			err = icpiIndexObj.AddData(dataItem)
+	//			if err != nil {
+	//				fmt.Println("icpiIndexObj AddData Err:" + err.Error())
+	//				return err
+	//			}
+	//		} else {
+	//			if dataVal != v.Today {
+	//				err = icpiIndexObj.ModifyData(indexCode, yesterday, v.Today)
+	//				if err != nil {
+	//					fmt.Println("icpiIndexObj ModifyData Err:" + err.Error())
+	//					return err
+	//				}
+	//			}
+	//		}
+	//	}
+	//
+	//	maxItem, err := icpiIndexObj.GetBaseFromIcpiIndexMaxAndMinInfo(indexCode)
+	//	if err != nil {
+	//		fmt.Println("GetBaseFromIcpiIndexMaxAndMinInfo Err:" + err.Error())
+	//		return err
+	//	}
+	//
+	//	err = icpiIndexObj.ModifyEdbInfoMaxAndMinInfo(indexCode, maxItem)
+	//	if err != nil {
+	//		fmt.Println("ModifyEdbInfoMaxAndMinInfo Err:" + err.Error())
+	//		return err
+	//	}
+
+	//}
+	return err
+}
+
+func CrawlerIcpiClassify(cookieStr string) (err error) {
 	// 创建一个HTTP客户端
 	client := &http.Client{}
 	// 创建一个GET请求
-	data := strings.NewReader("laber=1")
-	req, err := http.NewRequest("POST", "http://www.bdecon.com/indexqueryiCPI/", data)
+	//data := strings.NewReader("laber=1")
+	req, err := http.NewRequest("POST", "http://www.bdecon.com/monthly/LargeWeightListNew1/", nil)
 	if err != nil {
 		fmt.Println("创建请求失败:", err)
 		return
@@ -50,7 +198,7 @@ func CrawlerIcpi(cont context.Context) (err error) {
 	req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36")
 	req.Header.Set("X-Requested-With", "XMLHttpRequest")
 	// 发送请求并获取响应
-	resp, err = client.Do(req)
+	resp, err := client.Do(req)
 	if err != nil {
 		fmt.Println("发送请求失败:", err)
 		return
@@ -68,13 +216,308 @@ func CrawlerIcpi(cont context.Context) (err error) {
 	// 打印响应内容
 	fmt.Println(string(body))
 
-	list := make([]*IcpiItem, 0)
-	err = json.Unmarshal(body, &list)
+	classifyList := make([]models.IcpiClassify, 0)
+	err = json.Unmarshal(body, &classifyList)
 	if err != nil {
 		fmt.Println("CrawlerIcpi json.Unmarshal Err:" + err.Error())
 		return
 	}
 
+	icpiObj := new(models.BaseFromIcpiIndex)
+
+	classifyAll, err := icpiObj.GetBaseFromIcpiClassifyAll()
+	if err != nil {
+		fmt.Println("GetBaseFromIcpiClassifyAll Err:" + err.Error())
+		return
+	}
+
+	classifyMap := make(map[string]*models.BaseFromIcpiClassify)
+	for _, v := range classifyAll {
+		classifyMap[v.ClassifyName] = v
+	}
+	for _, v := range classifyList {
+		if _, ok := classifyMap[v.Category]; !ok {
+			classifyItem := new(models.BaseFromIcpiClassify)
+			classifyItem.ClassifyName = v.Category
+			classifyItem.ClassifyNameEn = v.Categoryenglish
+			classifyItem.ParentId = 0
+			classifyItem.CreateTime = time.Now()
+			classifyItem.ModifyTime = time.Now()
+			err = icpiObj.AddClassify(classifyItem)
+			if err != nil {
+				return err
+			}
+		}
+	}
+	return err
+}
+
+// 爬取ICPI指数大类及指标数据
+func CrawlerIcpiMaxClassifyIndex(cookieStr string) (err error) {
+	// 创建一个HTTP客户端
+	client := &http.Client{}
+	// 创建一个GET请求
+	icpiObj := new(models.BaseFromIcpiIndex)
+
+	classifyAll, err := icpiObj.GetBaseFromIcpiClassifyAll()
+	if err != nil {
+		fmt.Println("GetBaseFromIcpiClassifyAll Err:" + err.Error())
+		return
+	}
+
+	classifyMap := make(map[string]*models.BaseFromIcpiClassify)
+	for _, v := range classifyAll {
+		classifyMap[v.ClassifyName] = v
+	}
+
+	param := strings.NewReader("laber=1")
+	req, err := http.NewRequest("POST", "http://www.bdecon.com/indexqueryiCPI/", param)
+	if err != nil {
+		fmt.Println("创建请求失败:", err)
+		return
+	}
+	// 设置请求头
+	req.Header.Set("Accept", "*/*")
+	req.Header.Set("Accept-Encoding", "gzip, deflate")
+	req.Header.Set("Accept-Language", "zh-CN,zh;q=0.9,en;q=0.8")
+	req.Header.Set("Connection", "keep-alive")
+	req.Header.Set("Content-Length", "7")
+	req.Header.Set("Content-Type", "application/x-www-form-urlencoded; charset=UTF-8")
+	req.Header.Set("Cookie", cookieStr)
+	req.Header.Set("Host", "www.bdecon.com")
+	req.Header.Set("Origin", "http://www.bdecon.com")
+	req.Header.Set("Referer", "http://www.bdecon.com/dataIndex")
+	req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36")
+	req.Header.Set("X-Requested-With", "XMLHttpRequest")
+	// 发送请求并获取响应
+	resp, err := client.Do(req)
+	if err != nil {
+		fmt.Println("发送请求失败:", err)
+		return
+	}
+	defer resp.Body.Close()
+	// 读取响应的内容
+	body, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		fmt.Println("读取响应失败:", err)
+		return
+	}
+
+	utils.FileLog.Info("result:")
+	utils.FileLog.Info(string(body))
+	// 打印响应内容
+	fmt.Println(string(body))
+
+	indexList := make([]IcpiItem, 0)
+	err = json.Unmarshal(body, &indexList)
+	if err != nil {
+		fmt.Println("CrawlerIcpiMaxClassifyIndex json.Unmarshal Err:" + err.Error())
+		return
+	}
+
+	icpiIndexObj := new(models.BaseFromIcpiIndex)
+
+	indexAll, err := icpiIndexObj.GetBaseFromIcpiIndexAll()
+	if err != nil {
+		fmt.Println("GetBaseFromIcpiIndexAll Err:" + err.Error())
+		return
+	}
+
+	indexMap := make(map[string]*models.BaseFromIcpiIndex)
+	for _, v := range indexAll {
+		indexMap[v.IndexCode] = v
+	}
+
+	limit := 2
+	for _, v := range indexList {
+		indexCode = strings.ReplaceAll(v.EnglishName, " ", "_")
+
+		if indexItem, ok := indexMap[indexCode]; !ok { //新增指标
+			icpiIndexObj = new(models.BaseFromIcpiIndex)
+			icpiIndexObj.IndexCode = indexCode
+			if classify, ok := classifyMap[v.Name]; ok {
+				icpiIndexObj.BaseFromIcpiClassifyId = classify.BaseFromIcpiClassifyId
+			}
+
+			icpiIndexObj.IndexName = v.Name
+			icpiIndexObj.Frequency = "日度"
+			icpiIndexObj.CreateTime = time.Now()
+			icpiIndexObj.ModifyTime = time.Now()
+			err = icpiIndexObj.Add()
+			if err != nil {
+				fmt.Println("icpiIndexObj Add Err:" + err.Error())
+				return err
+			}
+
+			//新增指标对应的数据
+			dataItem := new(models.BaseFromIcpiData)
+			dataItem.IndexCode = indexCode
+			dataItem.BaseFromIcpiIndexId = icpiIndexObj.BaseFromIcpiIndexId
+			dataItem.DataTime = time.Now().Format(utils.FormatDate)
+			dataItem.Value = v.Today
+			dataItem.CreateTime = time.Now()
+			dataItem.ModifyTime = time.Now()
+			err = icpiIndexObj.AddData(dataItem)
+			if err != nil {
+				fmt.Println("icpiIndexObj AddData Err:" + err.Error())
+				return err
+			}
+
+			//新增指标对应的数据
+			dataItem = new(models.BaseFromIcpiData)
+			dataItem.IndexCode = indexCode
+			dataItem.BaseFromIcpiIndexId = icpiIndexObj.BaseFromIcpiIndexId
+			dataItem.DataTime = time.Now().AddDate(0, 0, -1).Format(utils.FormatDate)
+			dataItem.Value = v.Yesterday
+			dataItem.CreateTime = time.Now()
+			dataItem.ModifyTime = time.Now()
+			err = icpiIndexObj.AddData(dataItem)
+			if err != nil {
+				fmt.Println("icpiIndexObj AddData Err:" + err.Error())
+				return err
+			}
+		} else {
+			dataList, err := icpiIndexObj.GetBaseFromIcpiData(indexCode, limit)
+			if err != nil {
+				fmt.Println("GetBaseFromIcpiIndexAll Err:" + err.Error())
+				return err
+			}
+
+			dataMap := make(map[string]string)
+			for _, dv := range dataList {
+				dataMap[dv.DataTime] = dv.Value
+			}
+
+			today := time.Now().Format(utils.FormatDate)
+			if dataVal, ok := dataMap[today]; !ok {
+				dataItem := new(models.BaseFromIcpiData)
+				dataItem.IndexCode = indexCode
+				dataItem.BaseFromIcpiIndexId = indexItem.BaseFromIcpiIndexId
+				dataItem.DataTime = today
+				dataItem.Value = v.Today
+				dataItem.CreateTime = time.Now()
+				dataItem.ModifyTime = time.Now()
+				err = icpiIndexObj.AddData(dataItem)
+				if err != nil {
+					fmt.Println("icpiIndexObj AddData Err:" + err.Error())
+					return err
+				}
+			} else {
+				if dataVal != v.Today {
+					err = icpiIndexObj.ModifyData(indexCode, today, v.Today)
+					if err != nil {
+						fmt.Println("icpiIndexObj ModifyData Err:" + err.Error())
+						return err
+					}
+				}
+			}
+
+			yesterday := time.Now().AddDate(0, 0, -1).Format(utils.FormatDate)
+			if dataVal, ok := dataMap[yesterday]; !ok {
+				dataItem := new(models.BaseFromIcpiData)
+				dataItem.IndexCode = indexCode
+				dataItem.BaseFromIcpiIndexId = indexItem.BaseFromIcpiIndexId
+				dataItem.DataTime = yesterday
+				dataItem.Value = v.Yesterday
+				dataItem.CreateTime = time.Now()
+				dataItem.ModifyTime = time.Now()
+				err = icpiIndexObj.AddData(dataItem)
+				if err != nil {
+					fmt.Println("icpiIndexObj AddData Err:" + err.Error())
+					return err
+				}
+			} else {
+				if dataVal != v.Today {
+					err = icpiIndexObj.ModifyData(indexCode, yesterday, v.Today)
+					if err != nil {
+						fmt.Println("icpiIndexObj ModifyData Err:" + err.Error())
+						return err
+					}
+				}
+			}
+		}
+
+		maxItem, err := icpiIndexObj.GetBaseFromIcpiIndexMaxAndMinInfo(indexCode)
+		if err != nil {
+			fmt.Println("GetBaseFromIcpiIndexMaxAndMinInfo Err:" + err.Error())
+			return err
+		}
+
+		err = icpiIndexObj.ModifyEdbInfoMaxAndMinInfo(indexCode, maxItem)
+		if err != nil {
+			fmt.Println("ModifyEdbInfoMaxAndMinInfo Err:" + err.Error())
+			return err
+		}
+
+	}
+
+	return err
+}
+
+// 爬取ICPI中类指标信息
+func CrawlerIcpiMiddleClassifyIndex(cookieStr string) (err error) {
+	// 创建一个HTTP客户端
+	client := &http.Client{}
+	// 创建一个GET请求
+
+	icpiObj := new(models.BaseFromIcpiIndex)
+
+	classifyAll, err := icpiObj.GetBaseFromIcpiClassifyAll()
+	if err != nil {
+		fmt.Println("GetBaseFromIcpiClassifyAll Err:" + err.Error())
+		return
+	}
+
+	classifyMap := make(map[string]*models.BaseFromIcpiClassify)
+	for _, v := range classifyAll {
+		classifyMap[v.ClassifyName] = v
+	}
+
+	params := strings.NewReader("name=yueda=总类,食品烟酒,衣着,居住,生活用品及服务,交通和通信,教育文化和娱乐,医疗保健,其他用品和服务&yuezh=衣着——服装,衣着——服装材料,衣着——其他衣着及配件,衣着——衣着加工服务费,衣着——鞋类,交通和通信——交通,交通和通信——通信,生活用品及服务——家具及室内装饰品,生活用品及服务——家用器具,生活用品及服务——家用纺织品,生活用品及服务——家庭日用杂品,生活用品及服务——个人护理用品,生活用品及服务——家庭服务,居住——租赁房房租,居住——住房保养维修及管理,居住——水电燃料,居住——自有住房,医疗保健——药品及医疗器具,医疗保健——医疗服务,食品烟酒——食品,食品烟酒——茶及饮料,食品烟酒——烟酒,食品烟酒——在外餐饮,教育文化和娱乐——教育,教育文化和娱乐——文化娱乐,其他用品和服务——其他用品类,其他用品和服务——其他服务类&yuexi=&yuezi=&zhishu=日总类")
+	req, err := http.NewRequest("POST", "http://www.bdecon.com/indexqueryNew/", params)
+	if err != nil {
+		fmt.Println("创建请求失败:", err)
+		return
+	}
+	// 设置请求头
+	req.Header.Set("Accept", "*/*")
+	req.Header.Set("Accept-Encoding", "gzip, deflate")
+	req.Header.Set("Accept-Language", "zh-CN,zh;q=0.9,en;q=0.8")
+	req.Header.Set("Connection", "keep-alive")
+	req.Header.Set("Content-Length", "7")
+	req.Header.Set("Content-Type", "application/x-www-form-urlencoded; charset=UTF-8")
+	req.Header.Set("Cookie", cookieStr)
+	req.Header.Set("Host", "www.bdecon.com")
+	req.Header.Set("Origin", "http://www.bdecon.com")
+	req.Header.Set("Referer", "http://www.bdecon.com/dataIndex")
+	req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36")
+	req.Header.Set("X-Requested-With", "XMLHttpRequest")
+	// 发送请求并获取响应
+	resp, err := client.Do(req)
+	if err != nil {
+		fmt.Println("发送请求失败:", err)
+		return
+	}
+	defer resp.Body.Close()
+	// 读取响应的内容
+	body, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		fmt.Println("读取响应失败:", err)
+		return
+	}
+
+	utils.FileLog.Info("result:")
+	utils.FileLog.Info(string(body))
+	// 打印响应内容
+	fmt.Println(string(body))
+
+	indexList := make([]*IcpiItem, 0)
+	err = json.Unmarshal(body, &indexList)
+	if err != nil {
+		fmt.Println("CrawlerIcpiMiddleClassifyIndex json.Unmarshal Err:" + err.Error())
+		return
+	}
+
 	icpiIndexObj := new(models.BaseFromIcpiIndex)
 
 	indexAll, err := icpiIndexObj.GetBaseFromIcpiIndexAll()
@@ -89,8 +532,17 @@ func CrawlerIcpi(cont context.Context) (err error) {
 	}
 
 	limit := 2
-	for _, v := range list {
-		indexCode := strings.ReplaceAll(v.EnglishName, " ", "_")
+	for _, v := range indexList {
+		var indexCode string
+		if v.EnglishName == "" {
+			indexCode = utils.ChineseToPinyinInitials(v.Name)
+		} else {
+			indexCode = strings.ReplaceAll(v.EnglishName, " ", "_")
+		}
+
+		if indexCode == "" {
+			continue
+		}
 
 		dataList, err := icpiIndexObj.GetBaseFromIcpiData(indexCode, limit)
 		if err != nil {
@@ -104,9 +556,16 @@ func CrawlerIcpi(cont context.Context) (err error) {
 		}
 
 		if indexItem, ok := indexMap[indexCode]; !ok { //新增指标
+
+			classifyNameArr := strings.Split(v.Name, "-")
+			classifyName := classifyNameArr[0]
+
 			icpiIndexObj = new(models.BaseFromIcpiIndex)
 			icpiIndexObj.IndexCode = indexCode
 			icpiIndexObj.IndexName = v.Name
+			if classifyItem, ok := classifyMap[classifyName]; ok {
+				icpiIndexObj.BaseFromIcpiClassifyId = classifyItem.BaseFromIcpiClassifyId
+			}
 			icpiIndexObj.Frequency = "日度"
 			icpiIndexObj.CreateTime = time.Now()
 			icpiIndexObj.ModifyTime = time.Now()
@@ -206,6 +665,7 @@ func CrawlerIcpi(cont context.Context) (err error) {
 		}
 
 	}
+
 	return err
 }
 

services/task.go

@@ -29,7 +29,7 @@ func Task() {
 	syncYearComTrade := task.NewTask("SyncYearComTrade", "0 0 3 1 1 *", SyncYearComTrade)                // 每年一月一号同步
 	syncYearMonthComTrade := task.NewTask("SyncYearMonthComTrade", "0 0 3 1 * *", SyncYearMonthComTrade) // 每月1号同步
 
-	crawlerIcpi := task.NewTask("refreshData", "0 0,30 16-18 * * *", CrawlerIcpi) //居民消费价格指数
+	//crawlerIcpi := task.NewTask("refreshData", "0 0,30 16-18 * * *", CrawlerIcpi) //居民消费价格指数
 
 	// 统计局-分月季年爬
 	//refreshNationalMonthA := task.NewTask("RefreshNationalMonthDbA", "0 15 2 10 * *", national_data.RefreshNationalMonthDbA)
@@ -47,7 +47,7 @@ func Task() {
 	task.AddTask("UN年度数据", syncYearComTrade)      //每年一月一号的3点同步
 	task.AddTask("UN月度数据", syncYearMonthComTrade) //每月1号的3点同步
 
-	task.AddTask("居民消费价格指数", crawlerIcpi) //每月1号的3点同步
+	//task.AddTask("居民消费价格指数", crawlerIcpi) //每月1号的3点同步
 
 	//task.AddTask("统计局数据爬取-月度A", refreshNationalMonthA) // 每月10号2:15执行
 	//task.AddTask("统计局数据爬取-月度B", refreshNationalMonthB) // 每月16号2:15执行
@@ -170,20 +170,17 @@ func RefreshChangesVisitorsCovid(cont context.Context) (err error) {
 //}
 
 func Task123() {
-	fmt.Println("start")
-	n := 60
-	//日成交持仓排名
-	for i := n; i >= 0; i-- {
-		date := time.Now().AddDate(0, 0, -i)
-		if date.Weekday() == time.Saturday || date.Weekday() == time.Sunday {
-			continue
-		}
-		fmt.Println("date:", date.Format(utils.FormatDate))
-		CrawlerFromGuangzhouDealPosiQuotes(date)
-		rn := utils.RangeRand(3, 15)
-		fmt.Println("i:"+strconv.Itoa(i)+" date:", date.Format(utils.FormatDate)+";sleep:", strconv.Itoa(int(rn)))
-		time.Sleep(time.Duration(rn) * time.Second)
-	}
-	CrawlerFromGuangzhouIndexMinAndMaxDate()
-	fmt.Println("end")
+	fmt.Println("task start")
+	//n := 6
+	//for i := n; i >= 0; i-- {
+	//	date := time.Now().AddDate(0, -i, 0)
+	//	fmt.Println("date:", date.Format(utils.FormatDate))
+	//	CrawlerFromGuangzhouCongregateDeliveryQuotes(date)
+	//	rn := utils.RangeRand(3, 15)
+	//	fmt.Println("i:"+strconv.Itoa(i)+" date:", date.Format(utils.FormatDate)+";sleep:", strconv.Itoa(int(rn)))
+	//	time.Sleep(time.Duration(rn) * time.Second)
+	//}
+	ctx := new(context.Context)
+	CrawlerIcpi(*ctx)
+	fmt.Println("task end")
 }

services/trade_guangzhou.go

@@ -609,13 +609,13 @@ func CrawlerFromGuangzhouWbillWeeklyQuotes(date time.Time) {
 			if v.WhAbbr == "总计" {
 				continue
 			}
-			indexCode := utils.ChineseToPinyin(v.WhAbbr)
+			indexCode := utils.ChineseToPinyinInitials(v.WhAbbr)
 			if indexCode == "" {
 				continue
 			}
 			indexCode = v.VarietyOrder + indexCode
-			indexCode += "wbill"
-			indexName := v.WhAbbr + "仓单量"
+			indexCode += "cdl"
+			indexName := v.Variety + v.WhAbbr + "仓单量"
 
 			var indexId int
 			if indexItem, ok := indexMap[indexCode]; !ok {
@@ -772,74 +772,159 @@ func CrawlerFromGuangzhouCongregateDeliveryQuotes(date time.Time) {
 	if deliveryQuotes.Code == "0" {
 		addDataList := make([]*models.BaseFromTradeGuangzhouData, 0)
 		updateDataList := make([]*models.BaseFromTradeGuangzhouData, 0)
+
+		if len(deliveryQuotes.Data) <= 0 {
+			return
+		}
+
+		varietyId := deliveryQuotes.Data[0].VarietyId
+		varietyName := deliveryQuotes.Data[0].VarietyName
+
+		totalIndexCode := varietyId + utils.ChineseToPinyinInitials("一次性交割卖方仓单合计")
+		totalIndexName := varietyName + "一次性交割卖方仓单合计"
+
+		total := 0
 		for _, v := range deliveryQuotes.Data {
-			indexCode := utils.ChineseToPinyin(v.WhName)
-			if indexCode == "" {
-				continue
+			total += v.WbillNum
+		}
+
+		var indexId int
+		if indexItem, ok := indexMap[totalIndexCode]; !ok {
+			var classifyId int
+			if classifyItem, ok := classifyMap[varietyId]; !ok {
+				classifyItem = new(models.BaseFromTradeGuangzhouClassify)
+				classifyItem.ClassifyCode = varietyId
+				classifyItem.ClassifyName = varietyName
+				classifyItem.ParentId = parentId
+				classifyItem.ModifyTime = time.Now()
+				classifyItem.CreateTime = time.Now()
+				classifyLastId, err := gzObj.AddBaseFromTradeGuangzhouClassify(classifyItem)
+				if err != nil {
+					fmt.Println("AddBaseFromTradeGuangzhouClassify err:" + err.Error())
+					return
+				}
+				classifyId = int(classifyLastId)
+			} else {
+				classifyId = classifyItem.BaseFromTradeGuangzhouClassifyId
 			}
-			indexCode += "wbillseller"
-			indexName := v.WhName + "一次性交割卖方仓单量"
 
-			var indexId int
-			if indexItem, ok := indexMap[indexCode]; !ok {
-				var classifyId int
-				if classifyItem, ok := classifyMap[v.VarietyId]; !ok {
-					classifyItem = new(models.BaseFromTradeGuangzhouClassify)
-					classifyItem.ClassifyCode = v.VarietyId
-					classifyItem.ClassifyName = v.VarietyName
-					classifyItem.ParentId = 3
-					classifyItem.ModifyTime = time.Now()
-					classifyItem.CreateTime = time.Now()
-					classifyLastId, err := gzObj.AddBaseFromTradeGuangzhouClassify(classifyItem)
-					if err != nil {
-						fmt.Println("AddBaseFromTradeGuangzhouClassify err:" + err.Error())
-						return
-					}
-					classifyId = int(classifyLastId)
-				} else {
-					classifyId = classifyItem.BaseFromTradeGuangzhouClassifyId
+			gzObj = &models.BaseFromTradeGuangzhouIndex{
+				BaseFromTradeGuangzhouClassifyId: classifyId,
+				IndexCode:                        totalIndexCode,
+				IndexName:                        totalIndexName,
+				Frequency:                        "月度",
+				Unit:                             "手",
+				CreateTime:                       time.Now(),
+				ModifyTime:                       time.Now(),
+			}
+			lastId, err := gzObj.AddBaseFromTradeGuangzhouIndex()
+			if err != nil {
+				fmt.Println("AddBaseFromTradeGuangzhouIndex Err:" + err.Error())
+				return
+			}
+			indexId = int(lastId)
+		} else {
+			indexId = indexItem.BaseFromTradeGuangzhouIndexId
+		}
+
+		//判断数据是否存在
+		if dataItem, ok := dataMap[totalIndexCode]; !ok {
+			dataItem = new(models.BaseFromTradeGuangzhouData)
+			dataItem.BaseFromTradeGuangzhouIndexId = indexId
+			dataItem.IndexCode = totalIndexCode
+			dataItem.DataTime = monthEndDay
+			dataItem.Value = float64(total)
+			dataItem.CreateTime = time.Now()
+			dataItem.ModifyTime = time.Now()
+			addDataList = append(addDataList, dataItem)
+		} else {
+			if dataItem.Value != float64(total) {
+				dataItem.BaseFromTradeGuangzhouDataId = dataItem.BaseFromTradeGuangzhouDataId
+				dataItem.BaseFromTradeGuangzhouIndexId = indexId
+				dataItem.IndexCode = totalIndexCode
+				dataItem.DataTime = monthEndDay
+				dataItem.Value = float64(total)
+				dataItem.ModifyTime = time.Now()
+				updateDataList = append(updateDataList, dataItem)
+			}
+		}
+		fmt.Println("addDataList:", len(addDataList))
+		if len(addDataList) > 0 {
+			err = gzObj.AddBaseFromTradeGuangzhouData(addDataList)
+			if err != nil {
+				fmt.Println("AddBaseFromTradeGuangzhouData Err:" + err.Error())
+				return
+			}
+		}
+
+		if len(updateDataList) > 0 {
+			for _, v := range updateDataList {
+				updateParams := make(map[string]interface{})
+				whereParam := make(map[string]interface{})
+				updateParams["value"] = v.Value
+				updateParams["qty_sub"] = v.QtySub
+				updateParams["modify_time"] = time.Now()
+				whereParam["base_from_trade_guangzhou_data_id"] = v.BaseFromTradeGuangzhouDataId
+				err = gzObj.UpdateGuangzhouData(updateParams, whereParam)
+				if err != nil {
+					fmt.Println("UpdateGuangzhouData Err:" + err.Error())
+					return
 				}
+			}
+		}
+
+		//处理明细
+		addDataList = make([]*models.BaseFromTradeGuangzhouData, 0)
+		updateDataList = make([]*models.BaseFromTradeGuangzhouData, 0)
+
+		for _, v := range deliveryQuotes.Data {
+			var subIndexId int
+			indexCode = v.ContractId + utils.ChineseToPinyinInitials(v.WhName) + "cdl"
+			if indexItem, ok := indexMap[indexCode]; !ok {
 				gzObj = &models.BaseFromTradeGuangzhouIndex{
-					BaseFromTradeGuangzhouClassifyId: classifyId,
+					BaseFromTradeGuangzhouClassifyId: indexId,
 					IndexCode:                        indexCode,
-					IndexName:                        indexName,
+					IndexName:                        v.WhName,
 					Frequency:                        "月度",
 					Unit:                             "手",
 					CreateTime:                       time.Now(),
 					ModifyTime:                       time.Now(),
+					Value:                            float64(v.WbillNum),
+					StartDate:                        monthEndDay,
+					EndDate:                          monthEndDay,
 				}
 				lastId, err := gzObj.AddBaseFromTradeGuangzhouIndex()
 				if err != nil {
 					fmt.Println("AddBaseFromTradeGuangzhouIndex Err:" + err.Error())
 					return
 				}
-				indexId = int(lastId)
+				subIndexId = int(lastId)
 			} else {
-				indexId = indexItem.BaseFromTradeGuangzhouIndexId
+				subIndexId = indexItem.BaseFromTradeGuangzhouIndexId
 			}
 			//判断数据是否存在
 			if dataItem, ok := dataMap[indexCode]; !ok {
 				dataItem = new(models.BaseFromTradeGuangzhouData)
-				dataItem.BaseFromTradeGuangzhouIndexId = indexId
+				dataItem.BaseFromTradeGuangzhouIndexId = subIndexId
 				dataItem.IndexCode = indexCode
 				dataItem.DataTime = monthEndDay
-				dataItem.Value = float64(v.WbillNum)
+				dataItem.Value = float64(total)
 				dataItem.CreateTime = time.Now()
 				dataItem.ModifyTime = time.Now()
 				addDataList = append(addDataList, dataItem)
 			} else {
-				if dataItem.Value != float64(v.WbillNum) {
+				if dataItem.Value != float64(total) {
 					dataItem.BaseFromTradeGuangzhouDataId = dataItem.BaseFromTradeGuangzhouDataId
-					dataItem.BaseFromTradeGuangzhouIndexId = indexId
+					dataItem.BaseFromTradeGuangzhouIndexId = subIndexId
 					dataItem.IndexCode = indexCode
 					dataItem.DataTime = monthEndDay
-					dataItem.Value = float64(v.WbillNum)
+					dataItem.Value = float64(total)
 					dataItem.ModifyTime = time.Now()
 					updateDataList = append(updateDataList, dataItem)
 				}
 			}
 		}
-		fmt.Println("addDataList:", len(addDataList))
+
 		if len(addDataList) > 0 {
 			err = gzObj.AddBaseFromTradeGuangzhouData(addDataList)
 			if err != nil {

utils/common.go

@@ -1108,3 +1108,22 @@ func ChineseToPinyin(input string) string {
 
 	return result.String()
 }
+
+func ChineseToPinyinInitials(input string) string {
+	a := pinyin.NewArgs()
+	a.Style = pinyin.FirstLetter // 设置风格为首字母
+	var result strings.Builder
+
+	for _, r := range input {
+		if unicode.Is(unicode.Han, r) {
+			py := pinyin.Pinyin(string(r), a)
+			if len(py) > 0 && len(py[0]) > 0 {
+				result.WriteString(py[0][0])
+			}
+		} else if unicode.IsNumber(r) || unicode.IsLetter(r) {
+			result.WriteRune(r)
+		}
+	}
+
+	return result.String()
+}