eta_crawler/services/icpi.go

package services

import (
	"context"
	"encoding/json"
	"eta/eta_crawler/models"
	"eta/eta_crawler/utils"
	"fmt"
	"io/ioutil"
	"net/http"
	"strings"
	"time"
)

func CrawlerIcpi(cont context.Context) (err error) {
	dataIndexUrl := `http://www.bdecon.com/dataIndex`
	resp, err := http.Get(dataIndexUrl)
	fmt.Println(err)
	cookieLen := len(resp.Cookies())
	var cookieStr string
	for i := cookieLen - 1; i >= 0; i-- {
		cookie := resp.Cookies()[i]
		fmt.Println(i, cookie.Name, cookie.Value)
		cookieStr += cookie.Name + "=" + cookie.Value + ";"
	}

	cookieStr = strings.Trim(cookieStr, ";")
	fmt.Println(cookieStr)
	//获取分类
	CrawlerIcpiClassify(cookieStr)
	//大类
	CrawlerIcpiMaxClassifyIndex(cookieStr)
	//中类
	CrawlerIcpiMiddleClassifyIndex(cookieStr)
	return err
}

func CrawlerIcpiClassify(cookieStr string) (err error) {
	// 创建一个HTTP客户端
	client := &http.Client{}
	// 创建一个GET请求
	//data := strings.NewReader("laber=1")
	req, err := http.NewRequest("POST", "http://www.bdecon.com/monthly/LargeWeightListNew1/", nil)
	if err != nil {
		fmt.Println("创建请求失败:", err)
		return
	}
	// 设置请求头
	req.Header.Set("Accept", "*/*")
	req.Header.Set("Accept-Encoding", "gzip, deflate")
	req.Header.Set("Accept-Language", "zh-CN,zh;q=0.9,en;q=0.8")
	req.Header.Set("Connection", "keep-alive")
	req.Header.Set("Content-Length", "7")
	req.Header.Set("Content-Type", "application/x-www-form-urlencoded; charset=UTF-8")
	req.Header.Set("Cookie", cookieStr)
	req.Header.Set("Host", "www.bdecon.com")
	req.Header.Set("Origin", "http://www.bdecon.com")
	req.Header.Set("Referer", "http://www.bdecon.com/dataIndex")
	req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36")
	req.Header.Set("X-Requested-With", "XMLHttpRequest")
	// 发送请求并获取响应
	resp, err := client.Do(req)
	if err != nil {
		fmt.Println("发送请求失败:", err)
		return
	}
	defer resp.Body.Close()
	// 读取响应的内容
	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		fmt.Println("读取响应失败:", err)
		return
	}

	utils.FileLog.Info("result:")
	utils.FileLog.Info(string(body))
	// 打印响应内容
	fmt.Println(string(body))

	classifyList := make([]models.IcpiClassify, 0)
	err = json.Unmarshal(body, &classifyList)
	if err != nil {
		fmt.Println("CrawlerIcpi json.Unmarshal Err:" + err.Error())
		return
	}

	icpiObj := new(models.BaseFromIcpiIndex)

	classifyAll, err := icpiObj.GetBaseFromIcpiClassifyAll()
	if err != nil {
		fmt.Println("GetBaseFromIcpiClassifyAll Err:" + err.Error())
		return
	}

	classifyMap := make(map[string]*models.BaseFromIcpiClassify)
	for _, v := range classifyAll {
		classifyMap[v.ClassifyName] = v
	}
	for _, v := range classifyList {
		if v.Category == "总类" {
			v.Category = "日总类"
		}
		if _, ok := classifyMap[v.Category]; !ok {
			classifyItem := new(models.BaseFromIcpiClassify)
			classifyItem.ClassifyName = v.Category
			classifyItem.ClassifyNameEn = v.Categoryenglish
			classifyItem.ParentId = 0
			classifyItem.CreateTime = time.Now()
			classifyItem.ModifyTime = time.Now()
			err = icpiObj.AddClassify(classifyItem)
			if err != nil {
				return err
			}
		}
	}
	return err
}

// 爬取ICPI指数大类及指标数据
func CrawlerIcpiMaxClassifyIndex(cookieStr string) (err error) {
	// 创建一个HTTP客户端
	client := &http.Client{}
	// 创建一个GET请求
	icpiObj := new(models.BaseFromIcpiIndex)

	classifyAll, err := icpiObj.GetBaseFromIcpiClassifyAll()
	if err != nil {
		fmt.Println("GetBaseFromIcpiClassifyAll Err:" + err.Error())
		return
	}

	classifyMap := make(map[string]*models.BaseFromIcpiClassify)
	for _, v := range classifyAll {
		classifyMap[v.ClassifyName] = v
	}

	param := strings.NewReader("laber=1")
	req, err := http.NewRequest("POST", "http://www.bdecon.com/indexqueryiCPI/", param)
	if err != nil {
		fmt.Println("创建请求失败:", err)
		return
	}
	// 设置请求头
	req.Header.Set("Accept", "*/*")
	req.Header.Set("Accept-Encoding", "gzip, deflate")
	req.Header.Set("Accept-Language", "zh-CN,zh;q=0.9,en;q=0.8")
	req.Header.Set("Connection", "keep-alive")
	req.Header.Set("Content-Length", "7")
	req.Header.Set("Content-Type", "application/x-www-form-urlencoded; charset=UTF-8")
	req.Header.Set("Cookie", cookieStr)
	req.Header.Set("Host", "www.bdecon.com")
	req.Header.Set("Origin", "http://www.bdecon.com")
	req.Header.Set("Referer", "http://www.bdecon.com/dataIndex")
	req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36")
	req.Header.Set("X-Requested-With", "XMLHttpRequest")
	// 发送请求并获取响应
	resp, err := client.Do(req)
	if err != nil {
		fmt.Println("发送请求失败:", err)
		return
	}
	defer resp.Body.Close()
	// 读取响应的内容
	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		fmt.Println("读取响应失败:", err)
		return
	}

	utils.FileLog.Info("result:")
	utils.FileLog.Info(string(body))
	// 打印响应内容
	fmt.Println(string(body))

	indexList := make([]IcpiItem, 0)
	err = json.Unmarshal(body, &indexList)
	if err != nil {
		fmt.Println("CrawlerIcpiMaxClassifyIndex json.Unmarshal Err:" + err.Error())
		return
	}

	icpiIndexObj := new(models.BaseFromIcpiIndex)

	indexAll, err := icpiIndexObj.GetBaseFromIcpiIndexAll()
	if err != nil {
		fmt.Println("GetBaseFromIcpiIndexAll Err:" + err.Error())
		return
	}

	indexMap := make(map[string]*models.BaseFromIcpiIndex)
	for _, v := range indexAll {
		indexMap[v.IndexCode] = v
	}

	limit := 2
	for _, v := range indexList {
		indexCode = strings.ReplaceAll(v.EnglishName, " ", "_")

		if indexItem, ok := indexMap[indexCode]; !ok { //新增指标
			icpiIndexObj = new(models.BaseFromIcpiIndex)
			icpiIndexObj.IndexCode = indexCode
			if classify, ok := classifyMap[v.Name]; ok {
				icpiIndexObj.BaseFromIcpiClassifyId = classify.BaseFromIcpiClassifyId
			}

			icpiIndexObj.IndexName = v.Name
			icpiIndexObj.Frequency = "日度"
			icpiIndexObj.Unit = "%"
			icpiIndexObj.CreateTime = time.Now()
			icpiIndexObj.ModifyTime = time.Now()
			err = icpiIndexObj.Add()
			if err != nil {
				fmt.Println("icpiIndexObj Add Err:" + err.Error())
				return err
			}

			//新增指标对应的数据
			dataItem := new(models.BaseFromIcpiData)
			dataItem.IndexCode = indexCode
			dataItem.BaseFromIcpiIndexId = icpiIndexObj.BaseFromIcpiIndexId
			dataItem.DataTime = time.Now().Format(utils.FormatDate)
			dataItem.Value = v.Today
			dataItem.CreateTime = time.Now()
			dataItem.ModifyTime = time.Now()
			err = icpiIndexObj.AddData(dataItem)
			if err != nil {
				fmt.Println("icpiIndexObj AddData Err:" + err.Error())
				return err
			}

			//新增指标对应的数据
			dataItem = new(models.BaseFromIcpiData)
			dataItem.IndexCode = indexCode
			dataItem.BaseFromIcpiIndexId = icpiIndexObj.BaseFromIcpiIndexId
			dataItem.DataTime = time.Now().AddDate(0, 0, -1).Format(utils.FormatDate)
			dataItem.Value = v.Yesterday
			dataItem.CreateTime = time.Now()
			dataItem.ModifyTime = time.Now()
			err = icpiIndexObj.AddData(dataItem)
			if err != nil {
				fmt.Println("icpiIndexObj AddData Err:" + err.Error())
				return err
			}
		} else {
			dataList, err := icpiIndexObj.GetBaseFromIcpiData(indexCode, limit)
			if err != nil {
				fmt.Println("GetBaseFromIcpiIndexAll Err:" + err.Error())
				return err
			}

			dataMap := make(map[string]string)
			for _, dv := range dataList {
				dataMap[dv.DataTime] = dv.Value
			}

			today := time.Now().Format(utils.FormatDate)
			if dataVal, ok := dataMap[today]; !ok {
				dataItem := new(models.BaseFromIcpiData)
				dataItem.IndexCode = indexCode
				dataItem.BaseFromIcpiIndexId = indexItem.BaseFromIcpiIndexId
				dataItem.DataTime = today
				dataItem.Value = v.Today
				dataItem.CreateTime = time.Now()
				dataItem.ModifyTime = time.Now()
				err = icpiIndexObj.AddData(dataItem)
				if err != nil {
					fmt.Println("icpiIndexObj AddData Err:" + err.Error())
					return err
				}
			} else {
				if dataVal != v.Today {
					err = icpiIndexObj.ModifyData(indexCode, today, v.Today)
					if err != nil {
						fmt.Println("icpiIndexObj ModifyData Err:" + err.Error())
						return err
					}
				}
			}

			yesterday := time.Now().AddDate(0, 0, -1).Format(utils.FormatDate)
			if dataVal, ok := dataMap[yesterday]; !ok {
				dataItem := new(models.BaseFromIcpiData)
				dataItem.IndexCode = indexCode
				dataItem.BaseFromIcpiIndexId = indexItem.BaseFromIcpiIndexId
				dataItem.DataTime = yesterday
				dataItem.Value = v.Yesterday
				dataItem.CreateTime = time.Now()
				dataItem.ModifyTime = time.Now()
				err = icpiIndexObj.AddData(dataItem)
				if err != nil {
					fmt.Println("icpiIndexObj AddData Err:" + err.Error())
					return err
				}
			} else {
				if dataVal != v.Yesterday {
					err = icpiIndexObj.ModifyData(indexCode, yesterday, v.Yesterday)
					if err != nil {
						fmt.Println("icpiIndexObj ModifyData Err:" + err.Error())
						return err
					}
				}
			}
		}

		maxItem, err := icpiIndexObj.GetBaseFromIcpiIndexMaxAndMinInfo(indexCode)
		if err != nil {
			fmt.Println("GetBaseFromIcpiIndexMaxAndMinInfo Err:" + err.Error())
			return err
		}

		err = icpiIndexObj.ModifyEdbInfoMaxAndMinInfo(indexCode, maxItem)
		if err != nil {
			fmt.Println("ModifyEdbInfoMaxAndMinInfo Err:" + err.Error())
			return err
		}

	}

	return err
}

// 爬取ICPI中类指标信息
func CrawlerIcpiMiddleClassifyIndex(cookieStr string) (err error) {
	// 创建一个HTTP客户端
	client := &http.Client{}
	// 创建一个GET请求

	icpiObj := new(models.BaseFromIcpiIndex)

	classifyAll, err := icpiObj.GetBaseFromIcpiClassifyAll()
	if err != nil {
		fmt.Println("GetBaseFromIcpiClassifyAll Err:" + err.Error())
		return
	}

	classifyMap := make(map[string]*models.BaseFromIcpiClassify)
	for _, v := range classifyAll {
		classifyMap[v.ClassifyName] = v
	}

	params := strings.NewReader("name=yueda=总类,食品烟酒,衣着,居住,生活用品及服务,交通和通信,教育文化和娱乐,医疗保健,其他用品和服务&yuezh=衣着——服装,衣着——服装材料,衣着——其他衣着及配件,衣着——衣着加工服务费,衣着——鞋类,交通和通信——交通,交通和通信——通信,生活用品及服务——家具及室内装饰品,生活用品及服务——家用器具,生活用品及服务——家用纺织品,生活用品及服务——家庭日用杂品,生活用品及服务——个人护理用品,生活用品及服务——家庭服务,居住——租赁房房租,居住——住房保养维修及管理,居住——水电燃料,居住——自有住房,医疗保健——药品及医疗器具,医疗保健——医疗服务,食品烟酒——食品,食品烟酒——茶及饮料,食品烟酒——烟酒,食品烟酒——在外餐饮,教育文化和娱乐——教育,教育文化和娱乐——文化娱乐,其他用品和服务——其他用品类,其他用品和服务——其他服务类&yuexi=&yuezi=&zhishu=日总类")
	req, err := http.NewRequest("POST", "http://www.bdecon.com/indexqueryNew/", params)
	if err != nil {
		fmt.Println("创建请求失败:", err)
		return
	}
	// 设置请求头
	req.Header.Set("Accept", "*/*")
	req.Header.Set("Accept-Encoding", "gzip, deflate")
	req.Header.Set("Accept-Language", "zh-CN,zh;q=0.9,en;q=0.8")
	req.Header.Set("Connection", "keep-alive")
	req.Header.Set("Content-Length", "7")
	req.Header.Set("Content-Type", "application/x-www-form-urlencoded; charset=UTF-8")
	req.Header.Set("Cookie", cookieStr)
	req.Header.Set("Host", "www.bdecon.com")
	req.Header.Set("Origin", "http://www.bdecon.com")
	req.Header.Set("Referer", "http://www.bdecon.com/dataIndex")
	req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36")
	req.Header.Set("X-Requested-With", "XMLHttpRequest")
	// 发送请求并获取响应
	resp, err := client.Do(req)
	if err != nil {
		fmt.Println("发送请求失败:", err)
		return
	}
	defer resp.Body.Close()
	// 读取响应的内容
	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		fmt.Println("读取响应失败:", err)
		return
	}

	utils.FileLog.Info("result:")
	utils.FileLog.Info(string(body))
	// 打印响应内容
	fmt.Println(string(body))

	indexList := make([]*IcpiItem, 0)
	err = json.Unmarshal(body, &indexList)
	if err != nil {
		fmt.Println("CrawlerIcpiMiddleClassifyIndex json.Unmarshal Err:" + err.Error())
		return
	}

	icpiIndexObj := new(models.BaseFromIcpiIndex)

	indexAll, err := icpiIndexObj.GetBaseFromIcpiIndexAll()
	if err != nil {
		fmt.Println("GetBaseFromIcpiIndexAll Err:" + err.Error())
		return
	}

	indexMap := make(map[string]*models.BaseFromIcpiIndex)
	for _, v := range indexAll {
		indexMap[v.IndexCode] = v
	}

	limit := 2
	for _, v := range indexList {
		var indexCode string
		if v.EnglishName == "" {
			indexCode = utils.ChineseToPinyinInitials(v.Name)
		} else {
			indexCode = strings.ReplaceAll(v.EnglishName, " ", "_")
		}

		if indexCode == "" {
			continue
		}

		dataList, err := icpiIndexObj.GetBaseFromIcpiData(indexCode, limit)
		if err != nil {
			fmt.Println("GetBaseFromIcpiIndexAll Err:" + err.Error())
			return err
		}

		dataMap := make(map[string]string)
		for _, dv := range dataList {
			dataMap[dv.DataTime] = dv.Value
		}

		if indexItem, ok := indexMap[indexCode]; !ok { //新增指标

			classifyNameArr := strings.Split(v.Name, "-")
			classifyName := classifyNameArr[0]

			icpiIndexObj = new(models.BaseFromIcpiIndex)
			icpiIndexObj.IndexCode = indexCode
			icpiIndexObj.IndexName = v.Name
			if classifyItem, ok := classifyMap[classifyName]; ok {
				icpiIndexObj.BaseFromIcpiClassifyId = classifyItem.BaseFromIcpiClassifyId
			}
			icpiIndexObj.Unit = "%"
			icpiIndexObj.Frequency = "日度"
			icpiIndexObj.CreateTime = time.Now()
			icpiIndexObj.ModifyTime = time.Now()
			err = icpiIndexObj.Add()
			if err != nil {
				fmt.Println("icpiIndexObj Add Err:" + err.Error())
				return err
			}

			//新增指标对应的数据
			dataItem := new(models.BaseFromIcpiData)
			dataItem.IndexCode = indexCode
			dataItem.BaseFromIcpiIndexId = icpiIndexObj.BaseFromIcpiIndexId
			dataItem.DataTime = time.Now().Format(utils.FormatDate)
			dataItem.Value = v.Today
			dataItem.CreateTime = time.Now()
			dataItem.ModifyTime = time.Now()
			err = icpiIndexObj.AddData(dataItem)
			if err != nil {
				fmt.Println("icpiIndexObj AddData Err:" + err.Error())
				return err
			}

			//新增指标对应的数据
			dataItem = new(models.BaseFromIcpiData)
			dataItem.IndexCode = indexCode
			dataItem.BaseFromIcpiIndexId = icpiIndexObj.BaseFromIcpiIndexId
			dataItem.DataTime = time.Now().AddDate(0, 0, -1).Format(utils.FormatDate)
			dataItem.Value = v.Yesterday
			dataItem.CreateTime = time.Now()
			dataItem.ModifyTime = time.Now()
			err = icpiIndexObj.AddData(dataItem)
			if err != nil {
				fmt.Println("icpiIndexObj AddData Err:" + err.Error())
				return err
			}
		} else {
			today := time.Now().Format(utils.FormatDate)
			if dataVal, ok := dataMap[today]; !ok {
				dataItem := new(models.BaseFromIcpiData)
				dataItem.IndexCode = indexCode
				dataItem.BaseFromIcpiIndexId = indexItem.BaseFromIcpiIndexId
				dataItem.DataTime = today
				dataItem.Value = v.Today
				dataItem.CreateTime = time.Now()
				dataItem.ModifyTime = time.Now()
				err = icpiIndexObj.AddData(dataItem)
				if err != nil {
					fmt.Println("icpiIndexObj AddData Err:" + err.Error())
					return err
				}
			} else {
				if dataVal != v.Today {
					err = icpiIndexObj.ModifyData(indexCode, today, v.Today)
					if err != nil {
						fmt.Println("icpiIndexObj ModifyData Err:" + err.Error())
						return err
					}
				}
			}

			yesterday := time.Now().AddDate(0, 0, -1).Format(utils.FormatDate)
			if dataVal, ok := dataMap[yesterday]; !ok {
				dataItem := new(models.BaseFromIcpiData)
				dataItem.IndexCode = indexCode
				dataItem.BaseFromIcpiIndexId = indexItem.BaseFromIcpiIndexId
				dataItem.DataTime = yesterday
				dataItem.Value = v.Yesterday
				dataItem.CreateTime = time.Now()
				dataItem.ModifyTime = time.Now()
				err = icpiIndexObj.AddData(dataItem)
				if err != nil {
					fmt.Println("icpiIndexObj AddData Err:" + err.Error())
					return err
				}
			} else {
				if dataVal != v.Yesterday {
					err = icpiIndexObj.ModifyData(indexCode, yesterday, v.Yesterday)
					if err != nil {
						fmt.Println("icpiIndexObj ModifyData Err:" + err.Error())
						return err
					}
				}
			}
		}

		maxItem, err := icpiIndexObj.GetBaseFromIcpiIndexMaxAndMinInfo(indexCode)
		if err != nil {
			fmt.Println("GetBaseFromIcpiIndexMaxAndMinInfo Err:" + err.Error())
			return err
		}

		err = icpiIndexObj.ModifyEdbInfoMaxAndMinInfo(indexCode, maxItem)
		if err != nil {
			fmt.Println("ModifyEdbInfoMaxAndMinInfo Err:" + err.Error())
			return err
		}

	}

	return err
}

type IcpiItem struct {
	Name        string `json:"name"`
	EnglishName string `json:"englishName"`
	Today       string `json:"today"`
	Yesterday   string `json:"yesterday"`
}