fix:CCF网站解析

services/base_from_ccf/chart.go

@@ -47,6 +47,7 @@ type CCFChartAdditionRule struct {
 	ClassifyId int    `json:"ClassifyId"`
 	Frequency  string `json:"Frequency"`
 	ProdNames  string `json:"prodNames"`
+	LastNYear  int    `json:"LastNYear"`
 }
 
 func LoadCCFChartAdditionRule() (rules []*CCFChartAdditionRule, err error) {
@@ -71,6 +72,7 @@ func TaskGetCCFChartEdb(context.Context) (err error) {
 	_ = GetCCFChartEdb()
 	return
 }
+
 func GetCCFChartEdb() (err error) {
 	defer func() {
 		if err != nil {
@@ -150,12 +152,16 @@ func GetCCFChartEdb() (err error) {
 	now := time.Now()
 	for _, v := range additionRules {
 		param := make(map[string]string)
-		param["startdate"] = time.Date(now.Year()-4, 1, 1, 0, 0, 0, 0, time.Local).Format(utils.FormatDate2)
+		if v.LastNYear == 0 {
+			v.LastNYear = 5
+		}
+		param["startdate"] = time.Date(now.Year()+1-v.LastNYear, 1, 1, 0, 0, 0, 0, time.Local).Format(utils.FormatDate2)
 		param["enddate"] = now.Format(utils.FormatDate2)
 		param["type"] = "1"
 		param["prodNames"] = v.ProdNames
 		param["skin"] = "infographic"
 		param["page"] = "index.php"
+		fmt.Println(param)
 		htmlContent, er := postPageHtml(CCFCHARTDATAURL, param, 0)
 		if er != nil {
 			err = fmt.Errorf("获取首页报告失败, err: %v", er)

services/base_from_ccf/common.go

@@ -213,59 +213,73 @@ func postPageHtml(baseUrl string, formData map[string]string, fetchNum int) (res
 		}
 		return postPageHtml(baseUrl, formData, fetchNum)
 	}
+
 	var b bytes.Buffer
 	writer := multipart.NewWriter(&b)
-
 	for k, v := range formData {
 		_ = writer.WriteField(k, v)
 	}
 	writer.Close()
 
-	// 拉取网站内容
-	cli := new(http.Client)
-	req, e := http.NewRequest("Post", baseUrl, &b)
+	req, e := http.NewRequest("POST", baseUrl, &b)
 	if e != nil {
 		err = e
 		return
 	}
-
-	req.Header.Set("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7")
-	req.Header.Set("Accept-Encoding", "gzip, deflate, br")
-	req.Header.Set("Accept-Language", "zh-CN,zh;q=0.9,en;q=0.8,en-GB;q=0.7,en-US;q=0.6")
-	req.Header.Set("Connection", "keep-alive")
-	req.Header.Set("Cookie", strCookie)
-	req.Header.Set("Host", "www.ccf.com.cn")
-	req.Header.Set("Referer", baseUrl)
-	req.Header.Set("Sec-Ch-Ua", "\"Not A(Brand\";v=\"99\", \"Microsoft Edge\";v=\"121\", \"Chromium\";v=\"121\"")
-	req.Header.Set("Sec-Ch-Ua-Mobile", "?0")
-	req.Header.Set("Sec-Ch-Ua-Platform", "\"Windows\"")
-	req.Header.Set("Sec-Fetch-Dest", "empty")
-	req.Header.Set("Sec-Fetch-Mode", "cors")
-	req.Header.Set("Sec-Fetch-Site", "same-origin")
-	req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/121.0.0.0 Safari/537.36 Edg/121.0.0.0")
-	req.Header.Set("X-Requested-With", "XMLHttpRequest")
+	cli := new(http.Client)
+	// 设置请求头
+	req.Header.Add("Accept", " text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7")
+	req.Header.Add("Accept-Encoding", "gzip, deflate, br, zstd")
+	req.Header.Add("Accept-Language", "zh-CN,zh;q=0.9")
+	req.Header.Add("Cache-Control", "max-age=0")
+	req.Header.Add("Connection", "keep-alive")
+	req.Header.Add("Content-Type", writer.FormDataContentType())
+	req.Header.Add("Cookie", strCookie)
+	req.Header.Add("Host", "www.ccf.com.cn")
+	req.Header.Add("Origin", "https://www.ccf.com.cn")
+	req.Header.Add("Referer", baseUrl)
+	req.Header.Add("Sec-Ch-Ua", `"Google Chrome";v="131", "Chromium";v="131", "Not_A Brand";v="24"`)
+	req.Header.Add("Sec-Ch-Ua-Mobile", " ?0")
+	req.Header.Add("Sec-Ch-Ua-Platform", `"Windows"`)
+	req.Header.Add("Sec-Fetch-Dest", "document")
+	req.Header.Add("Sec-Fetch-Mode", "navigate")
+	req.Header.Add("Sec-Fetch-Site", "same-origin")
+	req.Header.Add("Sec-Fetch-User", "?1")
+	req.Header.Add("Upgrade-Insecure-Requests", "1")
+	req.Header.Add("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36")
 
 	resp, e := cli.Do(req)
 	if e != nil {
 		err = fmt.Errorf("HTTP client Do err: %s", e.Error())
+		fmt.Println("HTTP client Do err:", e.Error())
 		return
 	}
 	defer func() {
 		_ = resp.Body.Close()
 	}()
-
-	// 读取响应的内容
 	reader, e := gzip.NewReader(resp.Body)
 	if e != nil {
 		err = fmt.Errorf("gzip NewReader err: %s", e.Error())
 		return
 	}
+
 	body, e := io.ReadAll(reader)
 	if e != nil {
-		err = fmt.Errorf("read body err: %s", e.Error())
+		err = fmt.Errorf("读取body失败, err: %s", e.Error())
+		return
+	}
+
+	utf8Reader, e := charset.NewReaderLabel("gb2312", bytes.NewReader(body))
+	if e != nil {
+		err = fmt.Errorf("utf8 reader err: %s", e.Error())
 		return
 	}
-	respBody = body
+	utf8Body, e := io.ReadAll(utf8Reader)
+	if e != nil {
+		err = fmt.Errorf("utf8 body err: %s", e.Error())
+		return
+	}
+	respBody = utf8Body
 
 	isLoginPage := checkIsLoginPage(string(respBody))
 	fmt.Println("是否登录页：", isLoginPage)

static/ccf_chart_addition_rule.json

@@ -3,12 +3,14 @@
         "Name": "CCF/MEG港口库存指数",
         "ClassifyId": 21,
         "Frequency": "周度",
-        "prodNames": "kc-221000"
+        "prodNames": "kc-221000",
+        "LastNYear": 3
     },
     {
         "Name": "CCF/PTA FOB中国",
         "ClassifyId": 22,
         "Frequency": "日度",
-        "prodNames": "zs-235"
+        "prodNames": "zs-235",
+        "LastNYear": 3
     }
 ]