eta_data_analysis/services/base_from_oilchem/common.go

package services

import (
	"compress/gzip"
	"encoding/json"
	"eta/eta_data_analysis/utils"
	"fmt"
	"io"
	"net/http"
	"os"
	"strings"
	"time"
)

// savePageHtml 拉取历史报告详情
func savePageHtml(nameKey, saveDir string, historyPage bool, reportMax int) (files []string, err error) {
	if nameKey == "" {
		return
	}
	defer func() {
		if err != nil {
			tips := fmt.Sprintf("GetOilchemEdbHistory ErrMsg: %s", err.Error())
			utils.FileLog.Info(tips)
			fmt.Println(tips)
		}
	}()
	//fetchRule, e := loadDataRule(nameKey)
	//if e != nil {
	//	err = fmt.Errorf("loadDataRule, err: %v", e)
	//	return
	//}
	if saveDir == "" {
		saveDir = "static/ccf"
	}

	// 拉取报告留档
	strDate := time.Now().Format("20060102")
	url := "https://www.oilchem.net/24-0801-09-4036018c523e4bbc.html"
	htm, e := FetchPageHtml(url)
	if e != nil {
		utils.FileLog.Info("获取页面失败, err: %v", e)
	}
	dateDir := fmt.Sprintf("%s/%s", saveDir, strDate)
	if e = utils.MkDir(dateDir); e != nil {
		utils.FileLog.Info(fmt.Sprintf("创建目录失败, err: %v", e))
	}
	outputPath := fmt.Sprintf("%s/%d-%s.html", dateDir, "", "")
	if e = utils.WriteHTMLToFile(string(htm), outputPath); e != nil {
		utils.FileLog.Info(fmt.Sprintf("写入HTML出错, err: %v", e))
	}
	files = append(files, outputPath)
	fmt.Println("拉取报告 end")
	return
}

// fetchPageHtml 获取网站HTML文本
func FetchPageHtml(baseUrl string) (respBody []byte, err error) {
	defer func() {
		if err != nil {
			tips := fmt.Sprintf("BuildCCFRequest ErrMsg: %s", err.Error())
			utils.FileLog.Info(tips)
			fmt.Println(tips)
		}
	}()
	// 查询次数
	//fetchNum++
	if baseUrl == "" {
		err = fmt.Errorf("隆众请求地址为空")
		return
	}

	// 获取Cookie
	strCookie, e := getCookie()
	if e != nil {
		err = fmt.Errorf("读取cookie文件失败, err: %s", e.Error())
		return
	}

	// 拉取网站内容
	cli := new(http.Client)
	req, e := http.NewRequest("GET", baseUrl, nil)
	if e != nil {
		err = fmt.Errorf("")
		return
	}

	req.Header.Set("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7")
	req.Header.Set("Accept-Encoding", "gzip, deflate, br")
	req.Header.Set("Accept-Language", "zh-CN,zh;q=0.9,en;q=0.8,en-GB;q=0.7,en-US;q=0.6")
	req.Header.Set("Connection", "keep-alive")
	req.Header.Set("Cookie", strCookie)
	req.Header.Set("Host", "www.oilchem.net")
	req.Header.Set("Referer", "https://chem.oilchem.net/")
	req.Header.Set("Sec-Ch-Ua", "\"Not A(Brand\";v=\"99\", \"Microsoft Edge\";v=\"121\", \"Chromium\";v=\"121\"")
	req.Header.Set("Sec-Ch-Ua-Mobile", "?0")
	req.Header.Set("Sec-Ch-Ua-Platform", "\"Windows\"")
	req.Header.Set("Sec-Fetch-Dest", "empty")
	req.Header.Set("Sec-Fetch-Mode", "cors")
	req.Header.Set("Sec-Fetch-Site", "same-origin")
	req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/121.0.0.0 Safari/537.36 Edg/121.0.0.0")
	req.Header.Set("X-Requested-With", "XMLHttpRequest")

	resp, e := cli.Do(req)
	if e != nil {
		err = fmt.Errorf("HTTP client Do err: %s", e.Error())
		return
	}
	defer func() {
		_ = resp.Body.Close()
	}()

	// 读取响应的内容
	reader, e := gzip.NewReader(resp.Body)
	if e != nil {
		err = fmt.Errorf("gzip NewReader err: %s", e.Error())
		return
	}

	respBody, e = io.ReadAll(reader)
	if e != nil {
		err = fmt.Errorf("read body err: %s", e.Error())
		return
	}

	return
}


// getCookie
// @Description: 获取cookie
// @author: Roc
// @datetime 2024-07-09 14:00:53
// @return cookieStr string
// @return err error
func getCookie() (cookieStr string, err error) {
	// 读取Cookie
	if utils.OilchemCookieFile == "" {
		err = fmt.Errorf("cookie文件未配置")
		return
	}
	cookieByte, e := os.ReadFile(utils.OilchemCookieFile)
	if e != nil {
		err = fmt.Errorf("读取cookie文件失败, err: %s", e.Error())
		return
	}
	cookieStr = strings.TrimSpace(string(cookieByte))
	if cookieStr == "" {
		err = fmt.Errorf("cookie为空")
		return
	}

	return
}



// postEdbLib 调用指标接口
func postEdbLib(param map[string]interface{}, method string) (result []byte, err error) {
	postUrl := utils.EDB_LIB_URL + method
	postData, err := json.Marshal(param)
	if err != nil {
		return
	}
	result, err = httpPost(postUrl, string(postData), "application/json")
	if err != nil {
		return
	}
	return
}

// httpPost HTTP请求
func httpPost(url, postData string, params ...string) ([]byte, error) {
	fmt.Println("httpPost Url:" + url)
	body := io.NopCloser(strings.NewReader(postData))
	client := &http.Client{}
	req, err := http.NewRequest("POST", url, body)
	if err != nil {
		return nil, err
	}
	contentType := "application/x-www-form-urlencoded;charset=utf-8"
	if len(params) > 0 && params[0] != "" {
		contentType = params[0]
	}
	req.Header.Set("Content-Type", contentType)
	req.Header.Set("authorization", utils.MD5(utils.APP_EDB_LIB_NAME_EN+utils.EDB_LIB_Md5_KEY))
	resp, err := client.Do(req)
	if err != nil {
		fmt.Println("client.Do err:" + err.Error())
		return nil, err
	}
	defer func() {
		_ = resp.Body.Close()
	}()
	b, err := io.ReadAll(resp.Body)
	if err != nil {
		fmt.Println("httpPost:" + string(b))
	}
	return b, err
}