package services import ( "compress/gzip" "encoding/json" "eta/eta_data_analysis/utils" "fmt" "io" "net/http" "os" "strings" "time" ) // savePageHtml 拉取历史报告详情 func savePageHtml(nameKey, saveDir string, historyPage bool, reportMax int) (files []string, err error) { if nameKey == "" { return } defer func() { if err != nil { tips := fmt.Sprintf("GetOilchemEdbHistory ErrMsg: %s", err.Error()) utils.FileLog.Info(tips) fmt.Println(tips) } }() //fetchRule, e := loadDataRule(nameKey) //if e != nil { // err = fmt.Errorf("loadDataRule, err: %v", e) // return //} if saveDir == "" { saveDir = "static/ccf" } // 拉取报告留档 strDate := time.Now().Format("20060102") url := "https://www.oilchem.net/24-0801-09-4036018c523e4bbc.html" htm, e := FetchPageHtml(url) if e != nil { utils.FileLog.Info("获取页面失败, err: %v", e) } dateDir := fmt.Sprintf("%s/%s", saveDir, strDate) if e = utils.MkDir(dateDir); e != nil { utils.FileLog.Info(fmt.Sprintf("创建目录失败, err: %v", e)) } outputPath := fmt.Sprintf("%s/%d-%s.html", dateDir, "", "") if e = utils.WriteHTMLToFile(string(htm), outputPath); e != nil { utils.FileLog.Info(fmt.Sprintf("写入HTML出错, err: %v", e)) } files = append(files, outputPath) fmt.Println("拉取报告 end") return } // fetchPageHtml 获取网站HTML文本 func FetchPageHtml(baseUrl string) (respBody []byte, err error) { defer func() { if err != nil { tips := fmt.Sprintf("BuildCCFRequest ErrMsg: %s", err.Error()) utils.FileLog.Info(tips) fmt.Println(tips) } }() // 查询次数 //fetchNum++ if baseUrl == "" { err = fmt.Errorf("隆众请求地址为空") return } // 获取Cookie strCookie, e := getCookie() if e != nil { err = fmt.Errorf("读取cookie文件失败, err: %s", e.Error()) return } // 拉取网站内容 cli := new(http.Client) req, e := http.NewRequest("GET", baseUrl, nil) if e != nil { err = fmt.Errorf("") return } req.Header.Set("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7") req.Header.Set("Accept-Encoding", "gzip, deflate, br") req.Header.Set("Accept-Language", "zh-CN,zh;q=0.9,en;q=0.8,en-GB;q=0.7,en-US;q=0.6") req.Header.Set("Connection", "keep-alive") req.Header.Set("Cookie", strCookie) req.Header.Set("Host", "www.oilchem.net") req.Header.Set("Referer", "https://chem.oilchem.net/") req.Header.Set("Sec-Ch-Ua", "\"Not A(Brand\";v=\"99\", \"Microsoft Edge\";v=\"121\", \"Chromium\";v=\"121\"") req.Header.Set("Sec-Ch-Ua-Mobile", "?0") req.Header.Set("Sec-Ch-Ua-Platform", "\"Windows\"") req.Header.Set("Sec-Fetch-Dest", "empty") req.Header.Set("Sec-Fetch-Mode", "cors") req.Header.Set("Sec-Fetch-Site", "same-origin") req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/121.0.0.0 Safari/537.36 Edg/121.0.0.0") req.Header.Set("X-Requested-With", "XMLHttpRequest") resp, e := cli.Do(req) if e != nil { err = fmt.Errorf("HTTP client Do err: %s", e.Error()) return } defer func() { _ = resp.Body.Close() }() // 读取响应的内容 reader, e := gzip.NewReader(resp.Body) if e != nil { err = fmt.Errorf("gzip NewReader err: %s", e.Error()) return } respBody, e = io.ReadAll(reader) if e != nil { err = fmt.Errorf("read body err: %s", e.Error()) return } return } // getCookie // @Description: 获取cookie // @author: Roc // @datetime 2024-07-09 14:00:53 // @return cookieStr string // @return err error func getCookie() (cookieStr string, err error) { // 读取Cookie if utils.OilchemCookieFile == "" { err = fmt.Errorf("cookie文件未配置") return } cookieByte, e := os.ReadFile(utils.OilchemCookieFile) if e != nil { err = fmt.Errorf("读取cookie文件失败, err: %s", e.Error()) return } cookieStr = strings.TrimSpace(string(cookieByte)) if cookieStr == "" { err = fmt.Errorf("cookie为空") return } return } // postEdbLib 调用指标接口 func postEdbLib(param map[string]interface{}, method string) (result []byte, err error) { postUrl := utils.EDB_LIB_URL + method postData, err := json.Marshal(param) if err != nil { return } result, err = httpPost(postUrl, string(postData), "application/json") if err != nil { return } return } // httpPost HTTP请求 func httpPost(url, postData string, params ...string) ([]byte, error) { fmt.Println("httpPost Url:" + url) body := io.NopCloser(strings.NewReader(postData)) client := &http.Client{} req, err := http.NewRequest("POST", url, body) if err != nil { return nil, err } contentType := "application/x-www-form-urlencoded;charset=utf-8" if len(params) > 0 && params[0] != "" { contentType = params[0] } req.Header.Set("Content-Type", contentType) req.Header.Set("authorization", utils.MD5(utils.APP_EDB_LIB_NAME_EN+utils.EDB_LIB_Md5_KEY)) resp, err := client.Do(req) if err != nil { fmt.Println("client.Do err:" + err.Error()) return nil, err } defer func() { _ = resp.Body.Close() }() b, err := io.ReadAll(resp.Body) if err != nil { fmt.Println("httpPost:" + string(b)) } return b, err }