|
|
@@ -96,6 +96,15 @@ func fetchPageHtml(baseUrl string, fetchNum int) (respBody []byte, err error) {
|
|
|
err = fmt.Errorf("读取cookie文件失败, err: %s", e.Error())
|
|
|
return
|
|
|
}
|
|
|
+ if strCookie == "" && fetchNum < 2 {
|
|
|
+ fmt.Printf("文件cookie为空, 重新获取, fetchNum: %d\n", fetchNum)
|
|
|
+ utils.FileLog.Info(fmt.Sprintf("文件cookie为空, 重新获取, fetchNum: %d", fetchNum))
|
|
|
+ _, err = getCookieByChrome()
|
|
|
+ if err != nil {
|
|
|
+ return
|
|
|
+ }
|
|
|
+ return fetchPageHtml(baseUrl, fetchNum)
|
|
|
+ }
|
|
|
|
|
|
// 拉取网站内容
|
|
|
cli := new(http.Client)
|
|
|
@@ -419,7 +428,6 @@ func analysisReportHrefs(contents []byte, page int) (hrefs []ReportHrefs, err er
|
|
|
return
|
|
|
}
|
|
|
|
|
|
-
|
|
|
// extractReportPublishTime 提取报告发布时间
|
|
|
func extractReportPublishTime(text string) (time.Time, error) {
|
|
|
datePattern := `(\d{4}年\d{1,2}月\d{1,2}日\d{1,2}:\d{2})`
|
|
|
@@ -741,10 +749,10 @@ func getCookie() (cookieStr string, err error) {
|
|
|
return
|
|
|
}
|
|
|
cookieStr = strings.TrimSpace(string(cookieByte))
|
|
|
- if cookieStr == "" {
|
|
|
- err = fmt.Errorf("cookie为空")
|
|
|
- return
|
|
|
- }
|
|
|
+ //if cookieStr == "" {
|
|
|
+ // err = fmt.Errorf("cookie为空")
|
|
|
+ // return
|
|
|
+ //}
|
|
|
|
|
|
return
|
|
|
}
|
|
|
@@ -802,8 +810,8 @@ func getCookieByChrome() (cookieStr string, err error) {
|
|
|
for _, v := range cookies {
|
|
|
cookieStr = cookieStr + v.Name + "=" + v.Value + ";"
|
|
|
}
|
|
|
- //fmt.Println("header cookie:", cookieStr)
|
|
|
- //utils.FileLog.Info("header cookie:" + cookieStr)
|
|
|
+ fmt.Println("header cookie:", cookieStr)
|
|
|
+ utils.FileLog.Info("header cookie:" + cookieStr)
|
|
|
|
|
|
tmpFile, tmpErr := os.Create(utils.CCFCookieFile)
|
|
|
if tmpErr != nil {
|
