feat:添加上海有色爬虫

services/smm_shanghai.go

@@ -377,6 +377,9 @@ type EdbInfoData struct {
 	SmmData
 }
 
+// 如果历史数据入库，就不再执行OldExcel
+var isSave bool
+
 func SyncShangHaiSmm(cont context.Context) (err error) {
 	upMonth := time.Now().AddDate(0, -1, 0).Format(utils.FormatDate)
 	curDate := time.Now().Format(utils.FormatDate)
@@ -447,6 +450,9 @@ func SyncShangHaiSmm(cont context.Context) (err error) {
 		fmt.Println("上海有色爬虫更新失败")
 		return
 	}
+	if resp.Ret == 200 && utils.BusinessCode == utils.BusinessCodeJinRui {
+		OldExcel()
+	}
 	return
 }
 
@@ -506,7 +512,10 @@ func ExcelToSmm(f *excelize.File, s string) (edbInfoList []*EdbInfoData) {
 }
 
 func OldExcel() {
-	excelPath := ""
+	excelPath := utils.OLD_EXCEL_PATH_JR
+	if excelPath == "" || isSave {
+		return
+	}
 	f, err := excelize.OpenFile(excelPath)
 	if err != nil {
 		fmt.Print(err)
@@ -547,4 +556,5 @@ func OldExcel() {
 		}
 		time.Sleep(10 * time.Second)
 	}
+	isSave = true
 }

services/task.go

@@ -59,9 +59,11 @@ func Task() {
 		task.AddTask("卓创资讯", sci99)
 	}
 
-	crawlerSmm := task.NewTask("refreshShangHaiSmmData", "0 0 10-18 * * *", SyncShangHaiSmm) // 上海有色网爬虫
+	if utils.BusinessCode == utils.BusinessCodeJinRui {
+		crawlerSmm := task.NewTask("refreshShangHaiSmmData", "0 0 11-20 * * *", SyncShangHaiSmm) // 上海有色网爬虫
+		task.AddTask("上海有色网指标爬取", crawlerSmm)                                                    //每天10-18点, 每小时爬一次
+	}
 
-	task.AddTask("上海有色网指标爬取", crawlerSmm) //每天10-18点, 每小时爬一次
 	//task.AddTask("统计局数据爬取-月度A", refreshNationalMonthA) // 每月10号2:15执行
 	//task.AddTask("统计局数据爬取-月度B", refreshNationalMonthB) // 每月16号2:15执行
 	//task.AddTask("统计局数据爬取-季度", refreshNationalQuarter) // 每月15号1:25执行

utils/config.go

@@ -2,9 +2,10 @@ package utils
 
 import (
 	"fmt"
+	"strconv"
+
 	beeLogger "github.com/beego/bee/v2/logger"
 	"github.com/beego/beego/v2/server/web"
-	"strconv"
 )
 
 var (
@@ -38,6 +39,10 @@ var (
 	EDB_LIB_Md5_KEY     string
 )
 
+var (
+	OLD_EXCEL_PATH_JR string
+)
+
 func init() {
 	tmpRunMode, err := web.AppConfig.String("run_mode")
 	if err != nil {
@@ -80,6 +85,12 @@ func init() {
 		APP_EDB_LIB_NAME_EN = config["app_edb_lib_name_en"]
 		EDB_LIB_Md5_KEY = config["edb_lib_md5_key"]
 	}
+	if BusinessCode == BusinessCodeJinRui {
+		APP_EDB_LIB_NAME_EN = config["app_edb_lib_name_en"]
+		EDB_LIB_Md5_KEY = config["edb_lib_md5_key"]
+		EDB_LIB_URL = config["edb_lib_url"]
+		OLD_EXCEL_PATH_JR = config["old_excel_path_jr"]
+	}
 
 	if RunMode == "release" {
 

utils/constants.go

@@ -29,4 +29,5 @@ const (
 const (
 	BusinessCodeRelease = "E2023080900" // 生产环境
 	BusinessCodeFuBang  = "E2024020200"
+	BusinessCodeJinRui  = "E2024020205"
 )