fix

models/base_from_sci99.go

@@ -63,3 +63,40 @@ func GetBaseFromSci99Index() (list []*BaseFromSci99Index, err error) {
 	_, err = o.Raw(sql).QueryRows(&list)
 	return
 }
+
+
+func GetBaseFromSci99Classify() (list []*BaseFromSci99Classify, err error) {
+	o := orm.NewOrmUsingDB("data")
+	sql := `SELECT * FROM base_from_sci99_classify `
+	_, err = o.Raw(sql).QueryRows(&list)
+	return
+}
+
+func GetBaseFromTradeSci99DataAll(dateStr string) (list []*BaseFromSci99Data, err error) {
+	o := orm.NewOrmUsingDB("data")
+	sql := `SELECT * FROM base_from_sci99_data where data_time=?`
+	_, err = o.Raw(sql, dateStr).QueryRows(&list)
+	return
+}
+
+func GetBaseFromTradeSci99IndexAll(indexCode string) (list []*BaseFromSci99Data, err error) {
+	o := orm.NewOrmUsingDB("data")
+	sql := `SELECT * FROM base_from_sci99_data where index_code=?`
+	_, err = o.Raw(sql, indexCode).QueryRows(&list)
+	return
+}
+
+func GetBaseFromTradeSci99LatestData(indexCode string) (date time.Time, err error) {
+	o := orm.NewOrmUsingDB("data")
+	sql := `SELECT data_time FROM base_from_sci99_data where index_code=? ORDER BY data_time DESC limit 1 `
+	err = o.Raw(sql, indexCode).QueryRow(&date)
+	return
+}
+
+// UpdateBaseFromSci99Data
+func UpdateBaseFromSci99Data(value float64, indexCode, dataTime string) (err error) {
+	o := orm.NewOrmUsingDB("data")
+	sql := `UPDATE base_from_sci99_data SET value=?,modify_time=NOW() WHERE index_code = ? AND data_time = ? `
+	_, err = o.Raw(sql, value, indexCode, dataTime).Exec()
+	return
+}

services/sci99/list_struct.go

@@ -1,24 +1,6 @@
 package sci99
 
-const Cookie = "route=258ceb4bb660681c2cb2768af9756936; " +
-	"ASP.NET_SessionId=nzh5zm5aluxkryc11cg3s03x; " +
-	"guid=5975a35c-af63-8629-fad0-61b5278f8e97; " +
-	"Hm_lvt_78a951b1e2ee23efdc6af2ce70d6b9be=1715844009; " +
-	"href=https%3A%2F%2Fprices.sci99.com%2Fcn%2Fproduct.aspx%3Fppid%3D12947%26ppname%3D%25u5929%25u7136%25u4E73%25u80F6%26navid%3D595; " +
-	"sensorsdata2015jssdkcross=%7B%22distinct_id%22%3A%2218f8045cbaeeb4-0bc04c69e962898-1b525637-3686400-18f8045cbaf181a%22%2C%22first_id%22%3A%22%22%2C%22props%22%3A%7B%22%24" +
-	"latest_traffic_source_type%22%3A%22%E7%9B%B4%E6%8E%A5%E6%B5%81%E9%87%8F%22%2C%22%24" +
-	"latest_search_keyword%22%3A%22%E6%9C%AA%E5%8F%96%E5%88%B0%E5%80%BC_%E7%9B%B4%E6%8E%A5%E6%89%93%E5%BC%80%22%2C%22%24" +
-	"latest_referrer%22%3A%22%22%7D%2C%22%24device_id%22%3A%2218f8045cbaeeb4-0bc04c69e962898-1b525637-3686400-18f8045cbaf181a%22%7D; " +
-	"Hm_lvt_44c27e8e603ca3b625b6b1e9c35d712d=1715844560; " +
-	"Hm_lpvt_44c27e8e603ca3b625b6b1e9c35d712d=1715844560; " +
-	"openChat30dbced0-f5cb-11eb-893a-df95eeb4af27=true; " +
-	"accessId=30dbced0-f5cb-11eb-893a-df95eeb4af27; " +
-	"head_search_notes=%E7%8E%89%E7%B1%B3; " +
-	"Hm_lpvt_78a951b1e2ee23efdc6af2ce70d6b9be=1716609966; " +
-	"AUTH_COOKIE=7e198da57104c68c|2024-6-1 12:06:04; " +
-	"STATReferrerIndexId=2; " +
-	"qimo_seosource_30dbced0-f5cb-11eb-893a-df95eeb4af27=%E7%AB%99%E5%86%85; " +
-	"qimo_seokeywords_30dbced0-f5cb-11eb-893a-df95eeb4af27=; pageViewNum=20"
+
 
 // Header 代表表头信息
 type Header struct {

services/sci99/sci99.go

@@ -2,6 +2,7 @@ package sci99
 
 import (
 	"bytes"
+	"context"
 	"encoding/json"
 	"eta/eta_crawler/models"
 	"eta/eta_crawler/utils"
@@ -16,200 +17,376 @@ import (
 
 var IndexCodeMap = make(map[string]string)
 var IndexMap = make(map[string]*models.BaseFromSci99Index)
+var ClassifyMap = make(map[string]*models.BaseFromSci99Classify)
 
-func Scii99() {
-	// 定义请求的URL
-	listUrl := "https://prices.sci99.com/api/zh-cn/product/datavalue"
-	detailUrl := "https://prices.sci99.com/api/zh-cn/dataitem/datavalue"
-
-	// 创建请求体实例并填充数据
-	requestData := ListRequestBody{
-		Ppname:      "天然橡胶原料",
-		PriceTypeID: 34320,
-		PPIDs:       "13676",
-		Navid:       "593",
-		SiteType:    1,
-		PageNo:      1,
-		PageSize:    "1000",
-	}
-
-	// 将请求体序列化为JSON
-	jsonData, err := json.Marshal(requestData)
-	if err != nil {
-		fmt.Println("Error marshalling JSON:", err)
-		return
-	}
-
-	// 创建一个HTTP POST请求
-	req, err := http.NewRequest("POST", listUrl, bytes.NewBuffer(jsonData))
-	if err != nil {
-		fmt.Println("Error creating request:", err)
-		return
-	}
+var PriceTypeIDs = []int{34320, 34318}
+var requestList = []ListRequestBody{
+	{
+		Ppname: "天然橡胶原料",
+		PPIDs:  "13676",
+		Navid:  "593",
+	}, {
+		Ppname: "混合胶",
+		PPIDs:  "12973",
+		Navid:  "591",
+	}, {
+		Ppname: "标准胶",
+		PPIDs:  "12960",
+		Navid:  "590",
+	}, {
+		Ppname: "烟片胶",
+		PPIDs:  "12985",
+		Navid:  "592",
+	}, {
+		Ppname: "天然乳胶",
+		PPIDs:  "12947",
+		Navid:  "595",
+	}, {
+		Ppname: "丁苯橡胶",
+		PPIDs:  "12951",
+		Navid:  "596",
+	}, {
+		Ppname: "顺丁橡胶",
+		PPIDs:  "12964",
+		Navid:  "597",
+	}, {
+		Ppname: "SBS",
+		PPIDs:  "12948",
+		Navid:  "598",
+	}, {
+		Ppname: "丁腈橡胶",
+		PPIDs:  "12945",
+		Navid:  "605",
+	},
+}
 
-	// 设置请求头信息
-	req.Header.Set("Content-Type", "application/json")
-	req.Header.Set("Accept", "application/json")
-	req.Header.Set("Cookie", Cookie)
+func Scii99(cont context.Context) (err error) {
+	filePath := utils.SCI99_COOKIE_PATH
 
-	// 发送请求
-	client := &http.Client{}
-	resp, err := client.Do(req)
+	// 打开文件
+	file, err := ioutil.ReadFile(filePath)
 	if err != nil {
-		fmt.Println("Error sending request:", err)
-		utils.FileLog.Info("Error sending request:", err)
+		utils.FileLog.Info("ReadFile err:", err)
 		return
 	}
-	defer resp.Body.Close()
 
-	// 检查响应状态码
-	if resp.StatusCode != http.StatusOK {
-		fmt.Printf("Server returned non-200 status: %d\n", resp.StatusCode)
-		utils.FileLog.Info("Server returned non-200 status: %d", resp.StatusCode)
-		return
-	}
-	// 打印响应体
-	body, _ := ioutil.ReadAll(resp.Body)
-	fmt.Println("Response body:", string(body))
+	cookie := string(file)
 
-	var listResponse ListResponse
+	// 定义请求的URL
+	listUrl := "https://prices.sci99.com/api/zh-cn/product/datavalue"
+	detailUrl := "https://prices.sci99.com/api/zh-cn/dataitem/datavalue"
 
-	err = json.Unmarshal(body, &listResponse)
-	if err != nil {
-		fmt.Println(err)
-		utils.FileLog.Info("Error unmarshalling JSON:", err)
-		return
-	}
-	allCode, err := models.GetBaseFromSci99Index()
+	allClassify, err := models.GetBaseFromSci99Classify()
 	if err != nil {
 		fmt.Println("select Code err:", err)
+		utils.FileLog.Info("GetBaseFromSci99Index err:", err)
+		return
 	}
-	for _, item := range allCode {
-		IndexCodeMap[item.IndexName] = item.IndexCode
-		IndexMap[item.IndexName] = item
+	for _, item := range allClassify {
+		ClassifyMap[item.ClassifyName] = item
 	}
 
-	for _, v := range listResponse.FirstData.SecondData.Items {
-		// 组成指标
-		indexName := v.ProductName + v.PriceType + v.MarketSampleName + v.Model
-		indexCodeStr := "卓创资讯" + indexName
-		indexCode, needAdd := Sci99IndexCodeGenerator(indexName, indexCodeStr, v.MarketSampleName)
-		if needAdd {
-			index := models.BaseFromSci99Index{
-				IndexCode:  indexCode,
-				IndexName:  indexName,
-				ClassifyId: 0,
-				Unit:       v.Unit,
-				Frequency:  "day",
-				Describe:   v.DataItemName,
-				CreateTime: time.Now(),
-				ModifyTime: time.Now(),
+	for _, request := range requestList {
+		classifyId := 0
+		if classify, ok := ClassifyMap[request.Ppname]; !ok {
+			classifyItem := models.BaseFromSci99Classify{
+				ClassifyName: request.Ppname,
+				CreateTime:   time.Now(),
+				ModifyTime:   time.Now(),
 			}
-			id, err := models.AddBaseFromSci99Index(&index)
+
+			id, err := models.AddBaseFromSci99Classify(&classifyItem)
 			if err != nil {
 				fmt.Println("Error inserting into database:", err)
 				utils.FileLog.Info("Error inserting into database:", err)
 				return
 			}
-			index.BaseFromSciIndexId = int(id)
-			IndexMap[indexName] = &index
+			classifyId = int(id)
+			classifyItem.BaseFromSciClassifyId = int(id)
+			ClassifyMap[request.Ppname] = &classifyItem
+		} else {
+			classifyId = classify.BaseFromSciClassifyId
 		}
 
-		ppid := strconv.Itoa(v.ProductID)
-		dataTypeID := strconv.Itoa(v.DataTypeID)
-		diidd := strconv.Itoa(v.DIID)
-		deailReq := DetailRequest{
-			PPID:       ppid,
-			DataTypeID: dataTypeID,
-			DIIDD:      diidd,
-			APIStart:   "2021/01/01",
-			APIEnd:     "2024/09/11",
-			CycleType:  "day",
-		}
+		// 分别获取国内价格和国际价格
+		for _, priceTypeID := range PriceTypeIDs {
+			// 创建请求体实例并填充数据,这里的参数都是一样的，所以这边统一写
+			request.CycleType = "day"
+			request.SiteType = 1
+			request.PageNo = 1
+			request.PageSize = "1000"
+			request.PriceTypeID = priceTypeID
 
-		// 将请求体序列化为JSON
-		reqJson, e := json.Marshal(deailReq)
-		if e != nil {
-			err = e
-			fmt.Println("Error marshalling JSON:", err)
-			return
-		}
+			// 将请求体序列化为JSON
+			jsonData, err := json.Marshal(request)
+			if err != nil {
+				fmt.Println("Error marshalling JSON:", err)
+				utils.FileLog.Info("Error marshalling JSON:", err)
+				return
+			}
 
-		// 创建一个HTTP POST请求
-		req, err = http.NewRequest("POST", detailUrl, bytes.NewBuffer(reqJson))
-		if err != nil {
-			fmt.Println("Error creating request:", err)
-			return
-		}
+			// 创建一个HTTP POST请求
+			req, err := http.NewRequest("POST", listUrl, bytes.NewBuffer(jsonData))
+			if err != nil {
+				fmt.Println("Error creating request:", err)
+				utils.FileLog.Info("Error creating request:", err)
+				return
+			}
 
-		// 设置请求头信息
-		req.Header.Set("Content-Type", "application/json")
-		req.Header.Set("Accept", "application/json")
-		req.Header.Set("Cookie", Cookie)
-
-		// 发送请求
-		resp, err = client.Do(req)
-		if err != nil {
-			fmt.Println("Error sending request:", err)
-			utils.FileLog.Info("Error sending request:", err)
-			return
-		}
-		defer resp.Body.Close()
-		// 检查响应状态码
-		if resp.StatusCode != http.StatusOK {
-			fmt.Printf("Server returned non-200 status: %d\n", resp.StatusCode)
-			utils.FileLog.Info("Server returned non-200 status: %d", resp.StatusCode)
-			return
-		}
-		// 打印响应体
-		body, _ = ioutil.ReadAll(resp.Body)
-		fmt.Println("Response Detail body:", string(body))
+			// 设置请求头信息
+			req.Header.Set("Content-Type", "application/json")
+			req.Header.Set("Accept", "application/json")
+			req.Header.Set("Cookie", cookie)
+
+			// 发送请求
+			client := &http.Client{}
+			resp, err := client.Do(req)
+			if err != nil {
+				fmt.Println("Error sending request:", err)
+				utils.FileLog.Info("Error sending request:", err)
+				return
+			}
+			defer resp.Body.Close()
 
-		var detailResponse DetailResponse
+			// 检查响应状态码
+			if resp.StatusCode != http.StatusOK {
+				fmt.Printf("Server returned non-200 status: %d\n", resp.StatusCode)
+				utils.FileLog.Info("Server returned non-200 status: %d", resp.StatusCode)
+				return
+			}
+			// 打印响应体
+			body, _ := ioutil.ReadAll(resp.Body)
+			//fmt.Println("Response body:", string(body))
 
-		err = json.Unmarshal(body, &detailResponse)
-		if err != nil {
-			fmt.Println(err)
-			utils.FileLog.Info("Error unmarshalling JSON:", err)
-			return
-		}
+			var listResponse ListResponse
 
-		// 新增data数据
-		addList := make([]*models.BaseFromSci99Data, 0)
-		for _, v := range detailResponse.Data.List {
-			dataItem := models.BaseFromSci99Data{
-				BaseFromSciIndexId: IndexMap[indexName].BaseFromSciIndexId,
-				IndexCode:          indexCode,
-				DataTime:           v.RealDate,
-				Value:              v.MDataValue,
-				CreateTime:         time.Now(),
-				ModifyTime:         time.Now(),
+			err = json.Unmarshal(body, &listResponse)
+			if err != nil {
+				fmt.Println(err)
+				utils.FileLog.Info("Error unmarshalling JSON:", err)
+				return
 			}
-			addList = append(addList, &dataItem)
-		}
-		if len(addList) > 0 {
-			err = models.AddBaseFromSci99DataMulti(addList)
+			allCode, err := models.GetBaseFromSci99Index()
 			if err != nil {
-				fmt.Println("Error inserting into database:", err)
-				utils.FileLog.Info("Error inserting into database:", err)
+				fmt.Println("select Code err:", err)
+				utils.FileLog.Info("GetBaseFromSci99Index err:", err)
 				return
 			}
+			for _, item := range allCode {
+				IndexCodeMap[item.IndexName] = item.IndexCode
+				IndexMap[item.IndexName] = item
+			}
+
+			for _, v := range listResponse.FirstData.SecondData.Items {
+				if v.FromArea != "" && len(listResponse.FirstData.SecondData.Items) > 1 {
+					continue
+				}
+				// 组成指标
+				var indexName string
+				if v.FactorySampleName == "" || len(listResponse.FirstData.SecondData.Items) == 1 {
+					indexName = v.ProductName + "：" + v.PriceType + "：" + v.MarketSampleName + "：" + v.Model
+				} else {
+					indexName = v.ProductName + "：" + v.PriceType + "：" + v.MarketSampleName + "：" + v.FactorySampleName + "：" + v.Model
+				}
+				fmt.Println("indexName:", indexName)
+				indexCodeStr := "卓创资讯" + v.ProductName + v.PriceType
+				indexCode, needAdd := Sci99IndexCodeGenerator(indexName, indexCodeStr, v.MarketSampleName, v.FactorySampleName+v.Model)
+				if needAdd {
+					index := models.BaseFromSci99Index{
+						IndexCode:  indexCode,
+						IndexName:  indexName,
+						ClassifyId: classifyId,
+						Unit:       v.Unit,
+						Frequency:  "日度",
+						Describe:   v.DataItemName,
+						CreateTime: time.Now(),
+						ModifyTime: time.Now(),
+					}
+					id, err := models.AddBaseFromSci99Index(&index)
+					if err != nil {
+						fmt.Println("Error inserting into database:", err)
+						utils.FileLog.Info("Error inserting into database:", err)
+						return
+					}
+					index.BaseFromSciIndexId = int(id)
+					IndexMap[indexName] = &index
+				}
+
+				ppid := strconv.Itoa(v.ProductID)
+				dataTypeID := strconv.Itoa(v.DataTypeID)
+				diidd := strconv.Itoa(v.DIID)
+				deailReq := DetailRequest{
+					PPID:       ppid,
+					DataTypeID: dataTypeID,
+					DIIDD:      diidd,
+					APIStart:   "2021/01/01",
+					APIEnd:     time.Now().AddDate(1, 0, 0).Format(utils.FormatDate2),
+					CycleType:  "day",
+				}
+
+				// 获取最新日期
+				dateTime, err := models.GetBaseFromTradeSci99LatestData(indexCode)
+				if err != nil && err.Error() != utils.ErrNoRow() {
+					utils.FileLog.Info("GetBaseFromTradeSci99LatestData err:", err)
+					return
+				}
+				if !dateTime.IsZero() {
+					deailReq.Start = dateTime.Format(utils.FormatDate2)
+					deailReq.End = time.Now().Format(utils.FormatDate2)
+				}
+
+				// 将请求体序列化为JSON
+				reqJson, e := json.Marshal(deailReq)
+				if e != nil {
+					err = e
+					fmt.Println("Error marshalling JSON:", err)
+					utils.FileLog.Info("Error marshalling JSON:", err)
+					return
+				}
+
+				// 创建一个HTTP POST请求
+				req, err = http.NewRequest("POST", detailUrl, bytes.NewBuffer(reqJson))
+				if err != nil {
+					fmt.Println("Error creating request:", err)
+					utils.FileLog.Info("Error creating request:", err)
+					return
+				}
+
+				// 设置请求头信息
+				req.Header.Set("Content-Type", "application/json")
+				req.Header.Set("Accept", "application/json")
+				req.Header.Set("Cookie", cookie)
+
+				// 发送请求
+				resp, err = client.Do(req)
+				if err != nil {
+					fmt.Println("Error sending request:", err)
+					utils.FileLog.Info("Error sending request:", err)
+					return
+				}
+				defer resp.Body.Close()
+				// 检查响应状态码
+				if resp.StatusCode != http.StatusOK {
+					fmt.Printf("Server returned non-200 status: %d\n", resp.StatusCode)
+					utils.FileLog.Info("Server returned non-200 status: %d", resp.StatusCode)
+					return
+				}
+				// 打印响应体
+				body, _ = ioutil.ReadAll(resp.Body)
+				//fmt.Println("Response Detail body:", string(body))
+
+				var detailResponse DetailResponse
+
+				err = json.Unmarshal(body, &detailResponse)
+				if err != nil {
+					fmt.Println(err)
+					utils.FileLog.Info("Error unmarshalling JSON:", err)
+					return
+				}
+
+
+				existDataMap := make(map[string]*models.BaseFromSci99Data)
+
+				//获取所有指标信息
+				allIndex, err := models.GetBaseFromTradeSci99IndexAll(indexCode)
+				if err != nil {
+					fmt.Println("select err:", err)
+					utils.FileLog.Info("GetBaseFromTradeSci99IndexAll err:", err)
+				}
+				for _, v := range allIndex {
+					indexKey := v.IndexCode + v.DataTime
+					existDataMap[indexKey] = v
+				}
+
+
+				// 新增data数据
+				addList := make([]*models.BaseFromSci99Data, 0)
+				for _, v := range detailResponse.Data.List {
+					if dataItem, ok := existDataMap[indexCode + v.RealDate]; ok {
+						if dataItem.Value != v.MDataValue {
+							// 更新
+							err = models.UpdateBaseFromSci99Data(dataItem.Value, indexCode, v.RealDate)
+							if err != nil {
+								fmt.Println("Error update into database:", err)
+								utils.FileLog.Info("Error update into database:", err)
+								return
+							}
+						}
+					} else {
+						// 新增
+						dataItem := models.BaseFromSci99Data{
+							BaseFromSciIndexId: IndexMap[indexName].BaseFromSciIndexId,
+							IndexCode:          indexCode,
+							DataTime:           v.RealDate,
+							Value:              v.MDataValue,
+							CreateTime:         time.Now(),
+							ModifyTime:         time.Now(),
+						}
+						addList = append(addList, &dataItem)
+					}
+				}
+				if len(addList) > 0 {
+					err = models.AddBaseFromSci99DataMulti(addList)
+					if err != nil {
+						fmt.Println("Error inserting into database:", err)
+						utils.FileLog.Info("Error inserting into database:", err)
+						return
+					}
+				}
+				time.Sleep(1 * time.Second)
+			}
 		}
 	}
 }
 
-func Sci99IndexCodeGenerator(indexName, indexCodeStr, marketSampleName string) (indexCode string, needAdd bool) {
+func Sci99IndexCodeGenerator(indexName, indexCodeStr, marketSampleName, model string) (indexCode string, needAdd bool) {
 	strResult := ""
 	indexCode, _ = IndexCodeMap[indexName]
 	if indexCode == "" {
 		//首字母
 		a := pinyin.NewArgs()
-		a.Style = pinyin.FirstLetter
-		rows := pinyin.LazyPinyin(indexCodeStr, a)
+		a.Fallback = func(r rune, a pinyin.Args) []string {
+			return []string{string(r)}
+		}
+		rows := pinyin.Pinyin(indexCodeStr, a)
 		for i := 0; i < len(rows); i++ {
-			strResult += rows[i]
+			//strResult += rows[i][0]
+			if len(rows[i]) != 0 {
+				str := rows[i][0]
+				pi := str[0:1]
+				strResult += pi
+			}
+		}
+
+		// 处理市场名称
+		if province, ok := ProvinceMap[marketSampleName]; ok {
+			strResult += province
+		} else {
+			a := pinyin.NewArgs()
+			rows := pinyin.LazyPinyin(marketSampleName, a)
+			for i := 0; i < len(rows); i++ {
+				strResult += rows[i]
+			}
+		}
+
+		// 去除特殊符号
+		model = strings.Replace(model, " ", "", -1)
+		model = strings.Replace(model, "-", "", -1)
+		model = strings.Replace(model, "/", "", -1)
+		model = strings.Replace(model, "#", "", -1)
+		model = strings.Replace(model, "：", "", -1)
+		model = strings.Replace(model, "(", "", -1)
+		model = strings.Replace(model, ")", "", -1)
+
+		// 拼接型号
+		modelRows := pinyin.Pinyin(model, a)
+		for i := 0; i < len(modelRows); i++ {
+			if len(modelRows[i]) != 0 {
+				str := modelRows[i][0]
+				pi := str[0:1]
+				strResult += pi
+			}
 		}
-		strResult += ProvinceMap[marketSampleName]
 
 		needAdd = true
 		indexCode = strings.Replace(strResult, " ", "", -1)

services/task.go

@@ -2,6 +2,7 @@ package services
 
 import (
 	"context"
+	"eta/eta_crawler/services/sci99"
 	"eta/eta_crawler/utils"
 	"fmt"
 	"github.com/beego/beego/v2/task"
@@ -20,34 +21,42 @@ func Task() {
 	//SyncRankingFromIne()
 	//FileCoalCoastal()
 	//FileCoalInland()
-	refreshData := task.NewTask("refreshData", "0 0,30 16-18 * * *", RefreshData)
-	refreshEic := task.NewTask("RefreshEic", "0 0 2,6 * * *", RefreshEic)
-	refreshCoal := task.NewTask("RefreshCoal", "0 0,30 16-23 * * *", RefreshCoal)
-	refreshMeeting := task.NewTask("RefreshMeeting", "0 15 7-19/2 * * *", RefreshMeetingProbabilities)
-	//	refreshVisitors := task.NewTask("RefreshChangesVisitorsCovid", "0 30 2-22/10 * * *", RefreshChangesVisitorsCovid)
-	syncEiaSteoData := task.NewTask("SyncEiaSteoData", "0 0 22 * * *", SyncEiaSteoData)
-	syncYearComTrade := task.NewTask("SyncYearComTrade", "0 0 3 1 1 *", SyncYearComTrade)                // 每年一月一号同步
-	syncYearMonthComTrade := task.NewTask("SyncYearMonthComTrade", "0 0 3 1 * *", SyncYearMonthComTrade) // 每月1号同步
-
-	crawlerIcpi := task.NewTask("refreshData", "0 0,30 16-23 * * *", CrawlerIcpi) //居民消费价格指数
-
-	// 统计局-分月季年爬
-	//refreshNationalMonthA := task.NewTask("RefreshNationalMonthDbA", "0 15 2 10 * *", national_data.RefreshNationalMonthDbA)
-	//refreshNationalMonthB := task.NewTask("RefreshNationalMonthDbB", "0 15 2 16 * *", national_data.RefreshNationalMonthDbB)
-	//refreshNationalQuarter := task.NewTask("RefreshNationalQuarterDb", "0 25 1 15 * *", national_data.RefreshNationalQuarterDb)
-	//refreshNationalYearA := task.NewTask("RefreshNationalYearDbA", "0 45 1 20 * *", national_data.RefreshNationalYearDbA)
-	//refreshNationalYearB := task.NewTask("RefreshNationalYearDbB", "0 45 1 25 * *", national_data.RefreshNationalYearDbB)
-
-	task.AddTask("数据爬取", refreshData)
-	task.AddTask("欧洲天然气爬取", refreshEic)
-	task.AddTask("中国煤炭网爬取", refreshCoal)
-	task.AddTask("美联储加息概率爬取", refreshMeeting)
-	//task.AddTask("谷歌出行指数爬取", refreshVisitors)
-	task.AddTask("eia steo报告", syncEiaSteoData)   //每天22点爬一次
-	task.AddTask("UN年度数据", syncYearComTrade)      //每年一月一号的3点同步
-	task.AddTask("UN月度数据", syncYearMonthComTrade) //每月1号的3点同步
-
-	task.AddTask("居民消费价格指数", crawlerIcpi) //每月1号的3点同步
+	if utils.BusinessCode == utils.BusinessCodeRelease {
+		refreshData := task.NewTask("refreshData", "0 0,30 16-18 * * *", RefreshData)
+		refreshEic := task.NewTask("RefreshEic", "0 0 2,6 * * *", RefreshEic)
+		refreshCoal := task.NewTask("RefreshCoal", "0 0,30 16-23 * * *", RefreshCoal)
+		refreshMeeting := task.NewTask("RefreshMeeting", "0 15 7-19/2 * * *", RefreshMeetingProbabilities)
+		//	refreshVisitors := task.NewTask("RefreshChangesVisitorsCovid", "0 30 2-22/10 * * *", RefreshChangesVisitorsCovid)
+		syncEiaSteoData := task.NewTask("SyncEiaSteoData", "0 0 22 * * *", SyncEiaSteoData)
+		syncYearComTrade := task.NewTask("SyncYearComTrade", "0 0 3 1 1 *", SyncYearComTrade)                // 每年一月一号同步
+		syncYearMonthComTrade := task.NewTask("SyncYearMonthComTrade", "0 0 3 1 * *", SyncYearMonthComTrade) // 每月1号同步
+
+		crawlerIcpi := task.NewTask("refreshData", "0 0,30 16-23 * * *", CrawlerIcpi) //居民消费价格指数
+
+		// 统计局-分月季年爬
+		//refreshNationalMonthA := task.NewTask("RefreshNationalMonthDbA", "0 15 2 10 * *", national_data.RefreshNationalMonthDbA)
+		//refreshNationalMonthB := task.NewTask("RefreshNationalMonthDbB", "0 15 2 16 * *", national_data.RefreshNationalMonthDbB)
+		//refreshNationalQuarter := task.NewTask("RefreshNationalQuarterDb", "0 25 1 15 * *", national_data.RefreshNationalQuarterDb)
+		//refreshNationalYearA := task.NewTask("RefreshNationalYearDbA", "0 45 1 20 * *", national_data.RefreshNationalYearDbA)
+		//refreshNationalYearB := task.NewTask("RefreshNationalYearDbB", "0 45 1 25 * *", national_data.RefreshNationalYearDbB)
+
+		task.AddTask("数据爬取", refreshData)
+		task.AddTask("欧洲天然气爬取", refreshEic)
+		task.AddTask("中国煤炭网爬取", refreshCoal)
+		task.AddTask("美联储加息概率爬取", refreshMeeting)
+		//task.AddTask("谷歌出行指数爬取", refreshVisitors)
+		task.AddTask("eia steo报告", syncEiaSteoData)   //每天22点爬一次
+		task.AddTask("UN年度数据", syncYearComTrade)      //每年一月一号的3点同步
+		task.AddTask("UN月度数据", syncYearMonthComTrade) //每月1号的3点同步
+
+		task.AddTask("居民消费价格指数", crawlerIcpi) //每月1号的3点同步
+	}
+
+	if utils.BusinessCode == utils.BusinessCodeFuBang {
+		// 仅富邦
+		sci99 := task.NewTask("refreshSci99", "0 0,30 16-20 * * *", sci99.Scii99) //卓创资讯
+		task.AddTask("卓创资讯", sci99)
+	}
 
 	//task.AddTask("统计局数据爬取-月度A", refreshNationalMonthA) // 每月10号2:15执行
 	//task.AddTask("统计局数据爬取-月度B", refreshNationalMonthB) // 每月16号2:15执行

utils/config.go

@@ -24,6 +24,12 @@ var (
 	LogMaxDays int //日志最大保留天数
 )
 
+var BusinessCode  string //商户号
+
+var (
+	SCI99_COOKIE_PATH string
+)
+
 func init() {
 	tmpRunMode, err := web.AppConfig.String("run_mode")
 	if err != nil {
@@ -52,7 +58,9 @@ func init() {
 	beeLogger.Log.Info(RunMode + " 模式")
 	MYSQL_URL = config["mysql_url"]
 	MYSQL_URL_DATA = config["mysql_url_data"]
-
+	SCI99_COOKIE_PATH = config["sci99_cookie_path"]
+	// 商家编码
+	BusinessCode = config["business_code"]
 	if RunMode == "release" {
 
 	} else {

utils/constants.go

@@ -4,10 +4,11 @@ const (
 	Md5Key = "Ks@h64WJ#tcVgG8$&WlNfqvLAtMgpxWN"
 )
 
-//常量定义
+// 常量定义
 const (
 	FormatTime             = "15:04:05"                //时间格式
 	FormatDate             = "2006-01-02"              //日期格式
+	FormatDate2            = "2006/01/02"              //日期格式
 	FormatDateUnSpace      = "20060102"                //日期格式
 	FormatDateTime         = "2006-01-02 15:04:05"     //完整时间格式
 	HlbFormatDateTime      = "2006-01-02_15:04:05.999" //完整时间格式
@@ -24,3 +25,8 @@ const (
 	APPNAME          = "弘则-数据爬虫"
 	EmailSendToUsers = "glji@hzinsights.com;pyan@hzinsights.com;cxzhang@hzinsights.com;zwxi@hzinsights.com;"
 )
+
+const (
+	BusinessCodeRelease = "E2023080900" // 生产环境
+	BusinessCodeFuBang  = "E2024020200"
+)