Merge branch 'feature/cffex_products'

services/commodity_trade_cffex.go

@@ -1,15 +1,19 @@
 package services
 
 import (
+	"bufio"
+	"encoding/json"
 	"encoding/xml"
 	"eta/eta_crawler/models"
 	"eta/eta_crawler/utils"
 	"fmt"
-	"github.com/mozillazg/go-pinyin"
 	"io/ioutil"
 	netHttp "net/http"
+	"regexp"
 	"strings"
 	"time"
+
+	"github.com/mozillazg/go-pinyin"
 )
 
 type XmlItem struct {
@@ -56,6 +60,36 @@ func cffexIndexCodeGenerator(shortName, indexName, contractCode, suffix string)
 	return cffexIndexCode
 }
 
+// getProductId 获取中金所产品ID
+func getProductId() ([]string, error) {
+	var productUrl = "http://www.cffex.com.cn/r/cms/www/default/js/config.js"
+	req, _ := netHttp.NewRequest("GET", productUrl, nil)
+	req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.111 Safari/537.36")
+	client := netHttp.Client{}
+	res, err := client.Do(req)
+	if err != nil {
+		return []string{}, err
+	}
+	defer res.Body.Close()
+	scanner := bufio.NewScanner(res.Body)
+	for scanner.Scan() {
+		line := scanner.Text()
+		if strings.Contains(line, "productids :") {
+			reg := regexp.MustCompile(`\[(.*?)\]`)
+			var result []string
+			err = json.Unmarshal([]byte(reg.FindString(line)), &result)
+			if err != nil {
+				return []string{}, err
+			}
+			return result, nil
+		}
+	}
+	if err := scanner.Err(); err != nil {
+		return []string{}, err
+	}
+	return []string{}, err
+}
+
 // SyncRankingFromCffex 中国金融期货交易所
 func SyncRankingFromCffex() {
 	n := utils.GetRandInt(10, 120)
@@ -67,6 +101,10 @@ func SyncRankingFromCffex() {
 	for _, item := range allCode {
 		cffexIndexCodeMap[item.IndexName] = item.IndexCode
 	}
+	productIds, err := getProductId()
+	if err != nil {
+		utils.FileLog.Info("getProductId err:", err)
+	}
 	for i := 2; i >= 0; i-- {
 		zzUrl := "http://www.cffex.com.cn/sj/ccpm/%s/%s/"
 		date := time.Now().AddDate(0, 0, -i)
@@ -76,15 +114,23 @@ func SyncRankingFromCffex() {
 		zzUrl = fmt.Sprintf(zzUrl, pre, sub)
 		fmt.Println(zzUrl)
 
-		var xmlItems = make([]*XmlItem, 8)
-		xmlItems[0] = GetXmlItem(zzUrl, "IF")
-		xmlItems[1] = GetXmlItem(zzUrl, "IC")
-		xmlItems[2] = GetXmlItem(zzUrl, "IH")
-		xmlItems[3] = GetXmlItem(zzUrl, "TS")
-		xmlItems[4] = GetXmlItem(zzUrl, "TF")
-		xmlItems[5] = GetXmlItem(zzUrl, "T")
-		xmlItems[6] = GetXmlItem(zzUrl, "IM")
-		xmlItems[7] = GetXmlItem(zzUrl, "TL")
+		var xmlItems = make([]*XmlItem, len(productIds))
+		if len(productIds) == 0 {
+			utils.FileLog.Info("SyncRankingFromCffex warning:中金所无法获取产品信息, 使用默认产品")
+			xmlItems = make([]*XmlItem, 8)
+			xmlItems[0] = GetXmlItem(zzUrl, "IF")
+			xmlItems[1] = GetXmlItem(zzUrl, "IC")
+			xmlItems[2] = GetXmlItem(zzUrl, "IH")
+			xmlItems[3] = GetXmlItem(zzUrl, "TS")
+			xmlItems[4] = GetXmlItem(zzUrl, "TF")
+			xmlItems[5] = GetXmlItem(zzUrl, "T")
+			xmlItems[6] = GetXmlItem(zzUrl, "IM")
+			xmlItems[7] = GetXmlItem(zzUrl, "TL")
+		} else {
+			for i, productId := range productIds {
+				xmlItems[i] = GetXmlItem(zzUrl, productId)
+			}
+		}
 		//获取新的指标信息
 		var sRank string
 
@@ -116,8 +162,8 @@ func SyncRankingFromCffex() {
 		var indexKey string
 		for _, xmlItem := range xmlItems {
 			for _, i := range xmlItem.Data {
-				i.ShortName = strings.Replace(i.ShortName,"(经纪)","",-1)
-				i.ShortName = strings.Replace(i.ShortName,"(代客)","",-1)
+				i.ShortName = strings.Replace(i.ShortName, "(经纪)", "", -1)
+				i.ShortName = strings.Replace(i.ShortName, "(代客)", "", -1)
 				var item = new(models.BaseFromTradeCffexIndex)
 				item.DealValue = -1
 				item.BuyValue = -1