eta_crawler/services/commodity_trade_cffex.go

package services

import (
	"bufio"
	"encoding/json"
	"encoding/xml"
	"eta/eta_crawler/models"
	"eta/eta_crawler/utils"
	"fmt"
	"io/ioutil"
	netHttp "net/http"
	"regexp"
	"strings"
	"time"

	"github.com/mozillazg/go-pinyin"
)

type XmlItem struct {
	XMLName xml.Name `xml:"positionRank"`
	Data    []struct {
		ContractCode string `xml:"instrumentid"`
		Value        string `xml:"Value,attr"`
		Text         string `xml:"Text,attr"`
		Tradingday   string `xml:"tradingday"`
		Datatypeid   string `xml:"datatypeid"`
		Rank         int    `xml:"rank"`
		ShortName    string `xml:"shortname"`
		Volume       int    `xml:"volume"`
		Varvolume    int    `xml:"varvolume"`
		ProductId    string `xml:"productid"`
	} `xml:"data"`
}

var cffexIndexCode string
var cffexIndexCodeMap = make(map[string]string)
var cffexActionCodeMap = make(map[string]map[string]int)

func cffexIndexCodeGenerator(shortName, indexName, contractCode, suffix string) string {
	if shortName == "" {
		cffexIndexCode = ""
		return cffexIndexCode
	}
	//取公司全拼
	strResult := ""
	a := pinyin.NewArgs()
	rows := pinyin.LazyPinyin(shortName, a)
	for i := 0; i < len(rows); i++ {
		strResult += rows[i]
	}
	cffexIndexCode, _ := cffexIndexCodeMap[indexName]
	if cffexIndexCode == "" {
		cffexIndexCode = strResult + contractCode + suffix
		cffexIndexCodeMap[indexName] = cffexIndexCode
		err := models.AddBaseFromTradeMapping(indexName, cffexIndexCode, "CFFEX")
		if err != nil {
			fmt.Println("add Code err:", err)
		}
	}
	return cffexIndexCode
}

// getProductId 获取中金所产品ID
func getProductId() ([]string, error) {
	var productUrl = "http://www.cffex.com.cn/r/cms/www/default/js/config.js"
	req, _ := netHttp.NewRequest("GET", productUrl, nil)
	req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.111 Safari/537.36")
	client := netHttp.Client{}
	res, err := client.Do(req)
	if err != nil {
		return []string{}, err
	}
	defer res.Body.Close()
	scanner := bufio.NewScanner(res.Body)
	for scanner.Scan() {
		line := scanner.Text()
		if strings.Contains(line, "productids :") {
			reg := regexp.MustCompile(`\[(.*?)\]`)
			var result []string
			err = json.Unmarshal([]byte(reg.FindString(line)), &result)
			if err != nil {
				return []string{}, err
			}
			return result, nil
		}
	}
	if err := scanner.Err(); err != nil {
		return []string{}, err
	}
	return []string{}, err
}

// SyncRankingFromCffex 中国金融期货交易所
func SyncRankingFromCffex() {
	n := utils.GetRandInt(10, 120)
	time.Sleep(time.Duration(n) * time.Second)
	allCode, err := models.GetIndexCodeFromMapping("CFFEX")
	if err != nil {
		fmt.Println("select Code err:", err)
	}
	for _, item := range allCode {
		cffexIndexCodeMap[item.IndexName] = item.IndexCode
	}
	productIds, err := getProductId()
	if err != nil {
		utils.FileLog.Info("getProductId err:", err)
	}
	for i := 2; i >= 0; i-- {
		zzUrl := "http://www.cffex.com.cn/sj/ccpm/%s/%s/"
		date := time.Now().AddDate(0, 0, -i)
		dateStr := date.Format(utils.FormatDateUnSpace)
		pre := dateStr[:6]
		sub := dateStr[6:]
		zzUrl = fmt.Sprintf(zzUrl, pre, sub)
		fmt.Println(zzUrl)

		var xmlItems = make([]*XmlItem, len(productIds))
		if len(productIds) == 0 {
			utils.FileLog.Info("SyncRankingFromCffex warning:中金所无法获取产品信息, 使用默认产品")
			xmlItems = make([]*XmlItem, 8)
			xmlItems[0] = GetXmlItem(zzUrl, "IF")
			xmlItems[1] = GetXmlItem(zzUrl, "IC")
			xmlItems[2] = GetXmlItem(zzUrl, "IH")
			xmlItems[3] = GetXmlItem(zzUrl, "TS")
			xmlItems[4] = GetXmlItem(zzUrl, "TF")
			xmlItems[5] = GetXmlItem(zzUrl, "T")
			xmlItems[6] = GetXmlItem(zzUrl, "IM")
			xmlItems[7] = GetXmlItem(zzUrl, "TL")
		} else {
			for i, productId := range productIds {
				xmlItems[i] = GetXmlItem(zzUrl, productId)
			}
		}
		//获取新的指标信息
		var sRank string

		//获取所有指标信息
		allIndex, err := models.GetBaseFromTradeCffexIndexAll(dateStr)
		if err != nil {
			fmt.Println("select err:", err)
		}

		existDealMap := make(map[string]int)
		existBuyMap := make(map[string]int)
		existSoldMap := make(map[string]int)
		for _, v := range allIndex {
			existDealMap[v.DealName] = v.BaseFromTradeCffexIndexId
			existBuyMap[v.BuyName] = v.BaseFromTradeCffexIndexId
			existSoldMap[v.SoldName] = v.BaseFromTradeCffexIndexId
			sRank = fmt.Sprintf("%d", v.Rank)
			time := strings.Replace(v.DataTime, "-", "", -1)
			cffexActionCodeMap[sRank+v.ClassifyType+time] = map[string]int{"0": v.DealValue, "1": v.BuyValue, "2": v.SoldValue, "id": v.BaseFromTradeCffexIndexId}
		}
		shortNameColum := map[string]string{"0": "deal_short_name", "1": "buy_short_name", "2": "sold_short_name"}
		nameColum := map[string]string{"0": "deal_name", "1": "buy_name", "2": "sold_name"}
		codeColum := map[string]string{"0": "deal_code", "1": "buy_code", "2": "sold_code"}
		valueColum := map[string]string{"0": "deal_value", "1": "buy_value", "2": "sold_value"}
		changeColum := map[string]string{"0": "deal_change", "1": "buy_change", "2": "sold_change"}
		var dataName string
		var dataCode string

		var indexKey string
		for _, xmlItem := range xmlItems {
			for _, i := range xmlItem.Data {
				i.ShortName = strings.Replace(i.ShortName, "(经纪)", "", -1)
				i.ShortName = strings.Replace(i.ShortName, "(代客)", "", -1)
				var item = new(models.BaseFromTradeCffexIndex)
				item.DealValue = -1
				item.BuyValue = -1
				item.SoldValue = -1
				item.Rank = i.Rank
				switch i.Value {
				case "0":
					item.DealShortName = i.ShortName
					item.DealName = fmt.Sprintf("%s", i.ShortName+"_"+i.ContractCode+"_成交量(手)")
					item.DealCode = cffexIndexCodeGenerator(item.DealShortName, item.DealName, i.ContractCode, "deal")
					item.DealValue = i.Volume
					item.DealChange = i.Varvolume

					dataName = item.DealName
					dataCode = item.DealCode
				case "1":
					item.BuyShortName = i.ShortName
					item.BuyName = fmt.Sprintf("%s", i.ShortName+"_"+i.ContractCode+"_持买单量(手)")
					item.BuyCode = cffexIndexCodeGenerator(item.BuyShortName, item.BuyName, i.ContractCode, "buy")
					item.BuyValue = i.Volume
					item.BuyChange = i.Varvolume

					dataName = item.BuyName
					dataCode = item.BuyCode
				case "2":
					item.SoldShortName = i.ShortName
					item.SoldName = fmt.Sprintf("%s", i.ShortName+"_"+i.ContractCode+"_持卖单量(手)")
					item.SoldCode = cffexIndexCodeGenerator(item.SoldShortName, item.SoldName, i.ContractCode, "sold")
					item.SoldValue = i.Volume
					item.SoldChange = i.Varvolume

					dataName = item.SoldName
					dataCode = item.SoldCode
				default:
					fmt.Println("No data value")
				}
				item.Frequency = "日度"
				item.ClassifyName = i.ProductId
				item.ClassifyType = i.ContractCode
				item.CreateTime = time.Now()
				item.ModifyTime = time.Now()
				item.DataTime = i.Tradingday

				valueMap := map[string]int{"0": item.DealValue, "1": item.BuyValue, "2": item.SoldValue}
				//检查在actionCodeMap中是否已经有了
				indexKey = fmt.Sprintf("%d", item.Rank) + item.ClassifyType + item.DataTime

				if existMap, ok := cffexActionCodeMap[indexKey]; !ok {
					//没有，新增
					newID, err := models.AddBaseFromTradeCffexIndex(item)
					if err != nil {
						fmt.Println("insert error:", err)
					}
					fmt.Println("insert new indexID:", newID)
					existMap = make(map[string]int)
					existMap[i.Value] = i.Volume
					existMap["id"] = int(newID)
					cffexActionCodeMap[indexKey] = existMap
				} else if existMap != nil && existMap[i.Value] != i.Volume {
					//更新
					var columnList = [5]string{
						shortNameColum[i.Value],
						nameColum[i.Value],
						codeColum[i.Value],
						valueColum[i.Value],
						changeColum[i.Value],
					}
					var dataList = [5]interface{}{
						i.ShortName,
						dataName,
						dataCode,
						valueMap[i.Value],
						i.Varvolume,
					}
					err := models.ModifyBaseFromTradeCffexIndex(columnList, dataList, existMap["id"])
					if err != nil {
						fmt.Println("data update err:", err)
					}
					cffexActionCodeMap[indexKey][i.Value] = i.Volume
				} else if _, ok := existDealMap[item.DealName]; !ok {
					//更新
					var columnList = [5]string{
						shortNameColum[i.Value],
						nameColum[i.Value],
						codeColum[i.Value],
						valueColum[i.Value],
						changeColum[i.Value],
					}
					var dataList = [5]interface{}{
						i.ShortName,
						dataName,
						dataCode,
						valueMap[i.Value],
						i.Varvolume,
					}
					err := models.ModifyBaseFromTradeCffexIndex(columnList, dataList, existMap["id"])
					if err != nil {
						fmt.Println("data update err:", err)
					}
					cffexActionCodeMap[indexKey][i.Value] = i.Volume
				} else if _, ok := existBuyMap[item.BuyName]; !ok {
					//更新
					var columnList = [5]string{
						shortNameColum[i.Value],
						nameColum[i.Value],
						codeColum[i.Value],
						valueColum[i.Value],
						changeColum[i.Value],
					}
					var dataList = [5]interface{}{
						i.ShortName,
						dataName,
						dataCode,
						valueMap[i.Value],
						i.Varvolume,
					}
					err := models.ModifyBaseFromTradeCffexIndex(columnList, dataList, existMap["id"])
					if err != nil {
						fmt.Println("data update err:", err)
					}
					cffexActionCodeMap[indexKey][i.Value] = i.Volume
				} else if _, ok := existSoldMap[item.SoldName]; !ok {
					//更新
					var columnList = [5]string{
						shortNameColum[i.Value],
						nameColum[i.Value],
						codeColum[i.Value],
						valueColum[i.Value],
						changeColum[i.Value],
					}
					var dataList = [5]interface{}{
						i.ShortName,
						dataName,
						dataCode,
						valueMap[i.Value],
						i.Varvolume,
					}
					err := models.ModifyBaseFromTradeCffexIndex(columnList, dataList, existMap["id"])
					if err != nil {
						fmt.Println("data update err:", err)
					}
					cffexActionCodeMap[indexKey][i.Value] = i.Volume
				}
			}
		}
	}
}

func GetXmlItem(url, contract string) *XmlItem {
	zzUrl := url + contract + ".xml"
	fmt.Println(zzUrl)
	//body, err := http.Get(zzUrl)
	//if err != nil {
	//	fmt.Println("err:", err)
	//}
	req, _ := netHttp.NewRequest("GET", zzUrl, nil)
	req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.111 Safari/537.36")
	client := netHttp.Client{}
	res, err := client.Do(req)
	if err != nil {
		fmt.Println(err)
		return nil
	}
	defer res.Body.Close()
	// 读取响应的内容
	body, err := ioutil.ReadAll(res.Body)
	if err != nil {
		fmt.Println(err)
		return nil
	}

	xmlItem := new(XmlItem)
	err = xml.Unmarshal(body, &xmlItem)
	if err != nil {
		fmt.Println("xml.Unmarshal:", err.Error())
	}
	return xmlItem

}