eta_crawler/services/commodity_trade_zhengzhou.go

package services

import (
	"fmt"
	"github.com/mozillazg/go-pinyin"
	"github.com/rdlucklib/rdluck_tools/http"
	"hongze/hongze_data_crawler/models"
	"hongze/hongze_data_crawler/utils"
	"log"
	"strconv"
	"strings"
	"time"

	"github.com/PuerkitoBio/goquery"
)

var zIndexCode string
var zIndexCodeMap = make(map[string]string)
var zActionCodeMap = make(map[string]map[string]int)

func zIndexCodeGenerator(shortName, indexName, contractCode, suffix string) string {
	if shortName == "" || shortName == "-"{
		return ""
	}
	strResult := ""
	if shortName != "top20" {
		//取公司全拼
		a := pinyin.NewArgs()
		rows := pinyin.LazyPinyin(shortName, a)
		for i:=0;i<len(rows);i++{
			strResult += rows[i]
		}
	} else {
		strResult = "top20"
	}
	//郑州特殊处理，当合约号有中文时只取英文代号 如 苹果AP ---> AP
	if len(contractCode) > 7 {
		contractCode = contractCode[len(contractCode)-2:]
	}
	zIndexCode, _ := zIndexCodeMap[indexName]
	if zIndexCode == "" {
		zIndexCode = strResult + contractCode + suffix
		zIndexCodeMap[indexName] = zIndexCode
		err := models.AddBaseFromTradeMapping(indexName, zIndexCode, "Z")
		if err != nil {
			fmt.Println("add Code err:", err)
		}
	}
	return zIndexCode
}

//郑州商品交易所持单排名
func SyncRankingFromZhengzhou() {
	n := utils.GetRandInt(10, 120)
	time.Sleep(time.Duration(n) * time.Second)
	fmt.Println("start")
	var err error
	defer func() {
		if err != nil {
			fmt.Println("Err:" + err.Error())
		}
	}()
	allCode, err := models.GetIndexCodeFromMapping("Z")
	if err != nil {
		fmt.Println("select Code err:", err)
	}
	for _, item := range allCode {
		zIndexCodeMap[item.IndexName] = item.IndexCode
	}
	for i := 2; i >= 0; i-- {
		var itemVerifyCode int
		zzUrl := "http://www.czce.com.cn/cn/DFSStaticFiles/Future/%s/%s/FutureDataHolding.htm"
		date := time.Now().AddDate(0, 0, -i)
		year := date.Year()
		dateStr := date.Format(utils.FormatDateUnSpace)
		zzUrl = fmt.Sprintf(zzUrl, strconv.Itoa(year), dateStr)
		fmt.Println(zzUrl)
		body, err := http.Get(zzUrl)
		if err != nil {
			fmt.Println("GetData Err:" + err.Error())
			return
		}
		//获取所有指标信息  某一天的
		allIndex, err := models.GetBaseFromTradeZhengzhouIndexAll(dateStr)
		if err != nil {
			return
		}

		existIndexMap := make(map[string]*models.BaseFromTradeZhengzhouIndex)
		for _, v := range allIndex {
			indexKey := v.DealName + v.BuyName + v.SoldName
			existIndexMap[indexKey] = v
		}
		bodyStr := string(body)
		utils.FileLog.Info(bodyStr)
		if strings.Contains(bodyStr, "404.htm") {
			continue
		}
		doc, err := goquery.NewDocumentFromReader(strings.NewReader(bodyStr))
		if err != nil {
			log.Fatal(err)
		}
		table := doc.Find("table")
		var classifyName, classifyType, tradeDate string
		var dealSuffix, dealAddCutSuffix, buySuffix, buyAddCutSuffix, sellSuffix, sellAddCutSuffix string
		var dealVal, dealAddCutVal, buyVal, buyAddCutVal, sellVal, sellAddCutVal string
		var rank string
		table.Find("tr").Each(func(i int, tr *goquery.Selection) {
			var memberShortNameArr []string
			tds := tr.Find("td")
			if tds.Length() == 1 {
				tdText := tds.Text()
				utils.FileLog.Info(tdText)
				if tdText != "" {
					tdTextArr := strings.Split(tdText, "    ")
					for k, v := range tdTextArr {
						//fmt.Println(k, v)
						if k == 0 {
							classifyName = v
						} else {
							tradeDate = v
						}
					}
				}
			} else {
				tds.Each(func(tk int, td *goquery.Selection) {
					tdText := td.Text()
					tdText = strings.Trim(tdText, " ")
					if tdText == "" {
						return
					}
					//fmt.Println("tdText: ", tdText)
					if tk == 0 {
						if !(strings.Contains(tdText, "名次") || strings.Contains(tdText, "合计")) {
							rank = tdText
						}
					}

					if tk == 1 {
						if !strings.Contains(tdText, "会员简称") {
							memberShortNameArr = append(memberShortNameArr, tdText)
						}
					}

					if tk == 4 {
						if !strings.Contains(tdText, "会员简称") {
							memberShortNameArr = append(memberShortNameArr, tdText)
						}
					}

					if tk == 7 {
						if !strings.Contains(tdText, "会员简称") {
							memberShortNameArr = append(memberShortNameArr, tdText)
						}
					}

					if tk == 2 {
						if strings.Contains(tdText, "成交量") {
							dealSuffix = tdText
						} else {
							dealVal = tdText
						}
					}
					if tk == 3 {
						if strings.Contains(tdText, "增减量") {
							dealAddCutSuffix = tdText
						} else {
							dealAddCutVal = tdText
						}
					}
					if tk == 5 {
						if strings.Contains(tdText, "持买仓量") {
							buySuffix = tdText
						} else {
							buyVal = tdText
						}
					}
					if tk == 6 {
						if strings.Contains(tdText, "增减量") {
							buyAddCutSuffix = tdText
						} else {
							buyAddCutVal = tdText
						}
					}
					if tk == 8 {
						if strings.Contains(tdText, "持卖仓量") {
							sellSuffix = tdText
						} else {
							sellVal = tdText
						}
					}
					if tk == 9 {
						if strings.Contains(tdText, "增减量") {
							sellAddCutSuffix = tdText
						} else {
							sellAddCutVal = tdText
						}
					}
				})

				//处理指标
				dealVal = strings.Replace(dealVal, ",", "", -1)
				dealAddCutVal = strings.Replace(dealAddCutVal, ",", "", -1)
				buyVal = strings.Replace(buyVal, ",", "", -1)
				buyAddCutVal = strings.Replace(buyAddCutVal, ",", "", -1)
				sellVal = strings.Replace(sellVal, ",", "", -1)
				sellAddCutVal = strings.Replace(sellAddCutVal, ",", "", -1)
				tradeDate = strings.Replace(tradeDate, "日期：", "", -1)

				classifyNameArr := strings.Split(classifyName, "：")

				if len(classifyNameArr) > 1 {
					classifyType = classifyNameArr[0]
					classifyName = classifyNameArr[1]
				}
				//fmt.Printf("memberShortNameArr:%s,len:%d\n", memberShortNameArr, len(memberShortNameArr))
				if len(memberShortNameArr) != 0 {
					if len(memberShortNameArr[0]) != 2 {
						if len(classifyName) > 7 {
							classifyName = classifyName[len(classifyName)-2:]
						}
						item := new(models.BaseFromTradeZhengzhouIndex)
						item.Rank, _ = strconv.Atoi(rank)
						item.DealShortName = memberShortNameArr[0]
						item.DealName = memberShortNameArr[0] + "_" + classifyName + "_" + dealSuffix
						item.DealCode = zIndexCodeGenerator(item.DealShortName, item.DealName, classifyName, "deal")
						item.DealValue, _ = strconv.Atoi(dealVal)
						item.DealChange, _ = strconv.Atoi(dealAddCutVal)
						item.BuyShortName = memberShortNameArr[1]
						item.BuyName = memberShortNameArr[1] + "_" + classifyName + "_" + buySuffix
						item.BuyCode = zIndexCodeGenerator(item.BuyShortName, item.BuyName, classifyName, "buy")
						item.BuyValue, _ = strconv.Atoi(buyVal)
						item.BuyChange, _ = strconv.Atoi(buyAddCutVal)
						item.SoldShortName = memberShortNameArr[2]
						item.SoldName = memberShortNameArr[2] + "_" + classifyName + "_" + sellSuffix
						item.SoldCode = zIndexCodeGenerator(item.SoldShortName, item.SoldName, classifyName, "sold")
						item.SoldValue, _ = strconv.Atoi(sellVal)
						item.SoldChange, _ = strconv.Atoi(sellAddCutVal)
						item.Frequency = "日度"
						item.ClassifyName = classifyName
						item.ClassifyType = classifyType
						item.CreateTime = time.Now()
						item.ModifyTime = time.Now()
						item.DataTime = tradeDate

						itemVerifyCode = item.BuyValue + item.DealValue + item.SoldValue
						if existIndex, ok := existIndexMap[item.DealName+item.BuyName+item.SoldName]; !ok {
							newID, err := models.AddBaseFromTradeZhengzhouIndex(item)
							if err != nil {
								fmt.Println("insert error:", err)
							}
							fmt.Println("insert new indexID:", newID)
						} else if existIndex != nil && itemVerifyCode != (existIndex.DealValue+existIndex.BuyValue+existIndex.SoldValue) {
							//更新
							err := models.ModifyBaseFromTradeZhengzhouIndex(item.DealValue, item.BuyValue, item.SoldValue, existIndex.BaseFromTradeZhengzhouIndexId)
							if err != nil {
								fmt.Println("data update err:", err)
							}
						}
					} else {
						//合计
						if len(classifyName) > 7 {
							classifyName = classifyName[len(classifyName)-2:]
						}
						item := new(models.BaseFromTradeZhengzhouIndex)
						item.Rank = 999
						item.DealShortName = ""
						item.DealName = "top20" + "_" + classifyName + "_" + dealSuffix
						item.DealCode = zIndexCodeGenerator("top20", item.DealName, classifyName, "deal")
						item.DealValue, _ = strconv.Atoi(dealVal)
						item.DealChange, _ = strconv.Atoi(dealAddCutVal)
						item.BuyShortName = ""
						item.BuyName = "top20" + "_" + classifyName + "_" + buySuffix
						item.BuyCode = zIndexCodeGenerator("top20", item.BuyName, classifyName, "buy")
						item.BuyValue, _ = strconv.Atoi(buyVal)
						item.BuyChange, _ = strconv.Atoi(buyAddCutVal)
						item.SoldShortName = ""
						item.SoldName = "top20" + "_" + classifyName + "_" + sellSuffix
						item.SoldCode = zIndexCodeGenerator("top20", item.SoldName, classifyName, "sold")
						item.SoldValue, _ = strconv.Atoi(sellVal)
						item.SoldChange, _ = strconv.Atoi(sellAddCutVal)
						item.Frequency = "日度"
						item.ClassifyName = classifyName
						item.ClassifyType = classifyType
						item.CreateTime = time.Now()
						item.ModifyTime = time.Now()
						item.DataTime = tradeDate

						itemVerifyCode = item.BuyValue + item.DealValue + item.SoldValue
						if existIndex, ok := existIndexMap[item.DealName+item.BuyName+item.SoldName]; !ok {
							newID, err := models.AddBaseFromTradeZhengzhouIndex(item)
							if err != nil {
								fmt.Println("insert error:", err)
							}
							fmt.Println("insert new indexID:", newID)
						} else if existIndex != nil && itemVerifyCode != (existIndex.DealValue+existIndex.BuyValue+existIndex.SoldValue) {
							//更新
							err := models.ModifyBaseFromTradeZhengzhouIndex(item.DealValue, item.BuyValue, item.SoldValue, existIndex.BaseFromTradeZhengzhouIndexId)
							if err != nil {
								fmt.Println("data update err:", err)
							}
						}
					}
				}
			}
		})
	}
	fmt.Println("end")
}