eta_crawler/services/commodity_trade_zhengzhou.go

package services

import (
	"fmt"
	"hongze/hongze_data_crawler/models"
	"hongze/hongze_data_crawler/utils"
	"log"
	"rdluck_tools/http"
	"strconv"
	"strings"
	"time"

	"github.com/PuerkitoBio/goquery"
)

var zIndexCode string
var zIndexCodeMap = make(map[string]string)
var zActionCodeMap = make(map[string]map[string]int)

func zIndexCodeGenerator(indexName string) string {
	zIndexCode,_ := zIndexCodeMap[indexName]
	if zIndexCode == "" {
		zIndexCode = fmt.Sprintf("Z%s", time.Now().Format(utils.FormatDateTimeUnSpace)+strconv.Itoa(utils.GetRandInt(1, 100)))
		zIndexCodeMap[indexName] = zIndexCode
		err := models.AddBaseFromTradeMapping(indexName, zIndexCode, "Z")
		if err != nil {
			fmt.Println("add Code err:", err)
		}
	}
	return zIndexCode
}

//郑州商品交易所持单排名
func SyncRankingFromZhengzhou() {
	fmt.Println("start")
	var err error
	defer func() {
		if err != nil {
			fmt.Println("Err:" + err.Error())
		}
	}()

	for i := 180; i > 0; i-- {
		var itemVerifyCode int
		zzUrl := "http://www.czce.com.cn/cn/DFSStaticFiles/Future/%s/%s/FutureDataHolding.htm"
		date := time.Now().AddDate(0, 0, -i)
		year := date.Year()
		dateStr := date.Format(utils.FormatDateUnSpace)
		zzUrl = fmt.Sprintf(zzUrl, strconv.Itoa(year), dateStr)
		fmt.Println(zzUrl)
		body, err := http.Get(zzUrl)
		if err != nil {
			fmt.Println("GetData Err:" + err.Error())
			return
		}
		//获取所有指标信息  某一天的
		allIndex, err := models.GetBaseFromTradeZhengzhouIndexAll(dateStr)
		if err != nil {
			return
		}

		existIndexMap := make(map[string]*models.BaseFromTradeZhengzhouIndex)
		for _, v := range allIndex {
			indexKey := v.DealName + v.BuyName + v.SoldName
			existIndexMap[indexKey] = v
		}
		bodyStr := string(body)
		utils.FileLog.Info(bodyStr)
		if strings.Contains(bodyStr, "404.htm") {
			continue
		}
		doc, err := goquery.NewDocumentFromReader(strings.NewReader(bodyStr))
		if err != nil {
			log.Fatal(err)
		}
		table := doc.Find("table")
		var classifyName, classifyType, tradeDate string
		var dealSuffix, dealAddCutSuffix, buySuffix, buyAddCutSuffix, sellSuffix, sellAddCutSuffix string
		var dealVal, dealAddCutVal, buyVal, buyAddCutVal, sellVal, sellAddCutVal string
		var rank string
		table.Find("tr").Each(func(i int, tr *goquery.Selection) {
			var memberShortNameArr []string
			tds := tr.Find("td")
			if tds.Length() == 1 {
				tdText := tds.Text()
				utils.FileLog.Info(tdText)
				if tdText != "" {
					tdTextArr := strings.Split(tdText, "    ")
					for k, v := range tdTextArr {
						//fmt.Println(k, v)
						if k == 0 {
							classifyName = v
						} else {
							tradeDate = v
						}
					}
				}
			} else {
				tds.Each(func(tk int, td *goquery.Selection) {
					tdText := td.Text()
					tdText = strings.Trim(tdText, " ")
					if tdText == "" {
						return
					}
					//fmt.Println("tdText: ", tdText)
					if tk == 0 {
						if !(strings.Contains(tdText, "名次") || strings.Contains(tdText, "合计")){
							rank = tdText
						}
					}

					if tk == 1 {
						if !strings.Contains(tdText, "会员简称") {
							memberShortNameArr = append(memberShortNameArr, tdText)
						}
					}

					if tk == 4 {
						if !strings.Contains(tdText, "会员简称") {
							memberShortNameArr = append(memberShortNameArr, tdText)
						}
					}

					if tk == 7 {
						if !strings.Contains(tdText, "会员简称") {
							memberShortNameArr = append(memberShortNameArr, tdText)
						}
					}

					if tk == 2 {
						if strings.Contains(tdText, "成交量") {
							dealSuffix = tdText
						} else {
							dealVal = tdText
						}
					}
					if tk == 3 {
						if strings.Contains(tdText, "增减量") {
							dealAddCutSuffix = tdText
						} else {
							dealAddCutVal = tdText
						}
					}
					if tk == 5 {
						if strings.Contains(tdText, "持买仓量") {
							buySuffix = tdText
						} else {
							buyVal = tdText
						}
					}
					if tk == 6 {
						if strings.Contains(tdText, "增减量") {
							buyAddCutSuffix = tdText
						} else {
							buyAddCutVal = tdText
						}
					}
					if tk == 8 {
						if strings.Contains(tdText, "持卖仓量") {
							sellSuffix = tdText
						} else {
							sellVal = tdText
						}
					}
					if tk == 9 {
						if strings.Contains(tdText, "增减量") {
							sellAddCutSuffix = tdText
						} else {
							sellAddCutVal = tdText
						}
					}
				})

				//处理指标
				dealVal = strings.Replace(dealVal, ",", "", -1)
				dealAddCutVal = strings.Replace(dealAddCutVal, ",", "", -1)
				buyVal = strings.Replace(buyVal, ",", "", -1)
				buyAddCutVal = strings.Replace(buyAddCutVal, ",", "", -1)
				sellVal = strings.Replace(sellVal, ",", "", -1)
				sellAddCutVal = strings.Replace(sellAddCutVal, ",", "", -1)
				tradeDate = strings.Replace(tradeDate, "日期：", "", -1)

				classifyNameArr := strings.Split(classifyName, "：")

				if len(classifyNameArr) > 1 {
					classifyType = classifyNameArr[0]
					classifyName = classifyNameArr[1]
				}
				//fmt.Printf("memberShortNameArr:%s,len:%d\n", memberShortNameArr, len(memberShortNameArr))
				if len(memberShortNameArr) != 0 {
					if len(memberShortNameArr[0]) != 2 {
						item := new(models.BaseFromTradeZhengzhouIndex)
						item.Rank, _ = strconv.Atoi(rank)
						item.DealShortName = memberShortNameArr[0]
						item.DealName = memberShortNameArr[0] + "_" + dealSuffix + "_" + classifyName + "_" + classifyType
						item.DealCode = zIndexCodeGenerator(item.DealName)
						item.DealValue, _ = strconv.Atoi(dealVal)
						item.DealChange, _ = strconv.Atoi(dealAddCutVal)
						item.BuyShortName = memberShortNameArr[1]
						item.BuyName = memberShortNameArr[1] + "_" + buySuffix + "_" + classifyName + "_" + classifyType
						item.BuyCode = zIndexCodeGenerator(item.BuyName)
						item.BuyValue, _ = strconv.Atoi(buyVal)
						item.BuyChange, _ = strconv.Atoi(buyAddCutVal)
						item.SoldShortName = memberShortNameArr[2]
						item.SoldName = memberShortNameArr[2] + "_" + sellSuffix + "_" + classifyName + "_" + classifyType
						item.SoldCode = zIndexCodeGenerator(item.SoldName)
						item.SoldValue, _ = strconv.Atoi(sellVal)
						item.SoldChange, _ = strconv.Atoi(sellAddCutVal)
						item.Frequency = "日度"
						item.ClassifyName = classifyName
						item.ClassifyType = classifyType
						item.CreateTime = time.Now()
						item.ModifyTime = time.Now()
						item.DataTime = tradeDate

						itemVerifyCode = item.BuyValue + item.DealValue + item.SoldValue
						if existIndex, ok := existIndexMap[item.DealName + item.BuyName + item.SoldName]; !ok {
							newID, err := models.AddBaseFromTradeZhengzhouIndex(item)
							if err != nil {
								fmt.Println("insert error:", err)
							}
							fmt.Println("insert new indexID:", newID)
						} else if existIndex != nil && itemVerifyCode != (existIndex.DealValue+existIndex.BuyValue+existIndex.SoldValue) {
							//更新
							err := models.ModifyBaseFromTradeZhengzhouIndex(item.DealValue, item.BuyValue, item.SoldValue, existIndex.BaseFromTradeZhengzhouIndexId)
							if err != nil {
								fmt.Println("data update err:", err)
							}
						}
					}
				}
			}
		})
	}
	fmt.Println("end")
}