package services import ( "eta/eta_crawler/models" "eta/eta_crawler/utils" "fmt" "github.com/mozillazg/go-pinyin" "log" "strconv" "strings" "time" "github.com/PuerkitoBio/goquery" ) var zIndexCode string var zIndexCodeMap = make(map[string]string) var zActionCodeMap = make(map[string]map[string]int) func zIndexCodeGenerator(shortName, indexName, contractCode, suffix string) string { if shortName == "" || shortName == "-" { return "" } strResult := "" if shortName != "top20" { //取公司全拼 a := pinyin.NewArgs() rows := pinyin.LazyPinyin(shortName, a) for i := 0; i < len(rows); i++ { strResult += rows[i] } } else { strResult = "top20" } //郑州特殊处理,当合约号有中文时只取英文代号 如 苹果AP ---> AP if len(contractCode) > 7 { contractCode = contractCode[len(contractCode)-2:] } zIndexCode, _ := zIndexCodeMap[indexName] if zIndexCode == "" { zIndexCode = strResult + contractCode + suffix zIndexCodeMap[indexName] = zIndexCode err := models.AddBaseFromTradeMapping(indexName, zIndexCode, "Z") if err != nil { fmt.Println("add Code err:", err) } } return zIndexCode } // 郑州商品交易所持单排名 func SyncRankingFromZhengzhou() { utils.FileLog.Info("SyncRankingFromZhengzhou start:" + time.Now().Format(utils.FormatDateTime)) n := utils.GetRandInt(10, 120) time.Sleep(time.Duration(n) * time.Second) fmt.Println("start") var err error defer func() { if err != nil { fmt.Println("Err:" + err.Error()) } }() allCode, err := models.GetIndexCodeFromMapping("Z") if err != nil { fmt.Println("select Code err:", err) } for _, item := range allCode { zIndexCodeMap[item.IndexName] = item.IndexCode } for i := 2; i >= 0; i-- { var itemVerifyCode int zzUrl := "http://www.czce.com.cn/cn/DFSStaticFiles/Future/%s/%s/FutureDataHolding.htm" date := time.Now().AddDate(0, 0, -i) year := date.Year() dateStr := date.Format(utils.FormatDateUnSpace) zzUrl = fmt.Sprintf(zzUrl, strconv.Itoa(year), dateStr) fmt.Println(zzUrl) bodyStr, err := getSourceZhengZhou(zzUrl) if err != nil { fmt.Println("GetData Err:" + err.Error()) return } //获取所有指标信息 某一天的 allIndex, err := models.GetBaseFromTradeZhengzhouIndexAll(dateStr) if err != nil { return } existIndexMap := make(map[string]*models.BaseFromTradeZhengzhouIndex) for _, v := range allIndex { indexKey := v.DealName + v.BuyName + v.SoldName existIndexMap[indexKey] = v } utils.FileLog.Info(bodyStr) if strings.Contains(bodyStr, "出错") { continue } doc, err := goquery.NewDocumentFromReader(strings.NewReader(bodyStr)) if err != nil { log.Fatal(err) } table := doc.Find("table") var classifyName, classifyType, tradeDate string var dealSuffix, buySuffix, sellSuffix string var dealVal, dealAddCutVal, buyVal, buyAddCutVal, sellVal, sellAddCutVal string var rank string table.Find("tr").Each(func(i int, tr *goquery.Selection) { var memberShortNameArr []string tds := tr.Find("td") if tds.Length() == 1 { tdText := tds.Text() utils.FileLog.Info(tdText) if tdText != "" { tdTextArr := strings.Split(tdText, "    ") for k, v := range tdTextArr { //fmt.Println(k, v) if k == 0 { classifyName = v } else { tradeDate = v } } } } else { tds.Each(func(tk int, td *goquery.Selection) { tdText := td.Text() tdText = strings.Trim(tdText, " ") if tdText == "" { return } //fmt.Println("tdText: ", tdText) if tk == 0 { if !(strings.Contains(tdText, "名次") || strings.Contains(tdText, "合计")) { rank = tdText } } if tk == 1 { if !strings.Contains(tdText, "会员简称") { tdText = strings.Replace(tdText, "(代客)", "", -1) memberShortNameArr = append(memberShortNameArr, tdText) } } if tk == 4 { if !strings.Contains(tdText, "会员简称") { tdText = strings.Replace(tdText, "(代客)", "", -1) memberShortNameArr = append(memberShortNameArr, tdText) } } if tk == 7 { if !strings.Contains(tdText, "会员简称") { tdText = strings.Replace(tdText, "(代客)", "", -1) memberShortNameArr = append(memberShortNameArr, tdText) } } if tk == 2 { if strings.Contains(tdText, "成交量") { dealSuffix = tdText } else { dealVal = tdText } } if tk == 3 { if strings.Contains(tdText, "增减量") { //dealAddCutSuffix = tdText } else { dealAddCutVal = tdText } } if tk == 5 { if strings.Contains(tdText, "持买仓量") { buySuffix = tdText } else { buyVal = tdText } } if tk == 6 { if strings.Contains(tdText, "增减量") { //buyAddCutSuffix = tdText } else { buyAddCutVal = tdText } } if tk == 8 { if strings.Contains(tdText, "持卖仓量") { sellSuffix = tdText } else { sellVal = tdText } } if tk == 9 { if strings.Contains(tdText, "增减量") { //sellAddCutSuffix = tdText } else { sellAddCutVal = tdText } } }) //处理指标 dealVal = strings.Replace(dealVal, ",", "", -1) dealAddCutVal = strings.Replace(dealAddCutVal, ",", "", -1) buyVal = strings.Replace(buyVal, ",", "", -1) buyAddCutVal = strings.Replace(buyAddCutVal, ",", "", -1) sellVal = strings.Replace(sellVal, ",", "", -1) sellAddCutVal = strings.Replace(sellAddCutVal, ",", "", -1) tradeDate = strings.Replace(tradeDate, "日期:", "", -1) classifyNameArr := strings.Split(classifyName, ":") if len(classifyNameArr) > 1 { classifyType = classifyNameArr[0] classifyName = classifyNameArr[1] } //fmt.Printf("memberShortNameArr:%s,len:%d\n", memberShortNameArr, len(memberShortNameArr)) if len(memberShortNameArr) != 0 { if len(memberShortNameArr[0]) != 2 { if len(classifyName) > 7 { classifyName = classifyName[len(classifyName)-2:] } item := new(models.BaseFromTradeZhengzhouIndex) item.Rank, _ = strconv.Atoi(rank) item.DealShortName = memberShortNameArr[0] item.DealName = memberShortNameArr[0] + "_" + classifyName + "_" + dealSuffix item.DealCode = zIndexCodeGenerator(item.DealShortName, item.DealName, classifyName, "deal") item.DealValue, _ = strconv.Atoi(dealVal) item.DealChange, _ = strconv.Atoi(dealAddCutVal) item.BuyShortName = memberShortNameArr[1] item.BuyName = memberShortNameArr[1] + "_" + classifyName + "_" + buySuffix item.BuyCode = zIndexCodeGenerator(item.BuyShortName, item.BuyName, classifyName, "buy") item.BuyValue, _ = strconv.Atoi(buyVal) item.BuyChange, _ = strconv.Atoi(buyAddCutVal) item.SoldShortName = memberShortNameArr[2] item.SoldName = memberShortNameArr[2] + "_" + classifyName + "_" + sellSuffix item.SoldCode = zIndexCodeGenerator(item.SoldShortName, item.SoldName, classifyName, "sold") item.SoldValue, _ = strconv.Atoi(sellVal) item.SoldChange, _ = strconv.Atoi(sellAddCutVal) item.Frequency = "日度" item.ClassifyName = classifyName item.ClassifyType = classifyType item.CreateTime = time.Now() item.ModifyTime = time.Now() item.DataTime = tradeDate itemVerifyCode = item.BuyValue + item.DealValue + item.SoldValue if existIndex, ok := existIndexMap[item.DealName+item.BuyName+item.SoldName]; !ok { newID, err := models.AddBaseFromTradeZhengzhouIndex(item) if err != nil { fmt.Println("insert error:", err) } existIndexMap[item.DealName+item.BuyName+item.SoldName] = item fmt.Println("insert new indexID:", newID) } else if existIndex != nil && itemVerifyCode != (existIndex.DealValue+existIndex.BuyValue+existIndex.SoldValue) { //更新 err := models.ModifyBaseFromTradeZhengzhouIndex(item.DealValue, item.BuyValue, item.SoldValue, existIndex.BaseFromTradeZhengzhouIndexId) if err != nil { fmt.Println("data update err:", err) } } } else { //合计 if len(classifyName) > 7 { classifyName = classifyName[len(classifyName)-2:] } item := new(models.BaseFromTradeZhengzhouIndex) item.Rank = 999 item.DealShortName = "" item.DealName = "top20" + "_" + classifyName + "_" + dealSuffix item.DealCode = zIndexCodeGenerator("top20", item.DealName, classifyName, "deal") item.DealValue, _ = strconv.Atoi(dealVal) item.DealChange, _ = strconv.Atoi(dealAddCutVal) item.BuyShortName = "" item.BuyName = "top20" + "_" + classifyName + "_" + buySuffix item.BuyCode = zIndexCodeGenerator("top20", item.BuyName, classifyName, "buy") item.BuyValue, _ = strconv.Atoi(buyVal) item.BuyChange, _ = strconv.Atoi(buyAddCutVal) item.SoldShortName = "" item.SoldName = "top20" + "_" + classifyName + "_" + sellSuffix item.SoldCode = zIndexCodeGenerator("top20", item.SoldName, classifyName, "sold") item.SoldValue, _ = strconv.Atoi(sellVal) item.SoldChange, _ = strconv.Atoi(sellAddCutVal) item.Frequency = "日度" item.ClassifyName = classifyName item.ClassifyType = classifyType item.CreateTime = time.Now() item.ModifyTime = time.Now() item.DataTime = tradeDate itemVerifyCode = item.BuyValue + item.DealValue + item.SoldValue if existIndex, ok := existIndexMap[item.DealName+item.BuyName+item.SoldName]; !ok { newID, err := models.AddBaseFromTradeZhengzhouIndex(item) if err != nil { fmt.Println("insert error:", err) } fmt.Println("insert new indexID:", newID) existIndexMap[item.DealName+item.BuyName+item.SoldName] = item } else if existIndex != nil && itemVerifyCode != (existIndex.DealValue+existIndex.BuyValue+existIndex.SoldValue) { //更新 err := models.ModifyBaseFromTradeZhengzhouIndex(item.DealValue, item.BuyValue, item.SoldValue, existIndex.BaseFromTradeZhengzhouIndexId) if err != nil { fmt.Println("data update err:", err) } } } } } }) } fmt.Println("end") utils.FileLog.Info("SyncRankingFromZhengzhou end:" + time.Now().Format(utils.FormatDateTime)) }