Merge branch 'master' of http://8.136.199.33:3000/hongze/hongze_data_crawler

services/commodity_trade_cffex.go

@@ -30,10 +30,10 @@ var cffexIndexCode string
 var cffexIndexCodeMap = make(map[string]string)
 var cffexActionCodeMap = make(map[string]map[string]int)
 
-func cffexIndexCodeGenerator(indexName string) string {
-	cffexIndexCode,_ := cffexIndexCodeMap[indexName]
+func cffexIndexCodeGenerator(indexName, suffix string) string {
+	cffexIndexCode, _ := cffexIndexCodeMap[indexName]
 	if cffexIndexCode == "" {
-		cffexIndexCode = fmt.Sprintf("CFFEX%s", time.Now().Format(utils.FormatDateTimeUnSpace)+strconv.Itoa(utils.GetRandInt(1, 100)))
+		cffexIndexCode = fmt.Sprintf("CFFEX%s", time.Now().Format(utils.FormatDateTimeUnSpace)+strconv.Itoa(utils.GetRandInt(1, 100))+suffix)
 		cffexIndexCodeMap[indexName] = cffexIndexCode
 		err := models.AddBaseFromTradeMapping(indexName, cffexIndexCode, "CFFEX")
 		if err != nil {
@@ -45,7 +45,7 @@ func cffexIndexCodeGenerator(indexName string) string {
 
 // SyncRankingFromCffex 中国金融期货交易所
 func SyncRankingFromCffex() {
-	for i := 180; i > 0; i-- {
+	for i := 2; i > 0; i-- {
 		zzUrl := "http://www.cffex.com.cn/sj/ccpm/%s/%s/"
 		date := time.Now().AddDate(0, 0, -i)
 		dateStr := date.Format(utils.FormatDateUnSpace)
@@ -63,7 +63,6 @@ func SyncRankingFromCffex() {
 		xmlItems[5] = GetXmlItem(zzUrl, "T")
 		//获取新的指标信息
 		var sRank string
-		existIndexMap := make(map[string]*models.BaseFromTradeCffexIndex)
 
 		//获取所有指标信息
 		allIndex, err := models.GetBaseFromTradeCffexIndexAll(dateStr)
@@ -71,11 +70,15 @@ func SyncRankingFromCffex() {
 			fmt.Println("select err:", err)
 		}
 
+		existDealMap := make(map[string]int)
+		existBuyMap := make(map[string]int)
+		existSoldMap := make(map[string]int)
 		for _, v := range allIndex {
-			indexKey := v.DealName + v.BuyName + v.SoldName + v.DataTime
-			existIndexMap[indexKey] = v
+			existDealMap[v.DealName] = v.BaseFromTradeCffexIndexId
+			existBuyMap[v.BuyName] = v.BaseFromTradeCffexIndexId
+			existSoldMap[v.SoldName] = v.BaseFromTradeCffexIndexId
 			sRank = fmt.Sprintf("%d", v.Rank)
-			cffexActionCodeMap[sRank+v.ClassifyType+v.DataTime] = map[string]int{"0": v.DealValue, "1": v.BuyValue, "2": v.SoldValue, "id": v.BaseFromTradeCffexIndexId}
+			cffexActionCodeMap[sRank+v.ClassifyType] = map[string]int{"0": v.DealValue, "1": v.BuyValue, "2": v.SoldValue, "id": v.BaseFromTradeCffexIndexId}
 		}
 		shortNameColum := map[string]string{"0": "deal_short_name", "1": "buy_short_name", "2": "sold_short_name"}
 		nameColum := map[string]string{"0": "deal_name", "1": "buy_name", "2": "sold_name"}
@@ -94,7 +97,7 @@ func SyncRankingFromCffex() {
 				case "0":
 					item.DealShortName = i.ShortName
 					item.DealName = fmt.Sprintf("%s", i.ShortName+"_成交量_"+i.ContractCode)
-					item.DealCode = cffexIndexCodeGenerator(item.DealName)
+					item.DealCode = cffexIndexCodeGenerator(item.DealName, "deal")
 					item.DealValue = i.Volume
 					item.DealChange = i.Varvolume
 
@@ -103,7 +106,7 @@ func SyncRankingFromCffex() {
 				case "1":
 					item.BuyShortName = i.ShortName
 					item.BuyName = fmt.Sprintf("%s", i.ShortName+"_持买单量_"+i.ContractCode)
-					item.BuyCode = cffexIndexCodeGenerator(item.BuyName)
+					item.BuyCode = cffexIndexCodeGenerator(item.BuyName, "buy")
 					item.BuyValue = i.Volume
 					item.BuyChange = i.Varvolume
 
@@ -112,7 +115,7 @@ func SyncRankingFromCffex() {
 				case "2":
 					item.SoldShortName = i.ShortName
 					item.SoldName = fmt.Sprintf("%s", i.ShortName+"_持卖单量_"+i.ContractCode)
-					item.SoldCode = cffexIndexCodeGenerator(item.SoldName)
+					item.SoldCode = cffexIndexCodeGenerator(item.SoldName, "sold")
 					item.SoldValue = i.Volume
 					item.SoldChange = i.Varvolume
 
@@ -130,18 +133,24 @@ func SyncRankingFromCffex() {
 
 				valueMap := map[string]int{"0": item.DealValue, "1": item.BuyValue, "2": item.SoldValue}
 				//检查在actionCodeMap中是否已经有了
-				indexKey = fmt.Sprintf("%d", item.Rank) + item.ClassifyType + item.DataTime
+				indexKey = fmt.Sprintf("%d", item.Rank) + item.ClassifyType
 				if existMap, ok := cffexActionCodeMap[indexKey]; !ok {
-					//没有，新增
-					newID, err := models.AddBaseFromTradeCffexIndex(item)
-					if err != nil {
-						fmt.Println("insert error:", err)
+					if _, ok := existDealMap[item.DealName]; !ok {
+						if _, ok := existBuyMap[item.BuyName]; !ok {
+							if _, ok := existSoldMap[item.SoldName]; !ok {
+								//没有，新增
+								newID, err := models.AddBaseFromTradeCffexIndex(item)
+								if err != nil {
+									fmt.Println("insert error:", err)
+								}
+								fmt.Println("insert new indexID:", newID)
+								existMap = make(map[string]int)
+								existMap[i.Value] = i.Volume
+								existMap["id"] = int(newID)
+								cffexActionCodeMap[indexKey] = existMap
+							}
+						}
 					}
-					fmt.Println("insert new indexID:", newID)
-					existMap = make(map[string]int)
-					existMap[i.Value] = i.Volume
-					existMap["id"] = int(newID)
-					cffexActionCodeMap[indexKey] = existMap
 				} else if existMap != nil && existMap[i.Value] != i.Volume {
 					//更新
 					var columnList = [5]string{

services/commodity_trade_ine.go

@@ -43,7 +43,7 @@ type message struct {
 var ineIndexCode string
 var ineIndexCodeMap = make(map[string]string)
 
-func IneIndexCodeGenerator(indexName,suffix  string) string {
+func IneIndexCodeGenerator(indexName,suffix string) string {
 	ineIndexCode,_ := ineIndexCodeMap[indexName]
 	if ineIndexCode == "" {
 		ineIndexCode = fmt.Sprintf("INE%s", time.Now().Format(utils.FormatDateTimeUnSpace)+strconv.Itoa(utils.GetRandInt(1, 100))+suffix)
@@ -66,7 +66,7 @@ func SyncRankingFromIne() {
 		ineIndexCodeMap[item.IndexName] = item.IndexCode
 	}
 	//获取新的指标信息
-	for i := 180; i > 0; i-- {
+	for i := 2; i > 0; i-- {
 		var message Message
 		var item = new(models.BaseFromTradeIneIndex)
 		zzUrl := "http://www.ine.com.cn/data/dailydata/kx/pm%s.dat"
@@ -89,7 +89,7 @@ func SyncRankingFromIne() {
 			fmt.Println("select err:", err)
 		}
 		for _, v := range allIndex {
-			indexKey := v.DealName + v.BuyName + v.SoldName + tradeDate
+			indexKey := v.DealName + v.BuyName + v.SoldName
 			existIndexMap[indexKey] = v
 			ineIndexCodeMap[v.BuyName] = v.BuyCode
 			ineIndexCodeMap[v.SoldName] = v.SoldCode
@@ -124,7 +124,7 @@ func SyncRankingFromIne() {
 				item.DataTime = tradeDate
 
 				itemVerifyCode = item.BuyValue + item.DealValue + item.SoldValue
-				if existIndex, ok := existIndexMap[item.DealName+item.BuyName+item.SoldName+tradeDate]; !ok {
+				if existIndex, ok := existIndexMap[item.DealName+item.BuyName+item.SoldName]; !ok {
 					newID, err := models.AddBaseFromTradeIneIndex(item)
 					if err != nil {
 						fmt.Println("insert error:", err)

services/commodity_trade_shanghai.go

@@ -43,10 +43,10 @@ type Message struct {
 var indexCode string
 var indexCodeMap = make(map[string]string)
 
-func shIndexCodeGenerator(indexName string) string {
+func shIndexCodeGenerator(indexName,suffix string) string {
 	indexCode,_ := indexCodeMap[indexName]
 	if indexCode == "" {
-		indexCode = fmt.Sprintf("SH%s", time.Now().Format(utils.FormatDateTimeUnSpace)+strconv.Itoa(utils.GetRandInt(1, 100)))
+		indexCode = fmt.Sprintf("SH%s", time.Now().Format(utils.FormatDateTimeUnSpace)+strconv.Itoa(utils.GetRandInt(1, 100))+suffix)
 		indexCodeMap[indexName] = indexCode
 		err := models.AddBaseFromTradeMapping(indexName, indexCode, "SH")
 		if err != nil {
@@ -59,7 +59,7 @@ func shIndexCodeGenerator(indexName string) string {
 // SyncRankingFromShangHai 上海商品交易所持单排名
 func SyncRankingFromShangHai() {
 	//获取新的指标信息
-	for i := 180; i > 0; i-- {
+	for i := 2; i > 0; i-- {
 		var message Message
 		var item = new(models.BaseFromTradeShanghaiIndex)
 		zzUrl := "http://www.shfe.com.cn/data/dailydata/kx/pm%s.dat"
@@ -83,7 +83,7 @@ func SyncRankingFromShangHai() {
 
 		existIndexMap := make(map[string]*models.BaseFromTradeShanghaiIndex)
 		for _, v := range allIndex {
-			indexKey := v.DealName + v.BuyName + v.SoldName + v.DataTime
+			indexKey := v.DealName + v.BuyName + v.SoldName
 			existIndexMap[indexKey] = v
 		}
 		var itemVerifyCode int
@@ -98,9 +98,9 @@ func SyncRankingFromShangHai() {
 				item.DealName = strings.Replace(fmt.Sprintf("%s", p.ParticipantName1+"_成交量"+"_"+p.ProductName+"_"+p.ContractCode), " ", "", -1)
 				item.BuyName = strings.Replace(fmt.Sprintf("%s", p.ParticipantName2+"_持买单量"+"_"+p.ProductName+"_"+p.ContractCode), " ", "", -1)
 				item.SoldName = strings.Replace(fmt.Sprintf("%s", p.ParticipantName3+"_持卖单量"+"_"+p.ProductName+"_"+p.ContractCode), " ", "", -1)
-				item.DealCode =  shIndexCodeGenerator(item.DealName)
-				item.BuyCode =  shIndexCodeGenerator(item.BuyName)
-				item.SoldCode =  shIndexCodeGenerator(item.SoldCode)
+				item.DealCode =  shIndexCodeGenerator(item.DealName, "deal")
+				item.BuyCode =  shIndexCodeGenerator(item.BuyName, "buy")
+				item.SoldCode =  shIndexCodeGenerator(item.SoldCode, "sold")
 				item.DealValue = p.Deal
 				item.DealChange = p.Change1
 				item.BuyValue = p.BuyIn
@@ -115,7 +115,7 @@ func SyncRankingFromShangHai() {
 				item.DataTime = tradeDate
 
 				itemVerifyCode = item.BuyValue + item.DealValue + item.SoldValue
-				if existIndex, ok := existIndexMap[item.DealName + item.BuyName + item.SoldName + item.DataTime]; !ok {
+				if existIndex, ok := existIndexMap[item.DealName + item.BuyName + item.SoldName]; !ok {
 					newID, err := models.AddBaseFromTradeShangHaiIndex(item)
 					if err != nil {
 						fmt.Println("insert error:", err)

services/commodity_trade_zhengzhou.go

@@ -17,10 +17,10 @@ var zIndexCode string
 var zIndexCodeMap = make(map[string]string)
 var zActionCodeMap = make(map[string]map[string]int)
 
-func zIndexCodeGenerator(indexName string) string {
+func zIndexCodeGenerator(indexName,suffix string) string {
 	zIndexCode,_ := zIndexCodeMap[indexName]
 	if zIndexCode == "" {
-		zIndexCode = fmt.Sprintf("Z%s", time.Now().Format(utils.FormatDateTimeUnSpace)+strconv.Itoa(utils.GetRandInt(1, 100)))
+		zIndexCode = fmt.Sprintf("Z%s", time.Now().Format(utils.FormatDateTimeUnSpace)+strconv.Itoa(utils.GetRandInt(1, 100))+suffix)
 		zIndexCodeMap[indexName] = zIndexCode
 		err := models.AddBaseFromTradeMapping(indexName, zIndexCode, "Z")
 		if err != nil {
@@ -40,7 +40,7 @@ func SyncRankingFromZhengzhou() {
 		}
 	}()
 
-	for i := 180; i > 0; i-- {
+	for i := 2; i > 0; i-- {
 		var itemVerifyCode int
 		zzUrl := "http://www.czce.com.cn/cn/DFSStaticFiles/Future/%s/%s/FutureDataHolding.htm"
 		date := time.Now().AddDate(0, 0, -i)
@@ -193,17 +193,17 @@ func SyncRankingFromZhengzhou() {
 						item.Rank, _ = strconv.Atoi(rank)
 						item.DealShortName = memberShortNameArr[0]
 						item.DealName = memberShortNameArr[0] + "_" + dealSuffix + "_" + classifyName + "_" + classifyType
-						item.DealCode = zIndexCodeGenerator(item.DealName)
+						item.DealCode = zIndexCodeGenerator(item.DealName, "deal")
 						item.DealValue, _ = strconv.Atoi(dealVal)
 						item.DealChange, _ = strconv.Atoi(dealAddCutVal)
 						item.BuyShortName = memberShortNameArr[1]
 						item.BuyName = memberShortNameArr[1] + "_" + buySuffix + "_" + classifyName + "_" + classifyType
-						item.BuyCode = zIndexCodeGenerator(item.BuyName)
+						item.BuyCode = zIndexCodeGenerator(item.BuyName, "buy")
 						item.BuyValue, _ = strconv.Atoi(buyVal)
 						item.BuyChange, _ = strconv.Atoi(buyAddCutVal)
 						item.SoldShortName = memberShortNameArr[2]
 						item.SoldName = memberShortNameArr[2] + "_" + sellSuffix + "_" + classifyName + "_" + classifyType
-						item.SoldCode = zIndexCodeGenerator(item.SoldName)
+						item.SoldCode = zIndexCodeGenerator(item.SoldName, "sold")
 						item.SoldValue, _ = strconv.Atoi(sellVal)
 						item.SoldChange, _ = strconv.Atoi(sellAddCutVal)
 						item.Frequency = "日度"