indexCode生成方式修改

models/base_from_trade_ine.go

@@ -31,12 +31,6 @@ type BaseFromTradeIneIndex struct {
 	DataTime                string
 }
 
-type BaseFromTradeMapping struct {
-	BaseFromTradeMappingId int `json:"column(base_from_trade_mapping_id);pk"`
-	IndexName              string
-	IndexCode              string
-	Exchange               string
-}
 
 func AddBaseFromTradeIneIndex(item *BaseFromTradeIneIndex) (lastId int64, err error) {
 	o := orm.NewOrm()

models/db.go

@@ -31,7 +31,8 @@ func init() {
 		new(BaseFromTradeShanghaiIndex),
 		new(BaseFromTradeIneIndex),
 		new(BaseFromTradeCffexIndex),
-		//new(BaseFromEicIndex),
+		new(BaseFromTradeEicIndex),
+		new(BaseFromTradeMapping),
 		//
 		new(BaseFromTradeDalianIndex),
 		//new(BaseFromTradeDalianData),

services/commodity_trade_cffex.go

@@ -3,10 +3,10 @@ package services
 import (
 	"encoding/xml"
 	"fmt"
+	"github.com/mozillazg/go-pinyin"
 	"hongze/hongze_data_crawler/models"
 	"hongze/hongze_data_crawler/utils"
 	"rdluck_tools/http"
-	"strconv"
 	"strings"
 	"time"
 )
@@ -31,14 +31,23 @@ var cffexIndexCode string
 var cffexIndexCodeMap = make(map[string]string)
 var cffexActionCodeMap = make(map[string]map[string]int)
 
-func cffexIndexCodeGenerator(shortName, indexName, suffix string) string {
-	if shortName == ""{
+func cffexIndexCodeGenerator(shortName, indexName, contractCode, suffix string) string {
+	if shortName == "" {
 		cffexIndexCode = ""
 		return cffexIndexCode
 	}
+	//取公司前两个字的全拼
+	a := pinyin.NewArgs()
+	rows := pinyin.Pinyin(shortName, a)
+	strResult := ""
+	for i := 0; i < len(rows[:2]); i++ {
+		if len(rows[i]) != 0 {
+			strResult += rows[i][0]
+		}
+	}
 	cffexIndexCode, _ := cffexIndexCodeMap[indexName]
 	if cffexIndexCode == "" {
-		cffexIndexCode = fmt.Sprintf("CFFEX%s", strconv.FormatInt(time.Now().UnixNano(), 10)+suffix)
+		cffexIndexCode = strResult + contractCode + suffix
 		cffexIndexCodeMap[indexName] = cffexIndexCode
 		err := models.AddBaseFromTradeMapping(indexName, cffexIndexCode, "CFFEX")
 		if err != nil {
@@ -115,7 +124,7 @@ func SyncRankingFromCffex() {
 				case "0":
 					item.DealShortName = i.ShortName
 					item.DealName = fmt.Sprintf("%s", i.ShortName+"_成交量_"+i.ContractCode)
-					item.DealCode = cffexIndexCodeGenerator(item.DealShortName, item.DealName, "deal")
+					item.DealCode = cffexIndexCodeGenerator(item.DealShortName, item.DealName, i.ContractCode, "deal")
 					item.DealValue = i.Volume
 					item.DealChange = i.Varvolume
 
@@ -124,7 +133,7 @@ func SyncRankingFromCffex() {
 				case "1":
 					item.BuyShortName = i.ShortName
 					item.BuyName = fmt.Sprintf("%s", i.ShortName+"_持买单量_"+i.ContractCode)
-					item.BuyCode = cffexIndexCodeGenerator(item.BuyShortName, item.BuyName, "buy")
+					item.BuyCode = cffexIndexCodeGenerator(item.BuyShortName, item.BuyName, i.ContractCode, "buy")
 					item.BuyValue = i.Volume
 					item.BuyChange = i.Varvolume
 
@@ -133,7 +142,7 @@ func SyncRankingFromCffex() {
 				case "2":
 					item.SoldShortName = i.ShortName
 					item.SoldName = fmt.Sprintf("%s", i.ShortName+"_持卖单量_"+i.ContractCode)
-					item.SoldCode = cffexIndexCodeGenerator(item.SoldShortName, item.SoldName, "sold")
+					item.SoldCode = cffexIndexCodeGenerator(item.SoldShortName, item.SoldName, i.ContractCode, "sold")
 					item.SoldValue = i.Volume
 					item.SoldChange = i.Varvolume
 

services/commodity_trade_ine.go

@@ -3,10 +3,10 @@ package services
 import (
 	"encoding/json"
 	"fmt"
+	"github.com/mozillazg/go-pinyin"
 	"hongze/hongze_data_crawler/models"
 	"hongze/hongze_data_crawler/utils"
 	"rdluck_tools/http"
-	"strconv"
 	"strings"
 	"time"
 )
@@ -43,14 +43,23 @@ type message struct {
 var ineIndexCode string
 var ineIndexCodeMap = make(map[string]string)
 
-func IneIndexCodeGenerator(shortName, indexName,suffix string) string {
+func IneIndexCodeGenerator(shortName, indexName, contractCode, suffix string) string {
 	if shortName == ""{
-		ineIndexCode = ""
-		return ineIndexCode
+		indexCode = ""
+		return indexCode
+	}
+	//取公司前两个字的全拼
+	a := pinyin.NewArgs()
+	rows := pinyin.Pinyin(shortName, a)
+	strResult := ""
+	for i := 0; i < len(rows[:2]); i++ {
+		if len(rows[i]) != 0 {
+			strResult += rows[i][0]
+		}
 	}
 	ineIndexCode,_ := ineIndexCodeMap[indexName]
 	if ineIndexCode == "" {
-		ineIndexCode = fmt.Sprintf("INE%s", strconv.FormatInt(time.Now().UnixNano(), 10)+suffix)
+		ineIndexCode = strResult + contractCode + suffix
 		ineIndexCodeMap[indexName] = ineIndexCode
 		err := models.AddBaseFromTradeMapping(indexName, ineIndexCode, "INE")
 		if err != nil {
@@ -72,7 +81,7 @@ func SyncRankingFromIne() {
 		ineIndexCodeMap[item.IndexName] = item.IndexCode
 	}
 	//获取新的指标信息
-	for i := 5; i > 0; i-- {
+	for i := 180; i > 0; i-- {
 		var message Message
 		var item = new(models.BaseFromTradeIneIndex)
 		zzUrl := "http://www.ine.com.cn/data/dailydata/kx/pm%s.dat"
@@ -110,9 +119,9 @@ func SyncRankingFromIne() {
 				item.DealName = strings.Replace(fmt.Sprintf("%s", p.ParticipantName1+"_成交量"+"_"+p.ProductName+"_"+p.ContractCode), " ", "", -1)
 				item.BuyName = strings.Replace(fmt.Sprintf("%s", p.ParticipantName2+"_持买单量"+"_"+p.ProductName+"_"+p.ContractCode), " ", "", -1)
 				item.SoldName = strings.Replace(fmt.Sprintf("%s", p.ParticipantName3+"_持卖单量"+"_"+p.ProductName+"_"+p.ContractCode), " ", "", -1)
-				item.DealCode = IneIndexCodeGenerator(item.DealShortName, item.DealName, "deal")
-				item.BuyCode = IneIndexCodeGenerator(item.BuyShortName, item.BuyName, "buy")
-				item.SoldCode = IneIndexCodeGenerator(item.SoldShortName, item.SoldName, "sold")
+				item.DealCode = IneIndexCodeGenerator(item.DealShortName, item.DealName, p.ContractCode,  "deal")
+				item.BuyCode = IneIndexCodeGenerator(item.BuyShortName, item.BuyName, p.ContractCode, "buy")
+				item.SoldCode = IneIndexCodeGenerator(item.SoldShortName, item.SoldName, p.ContractCode, "sold")
 				item.DealValue = p.Deal
 				item.DealChange = p.Change1
 				item.BuyValue = p.BuyIn

services/commodity_trade_shanghai.go

@@ -3,10 +3,10 @@ package services
 import (
 	"encoding/json"
 	"fmt"
+	"github.com/mozillazg/go-pinyin"
 	"hongze/hongze_data_crawler/models"
 	"hongze/hongze_data_crawler/utils"
 	"rdluck_tools/http"
-	"strconv"
 	"strings"
 	"time"
 )
@@ -43,14 +43,23 @@ type Message struct {
 var indexCode string
 var indexCodeMap = make(map[string]string)
 
-func shIndexCodeGenerator(shortName, indexName,suffix string) string {
+func shIndexCodeGenerator(shortName, indexName, contractCode, suffix string) string {
 	if shortName == ""{
 		indexCode = ""
 		return indexCode
 	}
+	//取公司前两个字的全拼
+	a := pinyin.NewArgs()
+	rows := pinyin.Pinyin(shortName, a)
+	strResult := ""
+	for i := 0; i < len(rows[:2]); i++ {
+		if len(rows[i]) != 0 {
+			strResult += rows[i][0]
+		}
+	}
 	indexCode,_ := indexCodeMap[indexName]
 	if indexCode == "" {
-		indexCode = fmt.Sprintf("SH%s", strconv.FormatInt(time.Now().UnixNano(), 10)+suffix)
+		indexCode = strResult + contractCode + suffix
 		indexCodeMap[indexName] = indexCode
 		err := models.AddBaseFromTradeMapping(indexName, indexCode, "SH")
 		if err != nil {
@@ -98,50 +107,49 @@ func SyncRankingFromShangHai() {
 		for _, v := range allIndex {
 			indexKey := v.DealName + v.BuyName + v.SoldName
 			existIndexMap[indexKey] = v
-			indexCodeMap[v.DealName] = v.DealCode
-			indexCodeMap[v.BuyName] = v.BuyCode
-			indexCodeMap[v.SoldName] = v.SoldCode
 		}
 		var itemVerifyCode int
 		//处理指标
 		for _, p := range position {
 			if p.Rank > 0 && p.Rank <40 && p.ParticipantName1 != "" {
-				//成交量
-				item.Rank = p.Rank
-				item.DealShortName = p.ParticipantName1
-				item.BuyShortName = p.ParticipantName2
-				item.SoldShortName = p.ParticipantName3
-				item.DealName = strings.Replace(fmt.Sprintf("%s", p.ParticipantName1+"_成交量"+"_"+p.ProductName+"_"+p.ContractCode), " ", "", -1)
-				item.BuyName = strings.Replace(fmt.Sprintf("%s", p.ParticipantName2+"_持买单量"+"_"+p.ProductName+"_"+p.ContractCode), " ", "", -1)
-				item.SoldName = strings.Replace(fmt.Sprintf("%s", p.ParticipantName3+"_持卖单量"+"_"+p.ProductName+"_"+p.ContractCode), " ", "", -1)
-				item.DealCode =  shIndexCodeGenerator(item.DealShortName, item.DealName, "deal")
-				item.BuyCode =  shIndexCodeGenerator(item.BuyShortName, item.BuyName, "buy")
-				item.SoldCode =  shIndexCodeGenerator(item.SoldShortName, item.SoldCode, "sold")
-				item.DealValue = p.Deal
-				item.DealChange = p.Change1
-				item.BuyValue = p.BuyIn
-				item.BuyChange = p.Change2
-				item.SoldValue = p.SoldOut
-				item.SoldChange = p.Change3
-				item.ClassifyName = strings.Replace(p.ProductName, " ", "", -1)
-				item.ClassifyType = strings.Replace(p.ContractCode, " ", "", -1)
-				item.Frequency = "日度"
-				item.CreateTime = time.Now()
-				item.ModifyTime = time.Now()
-				item.DataTime = tradeDate
+				if strings.Replace(p.ProductName, " ", "", -1) != "20号胶"{
+					//成交量
+					item.Rank = p.Rank
+					item.DealShortName = p.ParticipantName1
+					item.BuyShortName = p.ParticipantName2
+					item.SoldShortName = p.ParticipantName3
+					item.DealName = strings.Replace(fmt.Sprintf("%s", p.ParticipantName1+"_成交量"+"_"+p.ProductName+"_"+p.ContractCode), " ", "", -1)
+					item.BuyName = strings.Replace(fmt.Sprintf("%s", p.ParticipantName2+"_持买单量"+"_"+p.ProductName+"_"+p.ContractCode), " ", "", -1)
+					item.SoldName = strings.Replace(fmt.Sprintf("%s", p.ParticipantName3+"_持卖单量"+"_"+p.ProductName+"_"+p.ContractCode), " ", "", -1)
+					item.DealCode =  shIndexCodeGenerator(item.DealShortName, item.DealName, p.ContractCode, "deal")
+					item.BuyCode =  shIndexCodeGenerator(item.BuyShortName, item.BuyName, p.ContractCode, "buy")
+					item.SoldCode =  shIndexCodeGenerator(item.SoldShortName, item.SoldName, p.ContractCode, "sold")
+					item.DealValue = p.Deal
+					item.DealChange = p.Change1
+					item.BuyValue = p.BuyIn
+					item.BuyChange = p.Change2
+					item.SoldValue = p.SoldOut
+					item.SoldChange = p.Change3
+					item.ClassifyName = strings.Replace(p.ProductName, " ", "", -1)
+					item.ClassifyType = strings.Replace(p.ContractCode, " ", "", -1)
+					item.Frequency = "日度"
+					item.CreateTime = time.Now()
+					item.ModifyTime = time.Now()
+					item.DataTime = tradeDate
 
-				itemVerifyCode = item.BuyValue + item.DealValue + item.SoldValue
-				if existIndex, ok := existIndexMap[item.DealName + item.BuyName + item.SoldName]; !ok {
-					newID, err := models.AddBaseFromTradeShangHaiIndex(item)
-					if err != nil {
-						fmt.Println("insert error:", err)
-					}
-					fmt.Println("insert new indexID:", newID)
-				} else if existIndex != nil && itemVerifyCode != (existIndex.DealValue+existIndex.BuyValue+existIndex.SoldValue) {
-					//更新
-					err := models.ModifyBaseFromTradeShangHaiIndex(item.DealValue, item.BuyValue, item.SoldValue, existIndex.BaseFromTradeShangHaiIndexId)
-					if err != nil {
-						fmt.Println("data update err:", err)
+					itemVerifyCode = item.BuyValue + item.DealValue + item.SoldValue
+					if existIndex, ok := existIndexMap[item.DealName + item.BuyName + item.SoldName]; !ok {
+						newID, err := models.AddBaseFromTradeShangHaiIndex(item)
+						if err != nil {
+							fmt.Println("insert error:", err)
+						}
+						fmt.Println("insert new indexID:", newID)
+					} else if existIndex != nil && itemVerifyCode != (existIndex.DealValue+existIndex.BuyValue+existIndex.SoldValue) {
+						//更新
+						err := models.ModifyBaseFromTradeShangHaiIndex(item.DealValue, item.BuyValue, item.SoldValue, existIndex.BaseFromTradeShangHaiIndexId)
+						if err != nil {
+							fmt.Println("data update err:", err)
+						}
 					}
 				}
 			}

services/commodity_trade_zhengzhou.go

@@ -2,6 +2,7 @@ package services
 
 import (
 	"fmt"
+	"github.com/mozillazg/go-pinyin"
 	"hongze/hongze_data_crawler/models"
 	"hongze/hongze_data_crawler/utils"
 	"log"
@@ -17,14 +18,29 @@ var zIndexCode string
 var zIndexCodeMap = make(map[string]string)
 var zActionCodeMap = make(map[string]map[string]int)
 
-func zIndexCodeGenerator(shortName, indexName,suffix string) string {
-	if shortName == ""{
+func zIndexCodeGenerator(shortName, indexName, contractCode, suffix string) string {
+	if len(shortName) == 0 || shortName == "-" {
+		return ""
+	}
+	a := pinyin.NewArgs()
+	rows := pinyin.Pinyin(shortName, a)
+	strResult := ""
+	for i := 0; i < len(rows[:2]); i++ {
+		if len(rows[i]) != 0 {
+			strResult += rows[i][0]
+		}
+	}
+	if shortName == "" {
 		zIndexCode = ""
 		return zIndexCode
 	}
-	zIndexCode,_ := zIndexCodeMap[indexName]
+	//郑州特殊处理，当合约号有中文时只取英文代号 如 苹果AP ---> AP
+	if len(contractCode) > 7 {
+		contractCode = contractCode[len(contractCode)-2:]
+	}
+	zIndexCode, _ := zIndexCodeMap[indexName]
 	if zIndexCode == "" {
-		zIndexCode = fmt.Sprintf("Z%s", strconv.FormatInt(time.Now().UnixNano(), 10)+suffix)
+		zIndexCode = strResult + contractCode + suffix
 		zIndexCodeMap[indexName] = zIndexCode
 		err := models.AddBaseFromTradeMapping(indexName, zIndexCode, "Z")
 		if err != nil {
@@ -116,7 +132,7 @@ func SyncRankingFromZhengzhou() {
 					}
 					//fmt.Println("tdText: ", tdText)
 					if tk == 0 {
-						if !(strings.Contains(tdText, "名次") || strings.Contains(tdText, "合计")){
+						if !(strings.Contains(tdText, "名次") || strings.Contains(tdText, "合计")) {
 							rank = tdText
 						}
 					}
@@ -205,17 +221,17 @@ func SyncRankingFromZhengzhou() {
 						item.Rank, _ = strconv.Atoi(rank)
 						item.DealShortName = memberShortNameArr[0]
 						item.DealName = memberShortNameArr[0] + "_" + dealSuffix + "_" + classifyName + "_" + classifyType
-						item.DealCode = zIndexCodeGenerator(item.DealShortName, item.DealName, "deal")
+						item.DealCode = zIndexCodeGenerator(item.DealShortName, item.DealName, classifyName, "deal")
 						item.DealValue, _ = strconv.Atoi(dealVal)
 						item.DealChange, _ = strconv.Atoi(dealAddCutVal)
 						item.BuyShortName = memberShortNameArr[1]
 						item.BuyName = memberShortNameArr[1] + "_" + buySuffix + "_" + classifyName + "_" + classifyType
-						item.BuyCode = zIndexCodeGenerator(item.DealShortName, item.BuyName, "buy")
+						item.BuyCode = zIndexCodeGenerator(item.BuyShortName, item.BuyName, classifyName, "buy")
 						item.BuyValue, _ = strconv.Atoi(buyVal)
 						item.BuyChange, _ = strconv.Atoi(buyAddCutVal)
 						item.SoldShortName = memberShortNameArr[2]
 						item.SoldName = memberShortNameArr[2] + "_" + sellSuffix + "_" + classifyName + "_" + classifyType
-						item.SoldCode = zIndexCodeGenerator(item.DealShortName, item.SoldName, "sold")
+						item.SoldCode = zIndexCodeGenerator(item.SoldShortName, item.SoldName, classifyName, "sold")
 						item.SoldValue, _ = strconv.Atoi(sellVal)
 						item.SoldChange, _ = strconv.Atoi(sellAddCutVal)
 						item.Frequency = "日度"
@@ -226,7 +242,7 @@ func SyncRankingFromZhengzhou() {
 						item.DataTime = tradeDate
 
 						itemVerifyCode = item.BuyValue + item.DealValue + item.SoldValue
-						if existIndex, ok := existIndexMap[item.DealName + item.BuyName + item.SoldName]; !ok {
+						if existIndex, ok := existIndexMap[item.DealName+item.BuyName+item.SoldName]; !ok {
 							newID, err := models.AddBaseFromTradeZhengzhouIndex(item)
 							if err != nil {
 								fmt.Println("insert error:", err)