上期所和上期能源改为本地爬虫获取

services/commodity_trade_ine.go

@@ -12,39 +12,36 @@ import (
 	"time"
 )
 
-type position []struct {
-	ContractCode     string `json:"INSTRUMENTID"`
-	ProductSortNo    int    `json:"PRODUCTSORTNO"`
-	Rank             int    `json:"RANK"`
-	ParticipantID1   string `json:"PARTICIPANTID1"`
-	ParticipantName1 string `json:"PARTICIPANTABBR1"`
-	Deal             int    `json:"CJ1"`
-	Change1          int    `json:"CJ1_CHG"`
-	ParticipantID2   string `json:"PARTICIPANTID2"`
-	ParticipantName2 string `json:"PARTICIPANTABBR2"`
-	BuyIn            int    `json:"CJ2"`
-	Change2          int    `json:"CJ2_CHG"`
-	ParticipantID3   string `json:"PARTICIPANTID3"`
-	ParticipantName3 string `json:"PARTICIPANTABBR3"`
-	SoldOut          int    `json:"CJ3"`
-	Change3          int    `json:"CJ3_CHG"`
-	ProductName      string `json:"PRODUCTNAME"`
+type JSONData struct {
+	OCursor    []OCursor   `json:"o_cursor"`
+	OCode      interface{} `json:"o_code"`
+	OMsg       string      `json:"o_msg"`
+	ReportDate string      `json:"report_date"`
+	UpdateDate string      `json:"update_date"`
 }
-
-type message struct {
-	Position   Position `json:"o_cursor"`
-	Length     string   `json:"showlength"`
-	Code       int      `json:"o_code"`
-	Msg        string   `json:"o_msg"`
-	ReportDate string   `json:"report_date"`
-	UpdateDate string   `json:"update_date"`
-	PrintDate  string   `json:"print_date"`
+type OCursor struct {
+	Instrumentid     string      `json:"INSTRUMENTID"`
+	Participantid3   string      `json:"PARTICIPANTID3"`
+	Participantid2   string      `json:"PARTICIPANTID2"`
+	Participantid1   string      `json:"PARTICIPANTID1"`
+	Participantabbr3 string      `json:"PARTICIPANTABBR3"`
+	Participantabbr2 string      `json:"PARTICIPANTABBR2"`
+	Rank             int         `json:"RANK"`
+	Participantabbr1 string      `json:"PARTICIPANTABBR1"`
+	BuyIn            interface{} `json:"CJ2"`
+	Deal             interface{} `json:"CJ1"`
+	Change1          interface{} `json:"CJ1_CHG"`
+	Change3          interface{} `json:"CJ3_CHG"`
+	Productname      string      `json:"Productname"`
+	Productsortno    interface{} `json:"PRODUCTSORTNO"`
+	SoldOut          interface{} `json:"CJ3"`
+	Change2          interface{} `json:"CJ2_CHG"`
 }
 
 var ineIndexCode string
 var ineIndexCodeMap = make(map[string]string)
 
-func IneIndexCodeGenerator(shortName, indexName, contractCode, suffix string) string {
+func IneIndexCodeGenerator(shortName, indexName, Instrumentid, suffix string) string {
 	if shortName == "" {
 		return ""
 	}
@@ -53,7 +50,7 @@ func IneIndexCodeGenerator(shortName, indexName, contractCode, suffix string) st
 		//取公司全拼
 		a := pinyin.NewArgs()
 		rows := pinyin.LazyPinyin(shortName, a)
-		for i:=0;i<len(rows);i++{
+		for i := 0; i < len(rows); i++ {
 			strResult += rows[i]
 		}
 	} else {
@@ -61,7 +58,7 @@ func IneIndexCodeGenerator(shortName, indexName, contractCode, suffix string) st
 	}
 	ineIndexCode, _ := ineIndexCodeMap[indexName]
 	if ineIndexCode == "" {
-		ineIndexCode = strResult + contractCode + suffix
+		ineIndexCode = strResult + Instrumentid + suffix
 		ineIndexCode = strings.Replace(ineIndexCode, " ", "", -1)
 		ineIndexCodeMap[indexName] = ineIndexCode
 		err := models.AddBaseFromTradeMapping(indexName, ineIndexCode, "INE")
@@ -92,7 +89,7 @@ func SyncRankingFromIne() {
 	}
 	//获取新的指标信息
 	for i := 10; i >= 0; i-- {
-		var message Message
+		var message JSONData
 		var item = new(models.BaseFromTradeIneIndex)
 		zzUrl := "http://www.ine.com.cn/data/dailydata/kx/pm%s.dat"
 		date := time.Now().AddDate(0, 0, -i)
@@ -103,12 +100,15 @@ func SyncRankingFromIne() {
 		if err != nil {
 			fmt.Println("err:", err)
 		}
+		if body == nil {
+			return
+		}
 		err = json.Unmarshal(body, &message)
 		if err != nil {
 			fmt.Println("Unmarshal Err:", err)
 			continue
 		}
-		var position = message.Position
+		var position = message.OCursor
 		var tradeDate = message.ReportDate
 		existIndexMap := make(map[string]*models.BaseFromTradeIneIndex)
 
@@ -124,41 +124,41 @@ func SyncRankingFromIne() {
 		var itemVerifyCode int
 		//处理指标
 		for _, p := range position {
-			if p.Rank > 0 && p.Rank < 40 && p.ParticipantName1 != "" {
+			if p.Rank > 0 && p.Rank < 40 && p.Participantabbr1 != "" {
 				//成交量
 				item.Rank = p.Rank
-				item.DealShortName = p.ParticipantName1
-				item.BuyShortName = p.ParticipantName2
-				item.SoldShortName = p.ParticipantName3
-				item.DealName = strings.Replace(fmt.Sprintf("%s", p.ParticipantName1+"_"+p.ContractCode+"_成交量(手)"), " ", "", -1)
-				item.BuyName = strings.Replace(fmt.Sprintf("%s", p.ParticipantName2+"_"+p.ContractCode+"_持买单量(手)"), " ", "", -1)
-				item.SoldName = strings.Replace(fmt.Sprintf("%s", p.ParticipantName3+"_"+p.ContractCode+"_持卖单量(手)"), " ", "", -1)
-				item.DealCode = IneIndexCodeGenerator(item.DealShortName, item.DealName, p.ContractCode, "deal")
-				item.BuyCode = IneIndexCodeGenerator(item.BuyShortName, item.BuyName, p.ContractCode, "buy")
-				item.SoldCode = IneIndexCodeGenerator(item.SoldShortName, item.SoldName, p.ContractCode, "sold")
-				item.ClassifyName = strings.Replace(p.ProductName, " ", "", -1)
-				item.ClassifyType = strings.Replace(p.ContractCode, " ", "", -1)
+				item.DealShortName = p.Participantabbr1
+				item.BuyShortName = p.Participantabbr2
+				item.SoldShortName = p.Participantabbr3
+				item.DealName = strings.Replace(fmt.Sprintf("%s", p.Participantabbr1+"_"+p.Instrumentid+"_成交量(手)"), " ", "", -1)
+				item.BuyName = strings.Replace(fmt.Sprintf("%s", p.Participantabbr2+"_"+p.Instrumentid+"_持买单量(手)"), " ", "", -1)
+				item.SoldName = strings.Replace(fmt.Sprintf("%s", p.Participantabbr3+"_"+p.Instrumentid+"_持卖单量(手)"), " ", "", -1)
+				item.DealCode = IneIndexCodeGenerator(item.DealShortName, item.DealName, p.Instrumentid, "deal")
+				item.BuyCode = IneIndexCodeGenerator(item.BuyShortName, item.BuyName, p.Instrumentid, "buy")
+				item.SoldCode = IneIndexCodeGenerator(item.SoldShortName, item.SoldName, p.Instrumentid, "sold")
+				item.ClassifyName = strings.Replace(p.Productname, " ", "", -1)
+				item.ClassifyType = strings.Replace(p.Instrumentid, " ", "", -1)
 				item.Frequency = "日度"
 				item.CreateTime = time.Now()
 				item.ModifyTime = time.Now()
 				item.DataTime = tradeDate
 
-				if deal, ok := p.Deal.(float64); ok{
+				if deal, ok := p.Deal.(float64); ok {
 					item.DealValue = int(deal)
 				}
-				if change1, ok := p.Change1.(float64); ok{
+				if change1, ok := p.Change1.(float64); ok {
 					item.DealChange = int(change1)
 				}
-				if buyIn, ok := p.BuyIn.(float64); ok{
+				if buyIn, ok := p.BuyIn.(float64); ok {
 					item.BuyValue = int(buyIn)
 				}
-				if change2, ok := p.Change2.(float64); ok{
+				if change2, ok := p.Change2.(float64); ok {
 					item.BuyChange = int(change2)
 				}
-				if soldOut, ok := p.SoldOut.(float64); ok{
+				if soldOut, ok := p.SoldOut.(float64); ok {
 					item.SoldValue = int(soldOut)
 				}
-				if change3, ok := p.Change3.(float64); ok{
+				if change3, ok := p.Change3.(float64); ok {
 					item.SoldChange = int(change3)
 				}
 
@@ -179,38 +179,38 @@ func SyncRankingFromIne() {
 			} else if p.Rank == 999 {
 				//Top 20
 				item.Rank = p.Rank
-				item.DealShortName = p.ParticipantName1
-				item.BuyShortName = p.ParticipantName2
-				item.SoldShortName = p.ParticipantName3
-				item.DealName = strings.Replace(fmt.Sprintf("%s", "top20_"+p.ContractCode+"_成交量(手)"), " ", "", -1)
-				item.BuyName = strings.Replace(fmt.Sprintf("%s", "top20_"+p.ContractCode+"_持买单量(手)"), " ", "", -1)
-				item.SoldName = strings.Replace(fmt.Sprintf("%s", "top20_"+p.ContractCode+"_持卖单量(手)"), " ", "", -1)
-				item.DealCode = IneIndexCodeGenerator("top20", item.DealName, p.ContractCode, "deal")
-				item.BuyCode = IneIndexCodeGenerator("top20", item.BuyName, p.ContractCode, "buy")
-				item.SoldCode = IneIndexCodeGenerator("top20", item.SoldName, p.ContractCode, "sold")
-				item.ClassifyName = strings.Replace(p.ProductName, " ", "", -1)
-				item.ClassifyType = strings.Replace(p.ContractCode, " ", "", -1)
+				item.DealShortName = p.Participantabbr1
+				item.BuyShortName = p.Participantabbr2
+				item.SoldShortName = p.Participantabbr3
+				item.DealName = strings.Replace(fmt.Sprintf("%s", "top20_"+p.Instrumentid+"_成交量(手)"), " ", "", -1)
+				item.BuyName = strings.Replace(fmt.Sprintf("%s", "top20_"+p.Instrumentid+"_持买单量(手)"), " ", "", -1)
+				item.SoldName = strings.Replace(fmt.Sprintf("%s", "top20_"+p.Instrumentid+"_持卖单量(手)"), " ", "", -1)
+				item.DealCode = IneIndexCodeGenerator("top20", item.DealName, p.Instrumentid, "deal")
+				item.BuyCode = IneIndexCodeGenerator("top20", item.BuyName, p.Instrumentid, "buy")
+				item.SoldCode = IneIndexCodeGenerator("top20", item.SoldName, p.Instrumentid, "sold")
+				item.ClassifyName = strings.Replace(p.Productname, " ", "", -1)
+				item.ClassifyType = strings.Replace(p.Instrumentid, " ", "", -1)
 				item.Frequency = "日度"
 				item.CreateTime = time.Now()
 				item.ModifyTime = time.Now()
 				item.DataTime = tradeDate
 
-				if deal, ok := p.Deal.(float64); ok{
+				if deal, ok := p.Deal.(float64); ok {
 					item.DealValue = int(deal)
 				}
-				if change1, ok := p.Change1.(float64); ok{
+				if change1, ok := p.Change1.(float64); ok {
 					item.DealChange = int(change1)
 				}
-				if buyIn, ok := p.BuyIn.(float64); ok{
+				if buyIn, ok := p.BuyIn.(float64); ok {
 					item.BuyValue = int(buyIn)
 				}
-				if change2, ok := p.Change2.(float64); ok{
+				if change2, ok := p.Change2.(float64); ok {
 					item.BuyChange = int(change2)
 				}
-				if soldOut, ok := p.SoldOut.(float64); ok{
+				if soldOut, ok := p.SoldOut.(float64); ok {
 					item.SoldValue = int(soldOut)
 				}
-				if change3, ok := p.Change3.(float64); ok{
+				if change3, ok := p.Change3.(float64); ok {
 					item.SoldChange = int(change3)
 				}
 

services/task.go

@@ -13,7 +13,8 @@ func Task() {
 	//syncAllComTradeMapping3()
 	//return
 	fmt.Println("start crawler")
-	SyncRankingFromShangHai()
+	//SyncRankingFromShangHai()
+	//SyncRankingFromIne()
 	FileCoalCoastal()
 	FileCoalInland()
 	refreshData := task.NewTask("refreshData", "0 0,30 16-23 * * *", RefreshData)
@@ -58,8 +59,8 @@ func RefreshData(cont context.Context) (err error) {
 	//大连
 	SyncRankingFromDalianSearch(0)
 	SyncRankingFromZhengzhou()
-	SyncRankingFromShangHai()
-	SyncRankingFromIne()
+	//SyncRankingFromShangHai()
+	//SyncRankingFromIne()
 	SyncRankingFromCffex()
 	//SyncStorageFromEic()