新增合计指标id爬取及新增

services/commodity_trade_ine.go

@@ -4,9 +4,9 @@ import (
 	"encoding/json"
 	"fmt"
 	"github.com/mozillazg/go-pinyin"
+	"github.com/rdlucklib/rdluck_tools/http"
 	"hongze/hongze_data_crawler/models"
 	"hongze/hongze_data_crawler/utils"
-	"github.com/rdlucklib/rdluck_tools/http"
 	"strings"
 	"time"
 )
@@ -45,28 +45,32 @@ var ineIndexCodeMap = make(map[string]string)
 
 func IneIndexCodeGenerator(shortName, indexName, contractCode, suffix string) string {
 	if shortName == "" {
-		indexCode = ""
-		return indexCode
+		return ""
 	}
-	//取公司前两个字的全拼
-	a := pinyin.NewArgs()
-	rows := pinyin.Pinyin(shortName, a)
 	strResult := ""
-	for i := 0; i < len(rows[:2]); i++ {
-		if len(rows[i]) != 0 {
-			strResult += rows[i][0]
+	if shortName != "top20" {
+		//取公司前两个字的全拼
+		a := pinyin.NewArgs()
+		rows := pinyin.Pinyin(shortName, a)
+		for i := 0; i < len(rows[:2]); i++ {
+			if len(rows[i]) != 0 {
+				strResult += rows[i][0]
+			}
 		}
+	} else {
+		strResult = "top20"
 	}
 	ineIndexCode, _ := ineIndexCodeMap[indexName]
 	if ineIndexCode == "" {
 		ineIndexCode = strResult + contractCode + suffix
+		ineIndexCode = strings.Replace(ineIndexCode, " ", "", -1)
 		ineIndexCodeMap[indexName] = ineIndexCode
 		err := models.AddBaseFromTradeMapping(indexName, ineIndexCode, "INE")
 		if err != nil {
 			fmt.Println("add Code err:", err)
 		}
 	}
-	return ineIndexCode
+	return strings.Replace(ineIndexCode, " ", "", -1)
 }
 
 // SyncRankingFromIne 上海能源交易中心持单排名
@@ -135,6 +139,45 @@ func SyncRankingFromIne() {
 				item.ModifyTime = time.Now()
 				item.DataTime = tradeDate
 
+				itemVerifyCode = item.BuyValue + item.DealValue + item.SoldValue
+				if existIndex, ok := existIndexMap[item.DealName+item.BuyName+item.SoldName]; !ok {
+					newID, err := models.AddBaseFromTradeIneIndex(item)
+					if err != nil {
+						fmt.Println("insert error:", err)
+					}
+					fmt.Println("insert new indexID:", newID)
+				} else if existIndex != nil && itemVerifyCode != (existIndex.DealValue+existIndex.BuyValue+existIndex.SoldValue) {
+					//更新
+					err := models.ModifyBaseFromTradeIneIndex(item.DealValue, item.BuyValue, item.SoldValue, existIndex.BaseFromTradeIneIndexId)
+					if err != nil {
+						fmt.Println("data update err:", err)
+					}
+				}
+			} else if p.Rank == 999 {
+				//Top 20
+				item.Rank = p.Rank
+				item.DealShortName = p.ParticipantName1
+				item.BuyShortName = p.ParticipantName2
+				item.SoldShortName = p.ParticipantName3
+				item.DealName = strings.Replace(fmt.Sprintf("%s", "top20_"+p.ContractCode+"_成交量(手)"), " ", "", -1)
+				item.BuyName = strings.Replace(fmt.Sprintf("%s", "top20_"+p.ContractCode+"_持买单量(手)"), " ", "", -1)
+				item.SoldName = strings.Replace(fmt.Sprintf("%s", "top20_"+p.ContractCode+"_持卖单量(手)"), " ", "", -1)
+				item.DealCode = IneIndexCodeGenerator("top20", item.DealName, p.ContractCode, "deal")
+				item.BuyCode = IneIndexCodeGenerator("top20", item.BuyName, p.ContractCode, "buy")
+				item.SoldCode = IneIndexCodeGenerator("top20", item.SoldName, p.ContractCode, "sold")
+				item.DealValue = p.Deal
+				item.DealChange = p.Change1
+				item.BuyValue = p.BuyIn
+				item.BuyChange = p.Change2
+				item.SoldValue = p.SoldOut
+				item.SoldChange = p.Change3
+				item.ClassifyName = strings.Replace(p.ProductName, " ", "", -1)
+				item.ClassifyType = strings.Replace(p.ContractCode, " ", "", -1)
+				item.Frequency = "日度"
+				item.CreateTime = time.Now()
+				item.ModifyTime = time.Now()
+				item.DataTime = tradeDate
+
 				itemVerifyCode = item.BuyValue + item.DealValue + item.SoldValue
 				if existIndex, ok := existIndexMap[item.DealName+item.BuyName+item.SoldName]; !ok {
 					newID, err := models.AddBaseFromTradeIneIndex(item)

services/commodity_trade_shanghai.go

@@ -4,9 +4,9 @@ import (
 	"encoding/json"
 	"fmt"
 	"github.com/mozillazg/go-pinyin"
+	"github.com/rdlucklib/rdluck_tools/http"
 	"hongze/hongze_data_crawler/models"
 	"hongze/hongze_data_crawler/utils"
-	"github.com/rdlucklib/rdluck_tools/http"
 	"strings"
 	"time"
 )
@@ -45,21 +45,25 @@ var indexCodeMap = make(map[string]string)
 
 func shIndexCodeGenerator(shortName, indexName, contractCode, suffix string) string {
 	if shortName == "" {
-		indexCode = ""
-		return indexCode
+		return ""
 	}
-	//取公司前两个字的全拼
-	a := pinyin.NewArgs()
-	rows := pinyin.Pinyin(shortName, a)
 	strResult := ""
-	for i := 0; i < len(rows[:2]); i++ {
-		if len(rows[i]) != 0 {
-			strResult += rows[i][0]
+	if shortName != "top20" {
+		//取公司前两个字的全拼
+		a := pinyin.NewArgs()
+		rows := pinyin.Pinyin(shortName, a)
+		for i := 0; i < len(rows[:2]); i++ {
+			if len(rows[i]) != 0 {
+				strResult += rows[i][0]
+			}
 		}
+	} else {
+		strResult = "top20"
 	}
 	indexCode, _ := indexCodeMap[indexName]
 	if indexCode == "" {
 		indexCode = strResult + contractCode + suffix
+		indexCode = strings.Replace(indexCode, " ", "", -1)
 		indexCodeMap[indexName] = indexCode
 		err := models.AddBaseFromTradeMapping(indexName, indexCode, "SH")
 		if err != nil {
@@ -73,7 +77,7 @@ func shIndexCodeGenerator(shortName, indexName, contractCode, suffix string) str
 func SyncRankingFromShangHai() {
 	n := utils.GetRandInt(10, 120)
 	time.Sleep(time.Duration(n) * time.Second)
-	allCode, err := models.GetIndexCodeFromMapping("Z")
+	allCode, err := models.GetIndexCodeFromMapping("SH")
 	if err != nil {
 		fmt.Println("select Code err:", err)
 	}
@@ -83,7 +87,6 @@ func SyncRankingFromShangHai() {
 	//获取新的指标信息
 	for i := 2; i > 0; i-- {
 		var message Message
-		var item = new(models.BaseFromTradeShanghaiIndex)
 		zzUrl := "http://www.shfe.com.cn/data/dailydata/kx/pm%s.dat"
 		date := time.Now().AddDate(0, 0, -i)
 		dateStr := date.Format(utils.FormatDateUnSpace)
@@ -111,19 +114,63 @@ func SyncRankingFromShangHai() {
 		var itemVerifyCode int
 		//处理指标
 		for _, p := range position {
+			var item = new(models.BaseFromTradeShanghaiIndex)
 			if p.Rank > 0 && p.Rank < 40 && p.ParticipantName1 != "" {
 				if strings.Replace(p.ProductName, " ", "", -1) != "20号胶" && strings.Replace(p.ProductName, " ", "", -1) != "低硫燃料油" {
+					contractCode := strings.Replace(p.ContractCode, " ", "", -1)
 					//成交量
 					item.Rank = p.Rank
-					item.DealShortName = p.ParticipantName1
-					item.BuyShortName = p.ParticipantName2
-					item.SoldShortName = p.ParticipantName3
+					item.DealShortName = strings.Replace(p.ParticipantName1, " ", "", -1)
+					item.BuyShortName = strings.Replace(p.ParticipantName2, " ", "", -1)
+					item.SoldShortName = strings.Replace(p.ParticipantName3, " ", "", -1)
 					item.DealName = strings.Replace(fmt.Sprintf("%s", p.ParticipantName1+"_"+p.ContractCode+"_成交量(手)"), " ", "", -1)
 					item.BuyName = strings.Replace(fmt.Sprintf("%s", p.ParticipantName2+"_"+p.ContractCode+"_持买单量(手)"), " ", "", -1)
 					item.SoldName = strings.Replace(fmt.Sprintf("%s", p.ParticipantName3+"_"+p.ContractCode+"_持卖单量(手)"), " ", "", -1)
-					item.DealCode = shIndexCodeGenerator(item.DealShortName, item.DealName, p.ContractCode, "deal")
-					item.BuyCode = shIndexCodeGenerator(item.BuyShortName, item.BuyName, p.ContractCode, "buy")
-					item.SoldCode = shIndexCodeGenerator(item.SoldShortName, item.SoldName, p.ContractCode, "sold")
+					item.DealCode = shIndexCodeGenerator(item.DealShortName, item.DealName, contractCode, "deal")
+					item.BuyCode = shIndexCodeGenerator(item.BuyShortName, item.BuyName, contractCode, "buy")
+					item.SoldCode = shIndexCodeGenerator(item.SoldShortName, item.SoldName, contractCode, "sold")
+					item.DealValue = p.Deal
+					item.DealChange = p.Change1
+					item.BuyValue = p.BuyIn
+					item.BuyChange = p.Change2
+					item.SoldValue = p.SoldOut
+					item.SoldChange = p.Change3
+					item.ClassifyName = strings.Replace(p.ProductName, " ", "", -1)
+					item.ClassifyType = strings.Replace(p.ContractCode, " ", "", -1)
+					item.Frequency = "日度"
+					item.CreateTime = time.Now()
+					item.ModifyTime = time.Now()
+					item.DataTime = tradeDate
+
+					itemVerifyCode = item.BuyValue + item.DealValue + item.SoldValue
+					if existIndex, ok := existIndexMap[item.DealName+item.BuyName+item.SoldName]; !ok {
+						newID, err := models.AddBaseFromTradeShangHaiIndex(item)
+						if err != nil {
+							fmt.Println("insert error:", err)
+						}
+						fmt.Println("insert new indexID:", newID)
+					} else if existIndex != nil && itemVerifyCode != (existIndex.DealValue+existIndex.BuyValue+existIndex.SoldValue) {
+						//更新
+						err := models.ModifyBaseFromTradeShangHaiIndex(item.DealValue, item.BuyValue, item.SoldValue, existIndex.BaseFromTradeShangHaiIndexId)
+						if err != nil {
+							fmt.Println("data update err:", err)
+						}
+					}
+				}
+			} else if p.Rank == 999 {
+				if strings.Replace(p.ProductName, " ", "", -1) != "20号胶" && strings.Replace(p.ProductName, " ", "", -1) != "低硫燃料油" {
+					contractCode := strings.Replace(p.ContractCode, " ", "", -1)
+					//Top 20
+					item.Rank = p.Rank
+					item.DealShortName = strings.Replace(p.ParticipantName1, " ", "", -1)
+					item.BuyShortName = strings.Replace(p.ParticipantName2, " ", "", -1)
+					item.SoldShortName = strings.Replace(p.ParticipantName3, " ", "", -1)
+					item.DealName = strings.Replace(fmt.Sprintf("%s", "top20_"+p.ContractCode+"_成交量(手)"), " ", "", -1)
+					item.BuyName = strings.Replace(fmt.Sprintf("%s", "top20_"+p.ContractCode+"_持买单量(手)"), " ", "", -1)
+					item.SoldName = strings.Replace(fmt.Sprintf("%s", "top20_"+p.ContractCode+"_持卖单量(手)"), " ", "", -1)
+					item.DealCode = shIndexCodeGenerator("top20", item.DealName, contractCode, "deal")
+					item.BuyCode = shIndexCodeGenerator("top20", item.BuyName, contractCode, "buy")
+					item.SoldCode = shIndexCodeGenerator("top20", item.SoldName, contractCode, "sold")
 					item.DealValue = p.Deal
 					item.DealChange = p.Change1
 					item.BuyValue = p.BuyIn

services/commodity_trade_zhengzhou.go

@@ -3,10 +3,10 @@ package services
 import (
 	"fmt"
 	"github.com/mozillazg/go-pinyin"
+	"github.com/rdlucklib/rdluck_tools/http"
 	"hongze/hongze_data_crawler/models"
 	"hongze/hongze_data_crawler/utils"
 	"log"
-	"github.com/rdlucklib/rdluck_tools/http"
 	"strconv"
 	"strings"
 	"time"
@@ -19,20 +19,21 @@ var zIndexCodeMap = make(map[string]string)
 var zActionCodeMap = make(map[string]map[string]int)
 
 func zIndexCodeGenerator(shortName, indexName, contractCode, suffix string) string {
-	if len(shortName) == 0 || shortName == "-" {
+	if shortName == "" || shortName == "-"{
 		return ""
 	}
-	a := pinyin.NewArgs()
-	rows := pinyin.Pinyin(shortName, a)
 	strResult := ""
-	for i := 0; i < len(rows[:2]); i++ {
-		if len(rows[i]) != 0 {
-			strResult += rows[i][0]
+	if shortName != "top20" {
+		//取公司前两个字的全拼
+		a := pinyin.NewArgs()
+		rows := pinyin.Pinyin(shortName, a)
+		for i := 0; i < len(rows[:2]); i++ {
+			if len(rows[i]) != 0 {
+				strResult += rows[i][0]
+			}
 		}
-	}
-	if shortName == "" {
-		zIndexCode = ""
-		return zIndexCode
+	} else {
+		strResult = "top20"
 	}
 	//郑州特殊处理，当合约号有中文时只取英文代号 如 苹果AP ---> AP
 	if len(contractCode) > 7 {
@@ -217,6 +218,9 @@ func SyncRankingFromZhengzhou() {
 				//fmt.Printf("memberShortNameArr:%s,len:%d\n", memberShortNameArr, len(memberShortNameArr))
 				if len(memberShortNameArr) != 0 {
 					if len(memberShortNameArr[0]) != 2 {
+						if len(classifyName) > 7 {
+							classifyName = classifyName[len(classifyName)-2:]
+						}
 						item := new(models.BaseFromTradeZhengzhouIndex)
 						item.Rank, _ = strconv.Atoi(rank)
 						item.DealShortName = memberShortNameArr[0]
@@ -241,6 +245,49 @@ func SyncRankingFromZhengzhou() {
 						item.ModifyTime = time.Now()
 						item.DataTime = tradeDate
 
+						itemVerifyCode = item.BuyValue + item.DealValue + item.SoldValue
+						if existIndex, ok := existIndexMap[item.DealName+item.BuyName+item.SoldName]; !ok {
+							newID, err := models.AddBaseFromTradeZhengzhouIndex(item)
+							if err != nil {
+								fmt.Println("insert error:", err)
+							}
+							fmt.Println("insert new indexID:", newID)
+						} else if existIndex != nil && itemVerifyCode != (existIndex.DealValue+existIndex.BuyValue+existIndex.SoldValue) {
+							//更新
+							err := models.ModifyBaseFromTradeZhengzhouIndex(item.DealValue, item.BuyValue, item.SoldValue, existIndex.BaseFromTradeZhengzhouIndexId)
+							if err != nil {
+								fmt.Println("data update err:", err)
+							}
+						}
+					} else {
+						//合计
+						if len(classifyName) > 7 {
+							classifyName = classifyName[len(classifyName)-2:]
+						}
+						item := new(models.BaseFromTradeZhengzhouIndex)
+						item.Rank = 999
+						item.DealShortName = ""
+						item.DealName = "top20" + "_" + classifyName + "_" + dealSuffix
+						item.DealCode = zIndexCodeGenerator("top20", item.DealName, classifyName, "deal")
+						item.DealValue, _ = strconv.Atoi(dealVal)
+						item.DealChange, _ = strconv.Atoi(dealAddCutVal)
+						item.BuyShortName = ""
+						item.BuyName = "top20" + "_" + classifyName + "_" + buySuffix
+						item.BuyCode = zIndexCodeGenerator("top20", item.BuyName, classifyName, "buy")
+						item.BuyValue, _ = strconv.Atoi(buyVal)
+						item.BuyChange, _ = strconv.Atoi(buyAddCutVal)
+						item.SoldShortName = ""
+						item.SoldName = "top20" + "_" + classifyName + "_" + sellSuffix
+						item.SoldCode = zIndexCodeGenerator("top20", item.SoldName, classifyName, "sold")
+						item.SoldValue, _ = strconv.Atoi(sellVal)
+						item.SoldChange, _ = strconv.Atoi(sellAddCutVal)
+						item.Frequency = "日度"
+						item.ClassifyName = classifyName
+						item.ClassifyType = classifyType
+						item.CreateTime = time.Now()
+						item.ModifyTime = time.Now()
+						item.DataTime = tradeDate
+
 						itemVerifyCode = item.BuyValue + item.DealValue + item.SoldValue
 						if existIndex, ok := existIndexMap[item.DealName+item.BuyName+item.SoldName]; !ok {
 							newID, err := models.AddBaseFromTradeZhengzhouIndex(item)