indexCode逻辑优化

models/db.go

@@ -31,6 +31,7 @@ func init() {
 		new(BaseFromTradeShanghaiIndex),
 		new(BaseFromTradeIneIndex),
 		new(BaseFromTradeCffexIndex),
+		new(BaseFromEicIndex),
 		//
 		new(BaseFromTradeDalianIndex),
 		//new(BaseFromTradeDalianData),

services/commodity_trade_cffex.go

@@ -7,6 +7,7 @@ import (
 	"hongze/hongze_data_crawler/utils"
 	"rdluck_tools/http"
 	"strconv"
+	"strings"
 	"time"
 )
 
@@ -30,7 +31,11 @@ var cffexIndexCode string
 var cffexIndexCodeMap = make(map[string]string)
 var cffexActionCodeMap = make(map[string]map[string]int)
 
-func cffexIndexCodeGenerator(indexName, suffix string) string {
+func cffexIndexCodeGenerator(shortName, indexName, suffix string) string {
+	if shortName == ""{
+		cffexIndexCode = ""
+		return cffexIndexCode
+	}
 	cffexIndexCode, _ := cffexIndexCodeMap[indexName]
 	if cffexIndexCode == "" {
 		cffexIndexCode = fmt.Sprintf("CFFEX%s", time.Now().Format(utils.FormatDateTimeUnSpace)+strconv.Itoa(utils.GetRandInt(1, 100))+suffix)
@@ -77,8 +82,12 @@ func SyncRankingFromCffex() {
 			existDealMap[v.DealName] = v.BaseFromTradeCffexIndexId
 			existBuyMap[v.BuyName] = v.BaseFromTradeCffexIndexId
 			existSoldMap[v.SoldName] = v.BaseFromTradeCffexIndexId
+			cffexIndexCodeMap[v.DealName] = v.DealCode
+			cffexIndexCodeMap[v.BuyName] = v.BuyCode
+			cffexIndexCodeMap[v.SoldName] = v.SoldCode
 			sRank = fmt.Sprintf("%d", v.Rank)
-			cffexActionCodeMap[sRank+v.ClassifyType] = map[string]int{"0": v.DealValue, "1": v.BuyValue, "2": v.SoldValue, "id": v.BaseFromTradeCffexIndexId}
+			time := strings.Replace(v.DataTime, "-", "", -1)
+			cffexActionCodeMap[sRank+v.ClassifyType+time] = map[string]int{"0": v.DealValue, "1": v.BuyValue, "2": v.SoldValue, "id": v.BaseFromTradeCffexIndexId}
 		}
 		shortNameColum := map[string]string{"0": "deal_short_name", "1": "buy_short_name", "2": "sold_short_name"}
 		nameColum := map[string]string{"0": "deal_name", "1": "buy_name", "2": "sold_name"}
@@ -92,12 +101,15 @@ func SyncRankingFromCffex() {
 		for _, xmlItem := range xmlItems {
 			for _, i := range xmlItem.Data {
 				var item = new(models.BaseFromTradeCffexIndex)
+				item.DealValue = -1
+				item.BuyValue = -1
+				item.SoldValue = -1
 				item.Rank = i.Rank
 				switch i.Value {
 				case "0":
 					item.DealShortName = i.ShortName
 					item.DealName = fmt.Sprintf("%s", i.ShortName+"_成交量_"+i.ContractCode)
-					item.DealCode = cffexIndexCodeGenerator(item.DealName, "deal")
+					item.DealCode = cffexIndexCodeGenerator(item.DealShortName, item.DealName, "deal")
 					item.DealValue = i.Volume
 					item.DealChange = i.Varvolume
 
@@ -106,7 +118,7 @@ func SyncRankingFromCffex() {
 				case "1":
 					item.BuyShortName = i.ShortName
 					item.BuyName = fmt.Sprintf("%s", i.ShortName+"_持买单量_"+i.ContractCode)
-					item.BuyCode = cffexIndexCodeGenerator(item.BuyName, "buy")
+					item.BuyCode = cffexIndexCodeGenerator(item.BuyShortName, item.BuyName, "buy")
 					item.BuyValue = i.Volume
 					item.BuyChange = i.Varvolume
 
@@ -115,7 +127,7 @@ func SyncRankingFromCffex() {
 				case "2":
 					item.SoldShortName = i.ShortName
 					item.SoldName = fmt.Sprintf("%s", i.ShortName+"_持卖单量_"+i.ContractCode)
-					item.SoldCode = cffexIndexCodeGenerator(item.SoldName, "sold")
+					item.SoldCode = cffexIndexCodeGenerator(item.SoldShortName, item.SoldName, "sold")
 					item.SoldValue = i.Volume
 					item.SoldChange = i.Varvolume
 
@@ -133,24 +145,19 @@ func SyncRankingFromCffex() {
 
 				valueMap := map[string]int{"0": item.DealValue, "1": item.BuyValue, "2": item.SoldValue}
 				//检查在actionCodeMap中是否已经有了
-				indexKey = fmt.Sprintf("%d", item.Rank) + item.ClassifyType
+				indexKey = fmt.Sprintf("%d", item.Rank) + item.ClassifyType + item.DataTime
+
 				if existMap, ok := cffexActionCodeMap[indexKey]; !ok {
-					if _, ok := existDealMap[item.DealName]; !ok {
-						if _, ok := existBuyMap[item.BuyName]; !ok {
-							if _, ok := existSoldMap[item.SoldName]; !ok {
-								//没有，新增
-								newID, err := models.AddBaseFromTradeCffexIndex(item)
-								if err != nil {
-									fmt.Println("insert error:", err)
-								}
-								fmt.Println("insert new indexID:", newID)
-								existMap = make(map[string]int)
-								existMap[i.Value] = i.Volume
-								existMap["id"] = int(newID)
-								cffexActionCodeMap[indexKey] = existMap
-							}
-						}
+					//没有，新增
+					newID, err := models.AddBaseFromTradeCffexIndex(item)
+					if err != nil {
+						fmt.Println("insert error:", err)
 					}
+					fmt.Println("insert new indexID:", newID)
+					existMap = make(map[string]int)
+					existMap[i.Value] = i.Volume
+					existMap["id"] = int(newID)
+					cffexActionCodeMap[indexKey] = existMap
 				} else if existMap != nil && existMap[i.Value] != i.Volume {
 					//更新
 					var columnList = [5]string{
@@ -172,6 +179,69 @@ func SyncRankingFromCffex() {
 						fmt.Println("data update err:", err)
 					}
 					cffexActionCodeMap[indexKey][i.Value] = i.Volume
+				} else if _, ok := existDealMap[item.DealName]; !ok {
+					//更新
+					var columnList = [5]string{
+						shortNameColum[i.Value],
+						nameColum[i.Value],
+						codeColum[i.Value],
+						valueColum[i.Value],
+						changeColum[i.Value],
+					}
+					var dataList = [5]interface{}{
+						i.ShortName,
+						dataName,
+						dataCode,
+						valueMap[i.Value],
+						i.Varvolume,
+					}
+					err := models.ModifyBaseFromTradeCffexIndex(columnList, dataList, existMap["id"])
+					if err != nil {
+						fmt.Println("data update err:", err)
+					}
+					cffexActionCodeMap[indexKey][i.Value] = i.Volume
+				} else if _, ok := existBuyMap[item.BuyName]; !ok {
+					//更新
+					var columnList = [5]string{
+						shortNameColum[i.Value],
+						nameColum[i.Value],
+						codeColum[i.Value],
+						valueColum[i.Value],
+						changeColum[i.Value],
+					}
+					var dataList = [5]interface{}{
+						i.ShortName,
+						dataName,
+						dataCode,
+						valueMap[i.Value],
+						i.Varvolume,
+					}
+					err := models.ModifyBaseFromTradeCffexIndex(columnList, dataList, existMap["id"])
+					if err != nil {
+						fmt.Println("data update err:", err)
+					}
+					cffexActionCodeMap[indexKey][i.Value] = i.Volume
+				} else if _, ok := existSoldMap[item.SoldName]; !ok {
+					//更新
+					var columnList = [5]string{
+						shortNameColum[i.Value],
+						nameColum[i.Value],
+						codeColum[i.Value],
+						valueColum[i.Value],
+						changeColum[i.Value],
+					}
+					var dataList = [5]interface{}{
+						i.ShortName,
+						dataName,
+						dataCode,
+						valueMap[i.Value],
+						i.Varvolume,
+					}
+					err := models.ModifyBaseFromTradeCffexIndex(columnList, dataList, existMap["id"])
+					if err != nil {
+						fmt.Println("data update err:", err)
+					}
+					cffexActionCodeMap[indexKey][i.Value] = i.Volume
 				}
 			}
 		}

services/commodity_trade_ine.go

@@ -43,7 +43,11 @@ type message struct {
 var ineIndexCode string
 var ineIndexCodeMap = make(map[string]string)
 
-func IneIndexCodeGenerator(indexName,suffix string) string {
+func IneIndexCodeGenerator(shortName, indexName,suffix string) string {
+	if shortName == ""{
+		ineIndexCode = ""
+		return ineIndexCode
+	}
 	ineIndexCode,_ := ineIndexCodeMap[indexName]
 	if ineIndexCode == "" {
 		ineIndexCode = fmt.Sprintf("INE%s", time.Now().Format(utils.FormatDateTimeUnSpace)+strconv.Itoa(utils.GetRandInt(1, 100))+suffix)
@@ -66,7 +70,7 @@ func SyncRankingFromIne() {
 		ineIndexCodeMap[item.IndexName] = item.IndexCode
 	}
 	//获取新的指标信息
-	for i := 2; i > 0; i-- {
+	for i := 20; i > 0; i-- {
 		var message Message
 		var item = new(models.BaseFromTradeIneIndex)
 		zzUrl := "http://www.ine.com.cn/data/dailydata/kx/pm%s.dat"
@@ -107,9 +111,9 @@ func SyncRankingFromIne() {
 				item.DealName = strings.Replace(fmt.Sprintf("%s", p.ParticipantName1+"_成交量"+"_"+p.ProductName+"_"+p.ContractCode), " ", "", -1)
 				item.BuyName = strings.Replace(fmt.Sprintf("%s", p.ParticipantName2+"_持买单量"+"_"+p.ProductName+"_"+p.ContractCode), " ", "", -1)
 				item.SoldName = strings.Replace(fmt.Sprintf("%s", p.ParticipantName3+"_持卖单量"+"_"+p.ProductName+"_"+p.ContractCode), " ", "", -1)
-				item.DealCode = IneIndexCodeGenerator(item.DealName, "deal")
-				item.BuyCode = IneIndexCodeGenerator(item.BuyName, "buy")
-				item.SoldCode = IneIndexCodeGenerator(item.SoldName, "sold")
+				item.DealCode = IneIndexCodeGenerator(item.DealShortName, item.DealName, "deal")
+				item.BuyCode = IneIndexCodeGenerator(item.BuyShortName, item.BuyName, "buy")
+				item.SoldCode = IneIndexCodeGenerator(item.SoldShortName, item.SoldName, "sold")
 				item.DealValue = p.Deal
 				item.DealChange = p.Change1
 				item.BuyValue = p.BuyIn

services/commodity_trade_shanghai.go

@@ -43,7 +43,11 @@ type Message struct {
 var indexCode string
 var indexCodeMap = make(map[string]string)
 
-func shIndexCodeGenerator(indexName,suffix string) string {
+func shIndexCodeGenerator(shortName, indexName,suffix string) string {
+	if shortName == ""{
+		indexCode = ""
+		return indexCode
+	}
 	indexCode,_ := indexCodeMap[indexName]
 	if indexCode == "" {
 		indexCode = fmt.Sprintf("SH%s", time.Now().Format(utils.FormatDateTimeUnSpace)+strconv.Itoa(utils.GetRandInt(1, 100))+suffix)
@@ -85,6 +89,9 @@ func SyncRankingFromShangHai() {
 		for _, v := range allIndex {
 			indexKey := v.DealName + v.BuyName + v.SoldName
 			existIndexMap[indexKey] = v
+			indexCodeMap[v.DealName] = v.DealCode
+			indexCodeMap[v.BuyName] = v.BuyCode
+			indexCodeMap[v.SoldName] = v.SoldCode
 		}
 		var itemVerifyCode int
 		//处理指标
@@ -98,9 +105,9 @@ func SyncRankingFromShangHai() {
 				item.DealName = strings.Replace(fmt.Sprintf("%s", p.ParticipantName1+"_成交量"+"_"+p.ProductName+"_"+p.ContractCode), " ", "", -1)
 				item.BuyName = strings.Replace(fmt.Sprintf("%s", p.ParticipantName2+"_持买单量"+"_"+p.ProductName+"_"+p.ContractCode), " ", "", -1)
 				item.SoldName = strings.Replace(fmt.Sprintf("%s", p.ParticipantName3+"_持卖单量"+"_"+p.ProductName+"_"+p.ContractCode), " ", "", -1)
-				item.DealCode =  shIndexCodeGenerator(item.DealName, "deal")
-				item.BuyCode =  shIndexCodeGenerator(item.BuyName, "buy")
-				item.SoldCode =  shIndexCodeGenerator(item.SoldCode, "sold")
+				item.DealCode =  shIndexCodeGenerator(item.DealShortName, item.DealName, "deal")
+				item.BuyCode =  shIndexCodeGenerator(item.BuyShortName, item.BuyName, "buy")
+				item.SoldCode =  shIndexCodeGenerator(item.SoldShortName, item.SoldCode, "sold")
 				item.DealValue = p.Deal
 				item.DealChange = p.Change1
 				item.BuyValue = p.BuyIn

services/commodity_trade_zhengzhou.go

@@ -17,7 +17,11 @@ var zIndexCode string
 var zIndexCodeMap = make(map[string]string)
 var zActionCodeMap = make(map[string]map[string]int)
 
-func zIndexCodeGenerator(indexName,suffix string) string {
+func zIndexCodeGenerator(shortName, indexName,suffix string) string {
+	if shortName == ""{
+		zIndexCode = ""
+		return zIndexCode
+	}
 	zIndexCode,_ := zIndexCodeMap[indexName]
 	if zIndexCode == "" {
 		zIndexCode = fmt.Sprintf("Z%s", time.Now().Format(utils.FormatDateTimeUnSpace)+strconv.Itoa(utils.GetRandInt(1, 100))+suffix)
@@ -63,6 +67,9 @@ func SyncRankingFromZhengzhou() {
 		for _, v := range allIndex {
 			indexKey := v.DealName + v.BuyName + v.SoldName
 			existIndexMap[indexKey] = v
+			zIndexCodeMap[v.DealName] = v.DealCode
+			zIndexCodeMap[v.BuyName] = v.BuyCode
+			zIndexCodeMap[v.SoldName] = v.SoldCode
 		}
 		bodyStr := string(body)
 		utils.FileLog.Info(bodyStr)
@@ -193,17 +200,17 @@ func SyncRankingFromZhengzhou() {
 						item.Rank, _ = strconv.Atoi(rank)
 						item.DealShortName = memberShortNameArr[0]
 						item.DealName = memberShortNameArr[0] + "_" + dealSuffix + "_" + classifyName + "_" + classifyType
-						item.DealCode = zIndexCodeGenerator(item.DealName, "deal")
+						item.DealCode = zIndexCodeGenerator(item.DealShortName, item.DealName, "deal")
 						item.DealValue, _ = strconv.Atoi(dealVal)
 						item.DealChange, _ = strconv.Atoi(dealAddCutVal)
 						item.BuyShortName = memberShortNameArr[1]
 						item.BuyName = memberShortNameArr[1] + "_" + buySuffix + "_" + classifyName + "_" + classifyType
-						item.BuyCode = zIndexCodeGenerator(item.BuyName, "buy")
+						item.BuyCode = zIndexCodeGenerator(item.DealShortName, item.BuyName, "buy")
 						item.BuyValue, _ = strconv.Atoi(buyVal)
 						item.BuyChange, _ = strconv.Atoi(buyAddCutVal)
 						item.SoldShortName = memberShortNameArr[2]
 						item.SoldName = memberShortNameArr[2] + "_" + sellSuffix + "_" + classifyName + "_" + classifyType
-						item.SoldCode = zIndexCodeGenerator(item.SoldName, "sold")
+						item.SoldCode = zIndexCodeGenerator(item.DealShortName, item.SoldName, "sold")
 						item.SoldValue, _ = strconv.Atoi(sellVal)
 						item.SoldChange, _ = strconv.Atoi(sellAddCutVal)
 						item.Frequency = "日度"

services/task.go

@@ -1,9 +1,7 @@
 package services
 
 import (
-	"context"
 	"fmt"
-	"github.com/beego/beego/v2/task"
 )
 
 func Task() {
@@ -18,20 +16,20 @@ func Task() {
 	//time.Sleep(20 * time.Second)
 	//SyncRankingFromDalianDo()
 	//SyncStorageFromEic()
-	//SyncStorageFromEic()
-	refreshData := task.NewTask("refreshData", "0 0 3 * * *", RefreshData)
-	task.AddTask("数据爬取", refreshData)
-	task.StartTask()
+
+	//refreshData := task.NewTask("refreshData", "0 0 3 * * *", RefreshData)
+	//task.AddTask("数据爬取", refreshData)
+	//task.StartTask()
 	fmt.Println("end crawler")
 }
 
-func RefreshData(cont context.Context) (err error) {
-	//大连
-	SyncRankingFromDalianSearch(1)
-	SyncRankingFromZhengzhou()
-	SyncRankingFromShangHai()
-	SyncRankingFromIne()
-	SyncRankingFromCffex()
-
-	return
-}
+//func RefreshData(cont context.Context) (err error) {
+//	//大连
+//	SyncRankingFromDalian(1)
+//	SyncRankingFromZhengzhou()
+//	SyncRankingFromShangHai()
+//	SyncRankingFromIne()
+//	SyncRankingFromCffex()
+//
+//	return
+//}