allindex优化，变量命名位置优化，sql查询优化

models/base_from_trade_cffex.go

@@ -38,11 +38,11 @@ func AddBaseFromTradeCffexIndex(item *BaseFromTradeCffexIndex) (lastId int64, er
 	return
 }
 
-func GetBaseFromTradeCffexIndexAll() (list []*BaseFromTradeCffexIndex, err error) {
+func GetBaseFromTradeCffexIndexAll(dateStr string) (list []*BaseFromTradeCffexIndex, err error) {
 	o := orm.NewOrm()
 	o.Using("data")
-	sql := `SELECT * FROM base_from_trade_cffex_index `
-	_, err = o.Raw(sql).QueryRows(&list)
+	sql := `SELECT * FROM base_from_trade_cffex_index WHERE date_time=?`
+	_, err = o.Raw(sql, dateStr).QueryRows(&list)
 	return
 }
 

models/base_from_trade_ine.go

@@ -38,11 +38,11 @@ func AddBaseFromTradeIneIndex(item *BaseFromTradeIneIndex) (lastId int64, err er
 	return
 }
 
-func GetBaseFromTradeIneIndexAll() (list []*BaseFromTradeIneIndex, err error) {
+func GetBaseFromTradeIneIndexAll(dateStr string) (list []*BaseFromTradeIneIndex, err error) {
 	o := orm.NewOrm()
 	o.Using("data")
-	sql := `SELECT * FROM base_from_trade_ine_index `
-	_, err = o.Raw(sql).QueryRows(&list)
+	sql := `SELECT * FROM base_from_trade_ine_index where date=?`
+	_, err = o.Raw(sql, dateStr).QueryRows(&list)
 	return
 }
 

models/base_from_trade_shanghai.go

@@ -38,11 +38,11 @@ func AddBaseFromTradeShangHaiIndex(item *BaseFromTradeShanghaiIndex) (lastId int
 	return
 }
 
-func GetBaseFromTradeShangHaiIndexAll() (list []*BaseFromTradeShanghaiIndex, err error) {
+func GetBaseFromTradeShangHaiIndexAll(dateStr string) (list []*BaseFromTradeShanghaiIndex, err error) {
 	o := orm.NewOrm()
 	o.Using("data")
-	sql := `SELECT * FROM base_from_trade_shanghai_index `
-	_, err = o.Raw(sql).QueryRows(&list)
+	sql := `SELECT * FROM base_from_trade_shanghai_index WHERE data_time=?`
+	_, err = o.Raw(sql, dateStr).QueryRows(&list)
 	return
 }
 

models/base_from_trade_zhengzhou.go

@@ -38,11 +38,11 @@ func AddBaseFromTradeZhengzhouIndex(item *BaseFromTradeZhengzhouIndex) (lastId i
 	return
 }
 
-func GetBaseFromTradeZhengzhouIndexAll() (list []*BaseFromTradeZhengzhouIndex, err error) {
+func GetBaseFromTradeZhengzhouIndexAll(dateStr string) (list []*BaseFromTradeZhengzhouIndex, err error) {
 	o := orm.NewOrm()
 	o.Using("data")
-	sql := `SELECT * FROM base_from_trade_zhengzhou_index `
-	_, err = o.Raw(sql).QueryRows(&list)
+	sql := `SELECT * FROM base_from_trade_zhengzhou_index WHERE data_time=?`
+	_, err = o.Raw(sql, dateStr).QueryRows(&list)
 	return
 }
 

services/commodity_trade_cffex.go

@@ -43,11 +43,6 @@ func cffexIndexCodeGenerator() string {
 
 // SyncRankingFromCffex 中国金融期货交易所
 func SyncRankingFromCffex() {
-	//获取所有指标信息
-	allIndex, err := models.GetBaseFromTradeCffexIndexAll()
-	if err != nil {
-		fmt.Println("select err:", err)
-	}
 	for i := 180; i > 0; i-- {
 		zzUrl := "http://www.cffex.com.cn/sj/ccpm/%s/%s/"
 		date := time.Now().AddDate(0, 0, -i)
@@ -67,6 +62,13 @@ func SyncRankingFromCffex() {
 		//获取新的指标信息
 		var sRank string
 		existIndexMap := make(map[string]*models.BaseFromTradeCffexIndex)
+
+		//获取所有指标信息
+		allIndex, err := models.GetBaseFromTradeCffexIndexAll(dateStr)
+		if err != nil {
+			fmt.Println("select err:", err)
+		}
+
 		for _, v := range allIndex {
 			indexKey := v.DealName + v.BuyName + v.SoldName + v.DataTime
 			existIndexMap[indexKey] = v

services/commodity_trade_ine.go

@@ -56,12 +56,6 @@ func IneIndexCodeGenerator() string {
 
 // SyncRankingFromIne 上海能源交易中心持单排名
 func SyncRankingFromIne() {
-	//获取所有指标信息
-	allIndex, err := models.GetBaseFromTradeIneIndexAll()
-	if err != nil {
-		fmt.Println("select err:", err)
-	}
-
 	//获取新的指标信息
 	for i := 180; i > 0; i-- {
 		var message Message
@@ -79,6 +73,12 @@ func SyncRankingFromIne() {
 		var position = message.Position
 		var tradeDate = message.ReportDate
 		existIndexMap := make(map[string]*models.BaseFromTradeIneIndex)
+
+		//获取所有指标信息
+		allIndex, err := models.GetBaseFromTradeIneIndexAll(dateStr)
+		if err != nil {
+			fmt.Println("select err:", err)
+		}
 		for _, v := range allIndex {
 			indexKey := v.DealName + v.BuyName + v.SoldName + tradeDate
 			existIndexMap[indexKey] = v

services/commodity_trade_shanghai.go

@@ -56,16 +56,10 @@ func indexCodeGenerator() string {
 
 // SyncRankingFromShangHai 上海商品交易所持单排名
 func SyncRankingFromShangHai() {
-	//获取所有指标信息
-	allIndex, err := models.GetBaseFromTradeShangHaiIndexAll()
-	if err != nil {
-		fmt.Println("select err:", err)
-	}
-
 	//获取新的指标信息
-	var message Message
-	var item = new(models.BaseFromTradeShanghaiIndex)
 	for i := 180; i > 0; i-- {
+		var message Message
+		var item = new(models.BaseFromTradeShanghaiIndex)
 		zzUrl := "http://www.shfe.com.cn/data/dailydata/kx/pm%s.dat"
 		date := time.Now().AddDate(0, 0, -i)
 		dateStr := date.Format(utils.FormatDateUnSpace)
@@ -78,6 +72,13 @@ func SyncRankingFromShangHai() {
 		err = json.Unmarshal(body, &message)
 		var position = message.Position
 		var tradeDate = message.ReportDate
+
+		//获取所有指标信息
+		allIndex, err := models.GetBaseFromTradeShangHaiIndexAll(dateStr)
+		if err != nil {
+			return
+		}
+
 		existIndexMap := make(map[string]*models.BaseFromTradeShanghaiIndex)
 		for _, v := range allIndex {
 			indexKey := v.DealName + v.BuyName + v.SoldName + v.DataTime

services/commodity_trade_zhengzhou.go

@@ -38,19 +38,8 @@ func SyncRankingFromZhengzhou() {
 		}
 	}()
 
-	//获取所有指标信息  某一天的
-	allIndex, err := models.GetBaseFromTradeZhengzhouIndexAll()
-	if err != nil {
-		return
-	}
-
-	existIndexMap := make(map[string]*models.BaseFromTradeZhengzhouIndex)
-	for _, v := range allIndex {
-		indexKey := v.DealName + v.BuyName + v.SoldName
-		existIndexMap[indexKey] = v
-	}
-	var itemVerifyCode int
 	for i := 180; i > 0; i-- {
+		var itemVerifyCode int
 		zzUrl := "http://www.czce.com.cn/cn/DFSStaticFiles/Future/%s/%s/FutureDataHolding.htm"
 		date := time.Now().AddDate(0, 0, -i)
 		year := date.Year()
@@ -62,6 +51,17 @@ func SyncRankingFromZhengzhou() {
 			fmt.Println("GetData Err:" + err.Error())
 			return
 		}
+		//获取所有指标信息  某一天的
+		allIndex, err := models.GetBaseFromTradeZhengzhouIndexAll(dateStr)
+		if err != nil {
+			return
+		}
+
+		existIndexMap := make(map[string]*models.BaseFromTradeZhengzhouIndex)
+		for _, v := range allIndex {
+			indexKey := v.DealName + v.BuyName + v.SoldName
+			existIndexMap[indexKey] = v
+		}
 		bodyStr := string(body)
 		utils.FileLog.Info(bodyStr)
 		if strings.Contains(bodyStr, "404.htm") {

services/task.go

@@ -10,10 +10,11 @@ func Task() {
 	//SyncRankingFromZhengzhou()
 
 	//SyncRankingFromShangHai()
-	SyncRankingFromIne()
+	//SyncRankingFromIne()
 	//SyncRankingFromCffex()
 
 	//SyncRankingFromDalian2()
-
+	//SyncStorageFromEic()
+	SyncStorageFromEic()
 	fmt.Println("end crawler")
 }