新增上海国际能源交易中心，重复indexCode问题优化，空列表插入问题优化

models/base_from_trade_ine.go

@@ -0,0 +1,77 @@
+package models
+
+import (
+	"rdluck_tools/orm"
+	"time"
+)
+
+type BaseFromTradeIneIndex struct {
+	BaseFromTradeIneIndexId int `orm:"column(base_from_trade_ine_index_id);pk"`
+	IndexName               string
+	IndexCode               string
+	Frequency               string
+	StartDate               time.Time
+	EndDate                 time.Time
+	CreateTime              time.Time
+	ModifyTime              time.Time
+	ClassifyName            string
+	ClassifyType            string
+}
+
+func AddBaseFromTradeIneIndex(item *BaseFromTradeIneIndex) (lastId int64, err error) {
+	o := orm.NewOrm()
+	o.Using("data")
+	lastId, err = o.Insert(item)
+	return
+}
+
+func GetBaseFromTradeIneIndexAll() (list []*BaseFromTradeIneIndex, err error) {
+	o := orm.NewOrm()
+	o.Using("data")
+	sql := `SELECT * FROM base_from_trade_ine_index `
+	_, err = o.Raw(sql).QueryRows(&list)
+	return
+}
+
+func GetCodeFromTradeIneIndexAll() (list []*BaseFromTradeIneIndex, err error) {
+	o := orm.NewOrm()
+	o.Using("data")
+	sql := `SELECT index_code FROM base_from_trade_ine_index `
+	_, err = o.Raw(sql).QueryRows(&list)
+	return
+}
+
+type BaseFromTradeIneData struct {
+	BaseFromTradeIneDataId  int `orm:"column(base_from_trade_ine_data_id);pk"`
+	BaseFromTradeIneIndexId int
+	IndexCode               string
+	DataTime                string
+	Value                   string
+	AddCutValue             string
+	CreateTime              time.Time
+	ModifyTime              time.Time
+	DataTimestamp           int64
+}
+
+func AddBaseFromTradeIneData(item *BaseFromTradeIneData) (lastId int64, err error) {
+	o := orm.NewOrm()
+	o.Using("data")
+	lastId, err = o.Insert(item)
+	return
+}
+
+func GetBaseFromTradeIneDataByIndexCode(indexCode string) (list []*BaseFromTradeIneData, err error) {
+	o := orm.NewOrm()
+	o.Using("data")
+	sql := `SELECT * FROM base_from_trade_ine_data WHERE index_code=? `
+	_, err = o.Raw(sql, indexCode).QueryRows(&list)
+	return
+}
+
+func ModifyBaseFromTradeIneDataData(value, addCutVal string, dataId int) (err error) {
+	o := orm.NewOrm()
+	o.Using("data")
+	sql := `UPDATE base_from_trade_ine_data SET value=?,add_cut_value=?,modify_time=NOW() WHERE base_from_trade_ine_data_id=? `
+	_, err = o.Raw(sql, value, addCutVal, dataId).Exec()
+	return
+}

models/base_from_trade_shanghai.go

@@ -7,15 +7,15 @@ import (
 
 type BaseFromTradeShanghaiIndex struct {
 	BaseFromTradeShangHaiIndexId int `orm:"column(base_from_trade_shanghai_index_id);pk"`
-	IndexName                     string
-	IndexCode                     string
-	Frequency                     string
-	StartDate                     time.Time
-	EndDate                       time.Time
-	CreateTime                    time.Time
-	ModifyTime                    time.Time
-	ClassifyName                  string
-	ClassifyType                  string
+	IndexName                    string
+	IndexCode                    string
+	Frequency                    string
+	StartDate                    time.Time
+	EndDate                      time.Time
+	CreateTime                   time.Time
+	ModifyTime                   time.Time
+	ClassifyName                 string
+	ClassifyType                 string
 }
 
 func AddBaseFromTradeShangHaiIndex(item *BaseFromTradeShanghaiIndex) (lastId int64, err error) {
@@ -33,16 +33,24 @@ func GetBaseFromTradeShangHaiIndexAll() (list []*BaseFromTradeShanghaiIndex, err
 	return
 }
 
+func GetCodeFromTradeShangHaiIndexAll() (list []*BaseFromTradeShanghaiIndex, err error) {
+	o := orm.NewOrm()
+	o.Using("data")
+	sql := `SELECT index_code FROM base_from_trade_shanghai_index `
+	_, err = o.Raw(sql).QueryRows(&list)
+	return
+}
+
 type BaseFromTradeShanghaiData struct {
 	BaseFromTradeShangHaiDataId  int `orm:"column(base_from_trade_shanghai_data_id);pk"`
 	BaseFromTradeShanghaiIndexId int
-	IndexCode                     string
-	DataTime                      string
-	Value                         string
-	AddCutValue                   string
-	CreateTime                    time.Time
-	ModifyTime                    time.Time
-	DataTimestamp                 int64
+	IndexCode                    string
+	DataTime                     string
+	Value                        string
+	AddCutValue                  string
+	CreateTime                   time.Time
+	ModifyTime                   time.Time
+	DataTimestamp                int64
 }
 
 func AddBaseFromTradeShangHaiData(item *BaseFromTradeShanghaiData) (lastId int64, err error) {

models/db.go

@@ -27,9 +27,11 @@ func init() {
 
 	//注册对象
 	orm.RegisterModel(
-		new(BaseFromTradeZhengzhouIndex),
-		new(BaseFromTradeZhengzhouData),
+		//new(BaseFromTradeZhengzhouIndex),
+		//new(BaseFromTradeZhengzhouData),
 		new(BaseFromTradeShanghaiIndex),
 		new(BaseFromTradeShanghaiData),
+		//new(BaseFromTradeIneIndex),
+		//new(BaseFromTradeIneData),
 	)
 }

services/commodity_trade_ine.go

@@ -0,0 +1,286 @@
+package services
+
+import (
+	"encoding/json"
+	"fmt"
+	"hongze/hongze_data_crawler/models"
+	"hongze/hongze_data_crawler/utils"
+	"rdluck_tools/http"
+	"strconv"
+	"strings"
+	"time"
+)
+
+type position []struct {
+	ContractCode     string `json:"INSTRUMENTID"`
+	ParticipantID3   string `json:"PARTICIPANTID3"`
+	ParticipantID2   string `json:"PARTICIPANTID2"`
+	ParticipantID1   string `json:"PARTICIPANTID1"`
+	ParticipantName3 string `json:"PARTICIPANTABBR3"`
+	ParticipantName2 string `json:"PARTICIPANTABBR2"`
+	Rank             int    `json:"RANK"`
+	ParticipantName1 string `json:"PARTICIPANTABBR1"`
+	BuyIn            int    `json:"CJ2"`
+	Deal             string `json:"CJ1"`
+	Change1          string `json:"CJ1_CHG"`
+	Change3          string `json:"CJ3_CHG"`
+	ProductName      string `json:"PRODUCTNAME"`
+	ProductSortNo    int    `json:"PRODUCTSORTNO"`
+	SoldOut          int    `json:"CJ3"`
+	Change2          string `json:"CJ2_CHG"`
+}
+
+type message struct {
+	Position   position `json:"o_cursor"`
+	Code       int      `json:"o_code"`
+	Msg        string   `json:"o_msg"`
+	ReportDate string   `json:"report_date"`
+	UpdateDate string   `json:"update_date"`
+}
+
+// SyncRankingFromIne 上海能源交易中心持单排名
+func SyncRankingFromIne() {
+	//获取所有指标信息
+	allIndex, err := models.GetBaseFromTradeIneIndexAll()
+	if err != nil {
+		fmt.Println("select err:", err)
+	}
+	existIndexMap := make(map[string]*models.BaseFromTradeIneIndex)
+	addIndexMap := make(map[string]string)
+	indexCodeMap := make(map[string]string)
+	for _, v := range allIndex {
+		indexKey := v.IndexName
+		existIndexMap[indexKey] = v
+	}
+
+	existDataMap := make(map[string]*models.BaseFromTradeIneData)
+
+	//获取新的指标信息
+	var message message
+	var indexCode string
+	var indexID int
+	var indexKey string
+	var dealItem = new(models.BaseFromTradeIneIndex)
+	var buyItem = new(models.BaseFromTradeIneIndex)
+	var soldItem = new(models.BaseFromTradeIneIndex)
+	var dataItem = new(models.BaseFromTradeIneData)
+	var existDataList []*models.BaseFromTradeIneData
+	var dataTime time.Time
+	var dataKey string
+	body, err := http.Get("http://www.ine.cn/data/dailydata/kx/pm20211028.dat")
+	if err != nil {
+		fmt.Println("err:", err)
+	}
+	err = json.Unmarshal(body, &message)
+	var position = message.Position
+	var tradeDate = message.ReportDate
+	//处理指标
+	for _, p := range position {
+		if p.ParticipantName1 != "" {
+			//成交量
+			dealItem.IndexName = strings.Replace(fmt.Sprintf("%s", p.ParticipantName1+"_成交量"+"_"+p.ProductName+"_"+p.ContractCode), " ", "", -1)
+			for {
+				indexCode = fmt.Sprintf("SHEE%s", time.Now().Format(utils.FormatDateTimeUnSpace)+strconv.Itoa(utils.GetRandInt(1, 100)))
+				if _, ok := indexCodeMap[indexCode]; !ok {
+					indexCodeMap[indexCode] = ""
+					break
+				}
+			}
+			dealItem.IndexCode = indexCode
+			dealItem.ClassifyName = strings.Replace(p.ProductName, " ", "", -1)
+			dealItem.ClassifyType = strings.Replace(p.ContractCode, " ", "", -1)
+			dealItem.CreateTime = time.Now()
+			dealItem.Frequency = "日度"
+			dealItem.ModifyTime = time.Now()
+			if existIndex, ok := existIndexMap[dealItem.IndexName]; !ok {
+				newID, err := models.AddBaseFromTradeIneIndex(dealItem)
+				if err != nil {
+					fmt.Println("insert error:", err)
+				}
+				indexID = int(newID)
+				fmt.Println("new indexID:", indexID)
+				addIndexMap[indexKey] = indexCode
+			} else {
+				indexID = existIndex.BaseFromTradeIneIndexId
+				indexCode = existIndex.IndexCode
+			}
+			existDataList, err = models.GetBaseFromTradeIneDataByIndexCode(indexCode)
+			if err != nil && err.Error() != utils.ErrNoRow() {
+				fmt.Println("GetBaseFromTradeIneDataByIndexCode error:", err)
+				return
+			}
+			for _, v := range existDataList {
+				key := v.IndexCode + v.DataTime
+				existDataMap[key] = v
+			}
+			dataTime, err = time.Parse(utils.FormatDateUnSpace, tradeDate)
+			if err != nil {
+				fmt.Println("time.Parse Err:" + err.Error())
+				return
+			}
+			dataKey = indexCode + dataTime.Format(utils.FormatDate)
+			if existDataItem, ok := existDataMap[dataKey]; !ok && indexCode != "" { //新增
+
+				dataItem.BaseFromTradeIneIndexId = indexID
+				dataItem.IndexCode = indexCode
+				dataItem.DataTime = tradeDate
+				dataItem.Value = p.Deal
+				dataItem.AddCutValue = p.Change1
+				dataItem.DataTimestamp = dataTime.UnixNano() / 1e6
+				dataItem.CreateTime = time.Now()
+				dataItem.ModifyTime = time.Now()
+				_, err = models.AddBaseFromTradeIneData(dataItem)
+				if err != nil {
+					fmt.Println("data insert err:", err)
+					return
+				}
+				existDataMap[indexCode+tradeDate] = dataItem
+			} else {
+				if existDataItem != nil && existDataItem.Value != p.Deal { //修改数据
+					err = models.ModifyBaseFromTradeIneDataData(p.Deal, p.Change1, existDataItem.BaseFromTradeIneDataId)
+					if err != nil {
+						fmt.Println("data update err:", err)
+					}
+				}
+			}
+		}
+
+		if p.ParticipantName2 != "" {
+			//持买单量
+			buyItem.IndexName = strings.Replace(fmt.Sprintf("%s", p.ParticipantName2+"_持买单量"+"_"+p.ProductName+"_"+p.ContractCode), " ", "", -1)
+			for {
+				indexCode = fmt.Sprintf("SHEE%s", time.Now().Format(utils.FormatDateTimeUnSpace)+strconv.Itoa(utils.GetRandInt(1, 100)))
+				if _, ok := indexCodeMap[indexCode]; !ok {
+					indexCodeMap[indexCode] = ""
+					break
+				}
+			}
+			buyItem.IndexCode = indexCode
+			buyItem.ClassifyName = strings.Replace(p.ProductName, " ", "", -1)
+			buyItem.ClassifyType = strings.Replace(p.ContractCode, " ", "", -1)
+			buyItem.CreateTime = time.Now()
+			buyItem.Frequency = "日度"
+			buyItem.ModifyTime = time.Now()
+			if existIndex, ok := existIndexMap[buyItem.IndexName]; !ok {
+				newID, err := models.AddBaseFromTradeIneIndex(buyItem)
+				if err != nil {
+					fmt.Println("insert error:", err)
+				}
+				indexID = int(newID)
+				fmt.Println("new indexID:", indexID)
+				addIndexMap[indexKey] = indexCode
+			} else {
+				indexID = existIndex.BaseFromTradeIneIndexId
+				indexCode = existIndex.IndexCode
+			}
+			existDataList, err = models.GetBaseFromTradeIneDataByIndexCode(indexCode)
+			if err != nil && err.Error() != utils.ErrNoRow() {
+				return
+			}
+			for _, v := range existDataList {
+				key := v.IndexCode + v.DataTime
+				existDataMap[key] = v
+			}
+			dataTime, err := time.Parse(utils.FormatDateUnSpace, tradeDate)
+			if err != nil {
+				fmt.Println("time.Parse Err:" + err.Error())
+				return
+			}
+			dataKey = indexCode + dataTime.Format(utils.FormatDate)
+			if existDataItem, ok := existDataMap[dataKey]; !ok && indexCode != "" { //新增
+
+				dataItem.BaseFromTradeIneIndexId = indexID
+				dataItem.IndexCode = indexCode
+				dataItem.DataTime = tradeDate
+				dataItem.Value = fmt.Sprintf("%d", p.BuyIn)
+				dataItem.AddCutValue = p.Change2
+				dataItem.DataTimestamp = dataTime.UnixNano() / 1e6
+				dataItem.CreateTime = time.Now()
+				dataItem.ModifyTime = time.Now()
+				_, err = models.AddBaseFromTradeIneData(dataItem)
+				if err != nil {
+					fmt.Println("data insert err:", err)
+					return
+				}
+				existDataMap[indexCode+tradeDate] = dataItem
+			} else {
+				if existDataItem != nil && existDataItem.Value != fmt.Sprintf("%d", p.BuyIn) { //修改数据
+					err = models.ModifyBaseFromTradeIneDataData(fmt.Sprintf("%d", p.BuyIn), p.Change2, existDataItem.BaseFromTradeIneDataId)
+					if err != nil {
+						fmt.Println("data update err:", err)
+					}
+				}
+			}
+		}
+
+		if p.ParticipantName3 != "" {
+			//持卖单量
+			soldItem.IndexName = strings.Replace(fmt.Sprintf("%s", p.ParticipantName3+"_持卖单量"+"_"+p.ProductName+"_"+p.ContractCode), " ", "", -1)
+			for {
+				indexCode = fmt.Sprintf("SHEE%s", time.Now().Format(utils.FormatDateTimeUnSpace)+strconv.Itoa(utils.GetRandInt(1, 100)))
+				if _, ok := indexCodeMap[indexCode]; !ok {
+					indexCodeMap[indexCode] = ""
+					break
+				}
+			}
+			soldItem.IndexCode = indexCode
+			soldItem.ClassifyName = strings.Replace(p.ProductName, " ", "", -1)
+			soldItem.ClassifyType = strings.Replace(p.ContractCode, " ", "", -1)
+			soldItem.CreateTime = time.Now()
+			soldItem.Frequency = "日度"
+			soldItem.ModifyTime = time.Now()
+			if existIndex, ok := existIndexMap[soldItem.IndexName]; !ok {
+				newID, err := models.AddBaseFromTradeIneIndex(soldItem)
+				if err != nil {
+					fmt.Println("insert error:", err)
+				}
+				indexID = int(newID)
+				fmt.Println("new indexID:", indexID)
+				addIndexMap[indexKey] = indexCode
+			} else {
+				indexID = existIndex.BaseFromTradeIneIndexId
+				indexCode = existIndex.IndexCode
+			}
+			existDataList, err = models.GetBaseFromTradeIneDataByIndexCode(indexCode)
+			if err != nil && err.Error() != utils.ErrNoRow() {
+				fmt.Println("GetBaseFromTradeIneDataByIndexCode error:", err)
+				return
+			}
+			for _, v := range existDataList {
+				key := v.IndexCode + v.DataTime
+				existDataMap[key] = v
+			}
+			dataTime, err = time.Parse(utils.FormatDateUnSpace, tradeDate)
+			if err != nil {
+				fmt.Println("time.Parse Err:" + err.Error())
+				return
+			}
+			dataKey = indexCode + dataTime.Format(utils.FormatDate)
+			if existDataItem, ok := existDataMap[dataKey]; !ok && indexCode != "" { //新增
+
+				dataItem.BaseFromTradeIneIndexId = indexID
+				dataItem.IndexCode = indexCode
+				dataItem.DataTime = tradeDate
+				dataItem.Value = fmt.Sprintf("%d", p.SoldOut)
+				dataItem.AddCutValue = p.Change3
+				dataItem.DataTimestamp = dataTime.UnixNano() / 1e6
+				dataItem.CreateTime = time.Now()
+				dataItem.ModifyTime = time.Now()
+				_, err = models.AddBaseFromTradeIneData(dataItem)
+				if err != nil {
+					fmt.Println("data insert err:", err)
+					return
+				}
+				existDataMap[indexCode+tradeDate] = dataItem
+			} else {
+				if existDataItem != nil && existDataItem.Value != fmt.Sprintf("%d", p.SoldOut) { //修改数据
+					err = models.ModifyBaseFromTradeIneDataData(fmt.Sprintf("%d", p.SoldOut), p.Change3, existDataItem.BaseFromTradeIneDataId)
+					if err != nil {
+						fmt.Println("data update err:", err)
+					}
+				}
+			}
+		}
+	}
+	fmt.Println("end")
+}

services/commodity_trade_shanghai.go

@@ -49,6 +49,8 @@ func SyncRankingFromShangHai() {
 	}
 	existIndexMap := make(map[string]*models.BaseFromTradeShanghaiIndex)
 	addIndexMap := make(map[string]string)
+	indexCodeMap := make(map[string]string)
+
 	for _, v := range allIndex {
 		indexKey := v.IndexName
 		existIndexMap[indexKey] = v
@@ -78,190 +80,212 @@ func SyncRankingFromShangHai() {
 	//处理指标
 	for _, p := range position {
 		//排除标题行和最后一行
-		if p.Rank > 0 && p.ParticipantID1 != "" && p.ParticipantID2 != "" && p.ParticipantID3 != "" {
-			//成交量
-			dealItem.IndexName = strings.Replace(fmt.Sprintf("%s", p.ParticipantName1+"_成交量"+"_"+p.ProductName+"_"+p.ContractCode), " ", "", -1)
-			dealItem.IndexCode = fmt.Sprintf("SH%s", time.Now().Format(utils.FormatDateTimeUnSpace)+strconv.Itoa(utils.GetRandInt(1, 1000)))
-			dealItem.ClassifyName = strings.Replace(p.ProductName, " ", "", -1)
-			dealItem.ClassifyType = strings.Replace(p.ContractCode, " ", "", -1)
-			dealItem.CreateTime = time.Now()
-			dealItem.Frequency = "日度"
-			dealItem.ModifyTime = time.Now()
-			if existIndex, ok := existIndexMap[dealItem.IndexName]; !ok {
-				newID, err := models.AddBaseFromTradeShangHaiIndex(dealItem)
-				if err != nil {
-					fmt.Println("insert error:", err)
+		if p.Rank > 0 {
+			if p.ParticipantName1 != "" {
+				//成交量
+				dealItem.IndexName = strings.Replace(fmt.Sprintf("%s", p.ParticipantName1+"_成交量"+"_"+p.ProductName+"_"+p.ContractCode), " ", "", -1)
+				indexCode = fmt.Sprintf("SH%s", time.Now().Format(utils.FormatDateTimeUnSpace)+strconv.Itoa(utils.GetRandInt(1, 100)))
+				for {
+					indexCode = fmt.Sprintf("SH%s", time.Now().Format(utils.FormatDateTimeUnSpace)+strconv.Itoa(utils.GetRandInt(1, 100)))
+					if _, ok := indexCodeMap[indexCode]; !ok {
+						indexCodeMap[indexCode] = ""
+						break
+					}
 				}
-				indexID = int(newID)
-				fmt.Println("new indexID:", indexID)
-				addIndexMap[indexKey] = indexCode
-				time.Sleep(100 * time.Millisecond)
-			} else {
-				indexID = existIndex.BaseFromTradeShangHaiIndexId
-				indexCode = existIndex.IndexCode
-			}
-			existDataList, err = models.GetBaseFromTradeShangHaiDataByIndexCode(indexCode)
-			if err != nil && err.Error() != utils.ErrNoRow() {
-				fmt.Println("GetBaseFromTradeShangHaiDataByIndexCode error:", err)
-				return
-			}
-			for _, v := range existDataList {
-				key := v.IndexCode + v.DataTime
-				existDataMap[key] = v
-			}
-			dataTime, err = time.Parse(utils.FormatDateUnSpace, tradeDate)
-			if err != nil {
-				fmt.Println("time.Parse Err:" + err.Error())
-				return
-			}
-			dataKey = indexCode + dataTime.Format(utils.FormatDate)
-			if existDataItem, ok := existDataMap[dataKey]; !ok && indexCode != "" { //新增
-
-				dataItem.BaseFromTradeShanghaiIndexId = indexID
-				dataItem.IndexCode = indexCode
-				dataItem.DataTime = tradeDate
-				dataItem.Value = fmt.Sprintf("%d", p.Deal)
-				dataItem.AddCutValue = fmt.Sprintf("%d", p.Change1)
-				dataItem.DataTimestamp = dataTime.UnixNano() / 1e6
-				dataItem.CreateTime = time.Now()
-				dataItem.ModifyTime = time.Now()
-				_, err = models.AddBaseFromTradeShangHaiData(dataItem)
+				dealItem.IndexCode = indexCode
+				dealItem.ClassifyName = strings.Replace(p.ProductName, " ", "", -1)
+				dealItem.ClassifyType = strings.Replace(p.ContractCode, " ", "", -1)
+				dealItem.CreateTime = time.Now()
+				dealItem.Frequency = "日度"
+				dealItem.ModifyTime = time.Now()
+				if existIndex, ok := existIndexMap[dealItem.IndexName]; !ok {
+					newID, err := models.AddBaseFromTradeShangHaiIndex(dealItem)
+					if err != nil {
+						fmt.Println("insert error:", err)
+					}
+					indexID = int(newID)
+					fmt.Println("new indexID:", indexID)
+					addIndexMap[indexKey] = indexCode
+				} else {
+					indexID = existIndex.BaseFromTradeShangHaiIndexId
+					indexCode = existIndex.IndexCode
+				}
+				existDataList, err = models.GetBaseFromTradeShangHaiDataByIndexCode(indexCode)
+				if err != nil && err.Error() != utils.ErrNoRow() {
+					fmt.Println("GetBaseFromTradeShangHaiDataByIndexCode error:", err)
+					return
+				}
+				for _, v := range existDataList {
+					key := v.IndexCode + v.DataTime
+					existDataMap[key] = v
+				}
+				dataTime, err = time.Parse(utils.FormatDateUnSpace, tradeDate)
 				if err != nil {
-					fmt.Println("data insert err:", err)
+					fmt.Println("time.Parse Err:" + err.Error())
 					return
 				}
-				existDataMap[indexCode+tradeDate] = dataItem
-			} else {
-				if existDataItem != nil && existDataItem.Value != fmt.Sprintf("%d", p.Deal) { //修改数据
-					err = models.ModifyBaseFromTradeShangHaiData(fmt.Sprintf("%d", p.Deal), fmt.Sprintf("%d", p.Change1), existDataItem.BaseFromTradeShangHaiDataId)
+				dataKey = indexCode + dataTime.Format(utils.FormatDate)
+				if existDataItem, ok := existDataMap[dataKey]; !ok && indexCode != "" { //新增
+
+					dataItem.BaseFromTradeShanghaiIndexId = indexID
+					dataItem.IndexCode = indexCode
+					dataItem.DataTime = tradeDate
+					dataItem.Value = fmt.Sprintf("%d", p.Deal)
+					dataItem.AddCutValue = fmt.Sprintf("%d", p.Change1)
+					dataItem.DataTimestamp = dataTime.UnixNano() / 1e6
+					dataItem.CreateTime = time.Now()
+					dataItem.ModifyTime = time.Now()
+					_, err = models.AddBaseFromTradeShangHaiData(dataItem)
 					if err != nil {
-						fmt.Println("data update err:", err)
+						fmt.Println("data insert err:", err)
+						return
+					}
+					existDataMap[indexCode+tradeDate] = dataItem
+				} else {
+					if existDataItem != nil && existDataItem.Value != fmt.Sprintf("%d", p.Deal) { //修改数据
+						err = models.ModifyBaseFromTradeShangHaiData(fmt.Sprintf("%d", p.Deal), fmt.Sprintf("%d", p.Change1), existDataItem.BaseFromTradeShangHaiDataId)
+						if err != nil {
+							fmt.Println("data update err:", err)
+						}
 					}
 				}
 			}
-
-			//持买单量
-			buyItem.IndexName = strings.Replace(fmt.Sprintf("%s", p.ParticipantName2+"_持买单量"+"_"+p.ProductName+"_"+p.ContractCode), " ", "", -1)
-			buyItem.IndexCode = fmt.Sprintf("SH%s", time.Now().Format(utils.FormatDateTimeUnSpace)+strconv.Itoa(utils.GetRandInt(1, 1000)))
-			buyItem.ClassifyName = strings.Replace(p.ProductName, " ", "", -1)
-			buyItem.ClassifyType = strings.Replace(p.ContractCode, " ", "", -1)
-			buyItem.CreateTime = time.Now()
-			buyItem.Frequency = "日度"
-			buyItem.ModifyTime = time.Now()
-			if existIndex, ok := existIndexMap[buyItem.IndexName]; !ok {
-				newID, err := models.AddBaseFromTradeShangHaiIndex(buyItem)
-				if err != nil {
-					fmt.Println("insert error:", err)
+			if p.ParticipantName2 != "" {
+				//持买单量
+				buyItem.IndexName = strings.Replace(fmt.Sprintf("%s", p.ParticipantName2+"_持买单量"+"_"+p.ProductName+"_"+p.ContractCode), " ", "", -1)
+				for {
+					indexCode = fmt.Sprintf("SH%s", time.Now().Format(utils.FormatDateTimeUnSpace)+strconv.Itoa(utils.GetRandInt(1, 100)))
+					if _, ok := indexCodeMap[indexCode]; !ok {
+						indexCodeMap[indexCode] = ""
+						break
+					}
 				}
-				indexID = int(newID)
-				fmt.Println("new indexID:", indexID)
-				addIndexMap[indexKey] = indexCode
-				time.Sleep(100 * time.Millisecond)
-			} else {
-				indexID = existIndex.BaseFromTradeShangHaiIndexId
-				indexCode = existIndex.IndexCode
-			}
-			existDataList, err = models.GetBaseFromTradeShangHaiDataByIndexCode(indexCode)
-			if err != nil && err.Error() != utils.ErrNoRow() {
-				return
-			}
-			for _, v := range existDataList {
-				key := v.IndexCode + v.DataTime
-				existDataMap[key] = v
-			}
-			dataTime, err := time.Parse(utils.FormatDateUnSpace, tradeDate)
-			if err != nil {
-				fmt.Println("time.Parse Err:" + err.Error())
-				return
-			}
-			dataKey = indexCode + dataTime.Format(utils.FormatDate)
-			if existDataItem, ok := existDataMap[dataKey]; !ok && indexCode != "" { //新增
-
-				dataItem.BaseFromTradeShanghaiIndexId = indexID
-				dataItem.IndexCode = indexCode
-				dataItem.DataTime = tradeDate
-				dataItem.Value = fmt.Sprintf("%d", p.BuyIn)
-				dataItem.AddCutValue = fmt.Sprintf("%d", p.Change2)
-				dataItem.DataTimestamp = dataTime.UnixNano() / 1e6
-				dataItem.CreateTime = time.Now()
-				dataItem.ModifyTime = time.Now()
-				_, err = models.AddBaseFromTradeShangHaiData(dataItem)
+				buyItem.IndexCode = indexCode
+				buyItem.ClassifyName = strings.Replace(p.ProductName, " ", "", -1)
+				buyItem.ClassifyType = strings.Replace(p.ContractCode, " ", "", -1)
+				buyItem.CreateTime = time.Now()
+				buyItem.Frequency = "日度"
+				buyItem.ModifyTime = time.Now()
+				if existIndex, ok := existIndexMap[buyItem.IndexName]; !ok {
+					newID, err := models.AddBaseFromTradeShangHaiIndex(buyItem)
+					if err != nil {
+						fmt.Println("insert error:", err)
+					}
+					indexID = int(newID)
+					fmt.Println("new indexID:", indexID)
+					addIndexMap[indexKey] = indexCode
+				} else {
+					indexID = existIndex.BaseFromTradeShangHaiIndexId
+					indexCode = existIndex.IndexCode
+				}
+				existDataList, err = models.GetBaseFromTradeShangHaiDataByIndexCode(indexCode)
+				if err != nil && err.Error() != utils.ErrNoRow() {
+					return
+				}
+				for _, v := range existDataList {
+					key := v.IndexCode + v.DataTime
+					existDataMap[key] = v
+				}
+				dataTime, err := time.Parse(utils.FormatDateUnSpace, tradeDate)
 				if err != nil {
-					fmt.Println("data insert err:", err)
+					fmt.Println("time.Parse Err:" + err.Error())
 					return
 				}
-				existDataMap[indexCode+tradeDate] = dataItem
-			} else {
-				if existDataItem != nil && existDataItem.Value != fmt.Sprintf("%d", p.BuyIn) { //修改数据
-					err = models.ModifyBaseFromTradeShangHaiData(fmt.Sprintf("%d", p.BuyIn), fmt.Sprintf("%d", p.Change2), existDataItem.BaseFromTradeShangHaiDataId)
+				dataKey = indexCode + dataTime.Format(utils.FormatDate)
+				if existDataItem, ok := existDataMap[dataKey]; !ok && indexCode != "" { //新增
+
+					dataItem.BaseFromTradeShanghaiIndexId = indexID
+					dataItem.IndexCode = indexCode
+					dataItem.DataTime = tradeDate
+					dataItem.Value = fmt.Sprintf("%d", p.BuyIn)
+					dataItem.AddCutValue = fmt.Sprintf("%d", p.Change2)
+					dataItem.DataTimestamp = dataTime.UnixNano() / 1e6
+					dataItem.CreateTime = time.Now()
+					dataItem.ModifyTime = time.Now()
+					_, err = models.AddBaseFromTradeShangHaiData(dataItem)
 					if err != nil {
-						fmt.Println("data update err:", err)
+						fmt.Println("data insert err:", err)
+						return
+					}
+					existDataMap[indexCode+tradeDate] = dataItem
+				} else {
+					if existDataItem != nil && existDataItem.Value != fmt.Sprintf("%d", p.BuyIn) { //修改数据
+						err = models.ModifyBaseFromTradeShangHaiData(fmt.Sprintf("%d", p.BuyIn), fmt.Sprintf("%d", p.Change2), existDataItem.BaseFromTradeShangHaiDataId)
+						if err != nil {
+							fmt.Println("data update err:", err)
+						}
 					}
 				}
 			}
 
-			//持卖单量
-			soldItem.IndexName = strings.Replace(fmt.Sprintf("%s", p.ParticipantName3+"_持卖单量"+"_"+p.ProductName+"_"+p.ContractCode), " ", "", -1)
-			soldItem.IndexCode = fmt.Sprintf("SH%s", time.Now().Format(utils.FormatDateTimeUnSpace)+strconv.Itoa(utils.GetRandInt(1, 1000)))
-			soldItem.ClassifyName = strings.Replace(p.ProductName, " ", "", -1)
-			soldItem.ClassifyType = strings.Replace(p.ContractCode, " ", "", -1)
-			soldItem.CreateTime = time.Now()
-			soldItem.Frequency = "日度"
-			soldItem.ModifyTime = time.Now()
-			if existIndex, ok := existIndexMap[soldItem.IndexName]; !ok {
-				newID, err := models.AddBaseFromTradeShangHaiIndex(soldItem)
-				if err != nil {
-					fmt.Println("insert error:", err)
+			if p.ParticipantName3 != "" {
+				//持卖单量
+				soldItem.IndexName = strings.Replace(fmt.Sprintf("%s", p.ParticipantName3+"_持卖单量"+"_"+p.ProductName+"_"+p.ContractCode), " ", "", -1)
+				for {
+					indexCode = fmt.Sprintf("SH%s", time.Now().Format(utils.FormatDateTimeUnSpace)+strconv.Itoa(utils.GetRandInt(1, 100)))
+					if _, ok := indexCodeMap[indexCode]; !ok {
+						indexCodeMap[indexCode] = ""
+						break
+					}
 				}
-				indexID = int(newID)
-				fmt.Println("new indexID:", indexID)
-				addIndexMap[indexKey] = indexCode
-				time.Sleep(100 * time.Millisecond)
-			} else {
-				indexID = existIndex.BaseFromTradeShangHaiIndexId
-				indexCode = existIndex.IndexCode
-			}
-			existDataList, err = models.GetBaseFromTradeShangHaiDataByIndexCode(indexCode)
-			if err != nil && err.Error() != utils.ErrNoRow() {
-				fmt.Println("GetBaseFromTradeShangHaiDataByIndexCode error:", err)
-				return
-			}
-			for _, v := range existDataList {
-				key := v.IndexCode + v.DataTime
-				existDataMap[key] = v
-			}
-			dataTime, err = time.Parse(utils.FormatDateUnSpace, tradeDate)
-			if err != nil {
-				fmt.Println("time.Parse Err:" + err.Error())
-				return
-			}
-			dataKey = indexCode + dataTime.Format(utils.FormatDate)
-			if existDataItem, ok := existDataMap[dataKey]; !ok && indexCode != "" { //新增
-
-				dataItem.BaseFromTradeShanghaiIndexId = indexID
-				dataItem.IndexCode = indexCode
-				dataItem.DataTime = tradeDate
-				dataItem.Value = fmt.Sprintf("%d", p.SoldOut)
-				dataItem.AddCutValue = fmt.Sprintf("%d", p.Change3)
-				dataItem.DataTimestamp = dataTime.UnixNano() / 1e6
-				dataItem.CreateTime = time.Now()
-				dataItem.ModifyTime = time.Now()
-				_, err = models.AddBaseFromTradeShangHaiData(dataItem)
+				soldItem.IndexCode = indexCode
+				soldItem.ClassifyName = strings.Replace(p.ProductName, " ", "", -1)
+				soldItem.ClassifyType = strings.Replace(p.ContractCode, " ", "", -1)
+				soldItem.CreateTime = time.Now()
+				soldItem.Frequency = "日度"
+				soldItem.ModifyTime = time.Now()
+				if existIndex, ok := existIndexMap[soldItem.IndexName]; !ok {
+					newID, err := models.AddBaseFromTradeShangHaiIndex(soldItem)
+					if err != nil {
+						fmt.Println("insert error:", err)
+					}
+					indexID = int(newID)
+					fmt.Println("new indexID:", indexID)
+					addIndexMap[indexKey] = indexCode
+				} else {
+					indexID = existIndex.BaseFromTradeShangHaiIndexId
+					indexCode = existIndex.IndexCode
+				}
+				existDataList, err = models.GetBaseFromTradeShangHaiDataByIndexCode(indexCode)
+				if err != nil && err.Error() != utils.ErrNoRow() {
+					fmt.Println("GetBaseFromTradeShangHaiDataByIndexCode error:", err)
+					return
+				}
+				for _, v := range existDataList {
+					key := v.IndexCode + v.DataTime
+					existDataMap[key] = v
+				}
+				dataTime, err = time.Parse(utils.FormatDateUnSpace, tradeDate)
 				if err != nil {
-					fmt.Println("data insert err:", err)
+					fmt.Println("time.Parse Err:" + err.Error())
 					return
 				}
-				existDataMap[indexCode+tradeDate] = dataItem
-			} else {
-				if existDataItem != nil && existDataItem.Value != fmt.Sprintf("%d", p.SoldOut) { //修改数据
-					err = models.ModifyBaseFromTradeShangHaiData(fmt.Sprintf("%d", p.SoldOut), fmt.Sprintf("%d", p.Change3), existDataItem.BaseFromTradeShangHaiDataId)
+				dataKey = indexCode + dataTime.Format(utils.FormatDate)
+				if existDataItem, ok := existDataMap[dataKey]; !ok && indexCode != "" { //新增
+
+					dataItem.BaseFromTradeShanghaiIndexId = indexID
+					dataItem.IndexCode = indexCode
+					dataItem.DataTime = tradeDate
+					dataItem.Value = fmt.Sprintf("%d", p.SoldOut)
+					dataItem.AddCutValue = fmt.Sprintf("%d", p.Change3)
+					dataItem.DataTimestamp = dataTime.UnixNano() / 1e6
+					dataItem.CreateTime = time.Now()
+					dataItem.ModifyTime = time.Now()
+					_, err = models.AddBaseFromTradeShangHaiData(dataItem)
 					if err != nil {
-						fmt.Println("data update err:", err)
+						fmt.Println("data insert err:", err)
+						return
+					}
+					existDataMap[indexCode+tradeDate] = dataItem
+				} else {
+					if existDataItem != nil && existDataItem.Value != fmt.Sprintf("%d", p.SoldOut) { //修改数据
+						err = models.ModifyBaseFromTradeShangHaiData(fmt.Sprintf("%d", p.SoldOut), fmt.Sprintf("%d", p.Change3), existDataItem.BaseFromTradeShangHaiDataId)
+						if err != nil {
+							fmt.Println("data update err:", err)
+						}
 					}
 				}
 			}
-
-
 		}
 	}
 	fmt.Println("end")

services/task.go

@@ -8,6 +8,7 @@ func Task() {
 	fmt.Println("start crawler")
 	//同步郑州商品交易所排名
 	//SyncRankingFromZhenzhou()
-	//SyncRankingFromShangHai()
+	SyncRankingFromShangHai()
+	//SyncRankingFromIne()
 	fmt.Println("end crawler")
 }