交易所爬虫优化

models/base_from_trade_cffex.go

@@ -7,15 +7,28 @@ import (
 
 type BaseFromTradeCffexIndex struct {
 	BaseFromTradeCffexIndexId int `orm:"column(base_from_trade_cffex_index_id);pk"`
-	IndexName               string
-	IndexCode               string
-	Frequency               string
-	StartDate               time.Time
-	EndDate                 time.Time
-	CreateTime              time.Time
-	ModifyTime              time.Time
-	ClassifyName            string
-	ClassifyType            string
+	Rank                         int
+	DealShortName                string
+	DealName                     string
+	DealCode                     string
+	DealValue                    int
+	DealChange                   int
+	BuyShortName                 string
+	BuyName                      string
+	BuyCode                      string
+	BuyValue                     int
+	BuyChange                    int
+	SoldShortName                string
+	SoldName                     string
+	SoldCode                     string
+	SoldValue                    int
+	SoldChange                   int
+	Frequency                    string
+	ClassifyName                 string
+	ClassifyType                 string
+	CreateTime                   time.Time
+	ModifyTime                   time.Time
+	DataTime                     string
 }
 
 func AddBaseFromTradeCffexIndex(item *BaseFromTradeCffexIndex) (lastId int64, err error) {
@@ -28,50 +41,17 @@ func AddBaseFromTradeCffexIndex(item *BaseFromTradeCffexIndex) (lastId int64, er
 func GetBaseFromTradeCffexIndexAll() (list []*BaseFromTradeCffexIndex, err error) {
 	o := orm.NewOrm()
 	o.Using("data")
-	sql := `SELECT * FROM base_from_trade_Cffex_index `
+	sql := `SELECT * FROM base_from_trade_cffex_index `
 	_, err = o.Raw(sql).QueryRows(&list)
 	return
 }
 
-func GetCodeFromTradeCffexIndexAll() (list []*BaseFromTradeCffexIndex, err error) {
+func ModifyBaseFromTradeCffexIndex(columnList [5]string, dataList [5]interface{}, dataId int) (err error) {
 	o := orm.NewOrm()
 	o.Using("data")
-	sql := `SELECT index_code FROM base_from_trade_Cffex_index `
-	_, err = o.Raw(sql).QueryRows(&list)
-	return
-}
-
-type BaseFromTradeCffexData struct {
-	BaseFromTradeCffexDataId  int `orm:"column(base_from_trade_Cffex_data_id);pk"`
-	BaseFromTradeCffexIndexId int
-	IndexCode               string
-	DataTime                string
-	Value                   string
-	AddCutValue             string
-	CreateTime              time.Time
-	ModifyTime              time.Time
-	DataTimestamp           int64
-}
-
-func AddBaseFromTradeCffexData(item *BaseFromTradeCffexData) (lastId int64, err error) {
-	o := orm.NewOrm()
-	o.Using("data")
-	lastId, err = o.Insert(item)
-	return
-}
-
-func GetBaseFromTradeCffexDataByIndexCode(indexCode string) (list []*BaseFromTradeCffexData, err error) {
-	o := orm.NewOrm()
-	o.Using("data")
-	sql := `SELECT * FROM base_from_trade_Cffex_data WHERE index_code=? `
-	_, err = o.Raw(sql, indexCode).QueryRows(&list)
-	return
-}
-
-func ModifyBaseFromTradeCffexDataData(value, addCutVal string, dataId int) (err error) {
-	o := orm.NewOrm()
-	o.Using("data")
-	sql := `UPDATE base_from_trade_Cffex_data SET value=?,add_cut_value=?,modify_time=NOW() WHERE base_from_trade_Cffex_data_id=? `
-	_, err = o.Raw(sql, value, addCutVal, dataId).Exec()
+	sql := "UPDATE base_from_trade_cffex_index SET "+
+		columnList[0]+"=?,"+columnList[1]+"=?,"+columnList[2]+"=?,"+columnList[3]+"=?,"+columnList[4]+
+		"=?,modify_time=NOW() WHERE base_from_trade_cffex_index_id=? "
+	_, err = o.Raw(sql, dataList[0], dataList[1], dataList[2], dataList[3], dataList[4], dataId).Exec()
 	return
-}
+}

models/base_from_trade_ine.go

@@ -7,15 +7,28 @@ import (
 
 type BaseFromTradeIneIndex struct {
 	BaseFromTradeIneIndexId int `orm:"column(base_from_trade_ine_index_id);pk"`
-	IndexName               string
-	IndexCode               string
-	Frequency               string
-	StartDate               time.Time
-	EndDate                 time.Time
-	CreateTime              time.Time
-	ModifyTime              time.Time
-	ClassifyName            string
-	ClassifyType            string
+	Rank                         int
+	DealShortName                string
+	DealName                     string
+	DealCode                     string
+	DealValue                    int
+	DealChange                   int
+	BuyShortName                 string
+	BuyName                      string
+	BuyCode                      string
+	BuyValue                     int
+	BuyChange                    int
+	SoldShortName                string
+	SoldName                     string
+	SoldCode                     string
+	SoldValue                    int
+	SoldChange                   int
+	Frequency                    string
+	ClassifyName                 string
+	ClassifyType                 string
+	CreateTime                   time.Time
+	ModifyTime                   time.Time
+	DataTime                     string
 }
 
 func AddBaseFromTradeIneIndex(item *BaseFromTradeIneIndex) (lastId int64, err error) {
@@ -33,45 +46,10 @@ func GetBaseFromTradeIneIndexAll() (list []*BaseFromTradeIneIndex, err error) {
 	return
 }
 
-func GetCodeFromTradeIneIndexAll() (list []*BaseFromTradeIneIndex, err error) {
+func ModifyBaseFromTradeIneIndex(dealValue, buyValue, soldValue int, dataId int) (err error) {
 	o := orm.NewOrm()
 	o.Using("data")
-	sql := `SELECT index_code FROM base_from_trade_ine_index `
-	_, err = o.Raw(sql).QueryRows(&list)
-	return
-}
-
-type BaseFromTradeIneData struct {
-	BaseFromTradeIneDataId  int `orm:"column(base_from_trade_ine_data_id);pk"`
-	BaseFromTradeIneIndexId int
-	IndexCode               string
-	DataTime                string
-	Value                   string
-	AddCutValue             string
-	CreateTime              time.Time
-	ModifyTime              time.Time
-	DataTimestamp           int64
-}
-
-func AddBaseFromTradeIneData(item *BaseFromTradeIneData) (lastId int64, err error) {
-	o := orm.NewOrm()
-	o.Using("data")
-	lastId, err = o.Insert(item)
-	return
-}
-
-func GetBaseFromTradeIneDataByIndexCode(indexCode string) (list []*BaseFromTradeIneData, err error) {
-	o := orm.NewOrm()
-	o.Using("data")
-	sql := `SELECT * FROM base_from_trade_ine_data WHERE index_code=? `
-	_, err = o.Raw(sql, indexCode).QueryRows(&list)
-	return
-}
-
-func ModifyBaseFromTradeIneDataData(value, addCutVal string, dataId int) (err error) {
-	o := orm.NewOrm()
-	o.Using("data")
-	sql := `UPDATE base_from_trade_ine_data SET value=?,add_cut_value=?,modify_time=NOW() WHERE base_from_trade_ine_data_id=? `
-	_, err = o.Raw(sql, value, addCutVal, dataId).Exec()
+	sql := `UPDATE base_from_trade_ine_index SET deal_value=?,buy_value=?,sold_value=?,modify_time=NOW() WHERE base_from_trade_ine_index_id=? `
+	_, err = o.Raw(sql, dealValue, buyValue, soldValue, dataId).Exec()
 	return
 }

models/base_from_trade_shanghai.go

@@ -7,15 +7,28 @@ import (
 
 type BaseFromTradeShanghaiIndex struct {
 	BaseFromTradeShangHaiIndexId int `orm:"column(base_from_trade_shanghai_index_id);pk"`
-	IndexName                    string
-	IndexCode                    string
+	Rank                         int
+	DealShortName                string
+	DealName                     string
+	DealCode                     string
+	DealValue                    int
+	DealChange                   int
+	BuyShortName                 string
+	BuyName                      string
+	BuyCode                      string
+	BuyValue                     int
+	BuyChange                    int
+	SoldShortName                string
+	SoldName                     string
+	SoldCode                     string
+	SoldValue                    int
+	SoldChange                   int
 	Frequency                    string
-	StartDate                    time.Time
-	EndDate                      time.Time
-	CreateTime                   time.Time
-	ModifyTime                   time.Time
 	ClassifyName                 string
 	ClassifyType                 string
+	CreateTime                   time.Time
+	ModifyTime                   time.Time
+	DataTime                     string
 }
 
 func AddBaseFromTradeShangHaiIndex(item *BaseFromTradeShanghaiIndex) (lastId int64, err error) {
@@ -33,45 +46,10 @@ func GetBaseFromTradeShangHaiIndexAll() (list []*BaseFromTradeShanghaiIndex, err
 	return
 }
 
-func GetCodeFromTradeShangHaiIndexAll() (list []*BaseFromTradeShanghaiIndex, err error) {
-	o := orm.NewOrm()
-	o.Using("data")
-	sql := `SELECT index_code FROM base_from_trade_shanghai_index `
-	_, err = o.Raw(sql).QueryRows(&list)
-	return
-}
-
-type BaseFromTradeShanghaiData struct {
-	BaseFromTradeShangHaiDataId  int `orm:"column(base_from_trade_shanghai_data_id);pk"`
-	BaseFromTradeShanghaiIndexId int
-	IndexCode                    string
-	DataTime                     string
-	Value                        string
-	AddCutValue                  string
-	CreateTime                   time.Time
-	ModifyTime                   time.Time
-	DataTimestamp                int64
-}
-
-func AddBaseFromTradeShangHaiData(item *BaseFromTradeShanghaiData) (lastId int64, err error) {
-	o := orm.NewOrm()
-	o.Using("data")
-	lastId, err = o.Insert(item)
-	return
-}
-
-func GetBaseFromTradeShangHaiDataByIndexCode(indexCode string) (list []*BaseFromTradeShanghaiData, err error) {
-	o := orm.NewOrm()
-	o.Using("data")
-	sql := `SELECT * FROM base_from_trade_shanghai_data WHERE index_code=? `
-	_, err = o.Raw(sql, indexCode).QueryRows(&list)
-	return
-}
-
-func ModifyBaseFromTradeShangHaiData(value, addCutVal string, dataId int) (err error) {
+func ModifyBaseFromTradeShangHaiIndex(dealValue, buyValue, soldValue int, dataId int) (err error) {
 	o := orm.NewOrm()
 	o.Using("data")
-	sql := `UPDATE base_from_trade_shanghai_data SET value=?,add_cut_value=?,modify_time=NOW() WHERE base_from_trade_shanghai_data_id=? `
-	_, err = o.Raw(sql, value, addCutVal, dataId).Exec()
+	sql := `UPDATE base_from_trade_shanghai_index SET deal_value=?,buy_value=?,sold_value=?,modify_time=NOW() WHERE base_from_trade_shanghai_index_id=? `
+	_, err = o.Raw(sql, dealValue, buyValue, soldValue, dataId).Exec()
 	return
 }

models/base_from_trade_zhengzhou.go

@@ -7,15 +7,28 @@ import (
 
 type BaseFromTradeZhengzhouIndex struct {
 	BaseFromTradeZhengzhouIndexId int `orm:"column(base_from_trade_zhengzhou_index_id);pk"`
-	IndexName                     string
-	IndexCode                     string
-	Frequency                     string
-	StartDate                     time.Time
-	EndDate                       time.Time
-	CreateTime                    time.Time
-	ModifyTime                    time.Time
-	ClassifyName                  string
-	ClassifyType                  string
+	Rank                         int
+	DealShortName                string
+	DealName                     string
+	DealCode                     string
+	DealValue                    int
+	DealChange                   int
+	BuyShortName                 string
+	BuyName                      string
+	BuyCode                      string
+	BuyValue                     int
+	BuyChange                    int
+	SoldShortName                string
+	SoldName                     string
+	SoldCode                     string
+	SoldValue                    int
+	SoldChange                   int
+	Frequency                    string
+	ClassifyName                 string
+	ClassifyType                 string
+	CreateTime                   time.Time
+	ModifyTime                   time.Time
+	DataTime                     string
 }
 
 func AddBaseFromTradeZhengzhouIndex(item *BaseFromTradeZhengzhouIndex) (lastId int64, err error) {
@@ -33,37 +46,10 @@ func GetBaseFromTradeZhengzhouIndexAll() (list []*BaseFromTradeZhengzhouIndex, e
 	return
 }
 
-type BaseFromTradeZhengzhouData struct {
-	BaseFromTradeZhengzhouDataId  int `orm:"column(base_from_trade_zhengzhou_data_id);pk"`
-	BaseFromTradeZhengzhouIndexId int
-	IndexCode                     string
-	DataTime                      string
-	Value                         string
-	AddCutValue                   string
-	CreateTime                    time.Time
-	ModifyTime                    time.Time
-	DataTimestamp                 int64
-}
-
-func AddBaseFromTradeZhengzhouData(item *BaseFromTradeZhengzhouData) (lastId int64, err error) {
-	o := orm.NewOrm()
-	o.Using("data")
-	lastId, err = o.Insert(item)
-	return
-}
-
-func GetBaseFromTradeZhengzhouDataByIndexCode(indexCode string) (list []*BaseFromTradeZhengzhouData, err error) {
-	o := orm.NewOrm()
-	o.Using("data")
-	sql := `SELECT * FROM base_from_trade_zhengzhou_data WHERE index_code=? `
-	_, err = o.Raw(sql, indexCode).QueryRows(&list)
-	return
-}
-
-func ModifyBaseFromTradeZhengzhouData(value, addCutVal string, dataId int) (err error) {
+func ModifyBaseFromTradeZhengzhouIndex(dealValue, buyValue, soldValue int, dataId int) (err error) {
 	o := orm.NewOrm()
 	o.Using("data")
-	sql := `UPDATE base_from_trade_zhengzhou_data SET value=?,add_cut_value=?,modify_time=NOW() WHERE base_from_trade_zhengzhou_data_id=? `
-	_, err = o.Raw(sql, value, addCutVal, dataId).Exec()
+	sql := `UPDATE base_from_trade_zhengzhou_index SET deal_value=?,buy_value=?,sold_value=?,modify_time=NOW() WHERE base_from_trade_zhengzhou_index_id=? `
+	_, err = o.Raw(sql, dealValue, buyValue, soldValue, dataId).Exec()
 	return
 }

models/db.go

@@ -27,17 +27,12 @@ func init() {
 
 	//注册对象
 	orm.RegisterModel(
-		//new(BaseFromTradeZhengzhouIndex),
-		//new(BaseFromTradeZhengzhouData),
-		//new(BaseFromTradeShanghaiIndex),
-		//new(BaseFromTradeShanghaiData),
-		//new(BaseFromTradeIneIndex),
-		//new(BaseFromTradeIneData),
-		new(BaseFromTradeCffexIndex),
-		new(BaseFromTradeCffexData),
-
 		new(BaseFromTradeZhengzhouIndex),
-		new(BaseFromTradeDalianIndex),
-		new(BaseFromTradeDalianData),
+		new(BaseFromTradeShanghaiIndex),
+		new(BaseFromTradeIneIndex),
+		new(BaseFromTradeCffexIndex),
+		//
+		//new(BaseFromTradeDalianIndex),
+		//new(BaseFromTradeDalianData),
 	)
 }

services/commodity_trade_cffex.go

@@ -18,14 +18,29 @@ type XmlItem struct {
 		Text         string `xml:"Text,attr"`
 		Tradingday   string `xml:"tradingday"`
 		Datatypeid   string `xml:"datatypeid"`
-		Rank         string `xml:"rank"`
+		Rank         int    `xml:"rank"`
 		ShortName    string `xml:"shortname"`
-		Volume       string `xml:"volume"`
-		Varvolume    string `xml:"varvolume"`
+		Volume       int    `xml:"volume"`
+		Varvolume    int    `xml:"varvolume"`
 		ProductId    string `xml:"productid"`
 	} `xml:"data"`
 }
 
+var cffexIndexCode string
+var cffexIndexCodeMap = make(map[string]struct{})
+var cffexActionCodeMap = make(map[string]map[string]int)
+
+func cffexIndexCodeGenerator() string {
+	for {
+		cffexIndexCode = fmt.Sprintf("CFFEX%s", time.Now().Format(utils.FormatDateTimeUnSpace)+strconv.Itoa(utils.GetRandInt(1, 100)))
+		if _, ok := cffexIndexCodeMap[cffexIndexCode]; !ok {
+			cffexIndexCodeMap[cffexIndexCode] = struct{}{}
+			break
+		}
+	}
+	return cffexIndexCode
+}
+
 // SyncRankingFromCffex 中国金融期货交易所
 func SyncRankingFromCffex() {
 	//获取所有指标信息
@@ -33,116 +48,127 @@ func SyncRankingFromCffex() {
 	if err != nil {
 		fmt.Println("select err:", err)
 	}
-	existIndexMap := make(map[string]*models.BaseFromTradeCffexIndex)
+	for i := 180; i > 0; i-- {
+		zzUrl := "http://www.cffex.com.cn/sj/ccpm/%s/%s/"
+		date := time.Now().AddDate(0, 0, -i)
+		dateStr := date.Format(utils.FormatDateUnSpace)
+		pre := dateStr[:6]
+		sub := dateStr[6:]
+		zzUrl = fmt.Sprintf(zzUrl, pre, sub)
+		fmt.Println(zzUrl)
 
-	addIndexMap := make(map[string]string)
-	indexCodeMap := make(map[string]struct{})
-	for _, v := range allIndex {
-		indexKey := v.IndexName
-		existIndexMap[indexKey] = v
-	}
+		var xmlItems = make([]*XmlItem, 6)
+		xmlItems[0] = GetXmlItem(zzUrl, "IF")
+		xmlItems[1] = GetXmlItem(zzUrl, "IC")
+		xmlItems[2] = GetXmlItem(zzUrl, "IH")
+		xmlItems[3] = GetXmlItem(zzUrl, "TS")
+		xmlItems[4] = GetXmlItem(zzUrl, "TF")
+		xmlItems[5] = GetXmlItem(zzUrl, "T")
+		//获取新的指标信息
+		var sRank string
+		existIndexMap := make(map[string]*models.BaseFromTradeCffexIndex)
+		for _, v := range allIndex {
+			indexKey := v.DealName + v.BuyName + v.SoldName + v.DataTime
+			existIndexMap[indexKey] = v
+			sRank = fmt.Sprintf("%d", v.Rank)
+			cffexActionCodeMap[sRank+v.ClassifyType+v.DataTime] = map[string]int{"0": v.DealValue, "1": v.BuyValue, "2": v.SoldValue, "id": v.BaseFromTradeCffexIndexId}
+		}
+		shortNameColum := map[string]string{"0": "deal_short_name", "1": "buy_short_name", "2": "sold_short_name"}
+		nameColum := map[string]string{"0": "deal_name", "1": "buy_name", "2": "sold_name"}
+		codeColum := map[string]string{"0": "deal_code", "1": "buy_code", "2": "sold_code"}
+		valueColum := map[string]string{"0": "deal_value", "1": "buy_value", "2": "sold_value"}
+		changeColum := map[string]string{"0": "deal_change", "1": "buy_change", "2": "sold_change"}
+		var dataName string
+		var dataCode string
 
-	existDataMap := make(map[string]*models.BaseFromTradeCffexData)
-	var existDataList []*models.BaseFromTradeCffexData
-	var dataItem = new(models.BaseFromTradeCffexData)
-	var item = new(models.BaseFromTradeCffexIndex)
+		var indexKey string
+		for _, xmlItem := range xmlItems {
+			for _, i := range xmlItem.Data {
+				var item = new(models.BaseFromTradeCffexIndex)
+				item.Rank = i.Rank
+				switch i.Value {
+				case "0":
+					item.DealShortName = i.ShortName
+					item.DealName = fmt.Sprintf("%s", i.ShortName+"_成交量_"+i.ContractCode)
+					item.DealCode = cffexIndexCodeGenerator()
+					item.DealValue = i.Volume
+					item.DealChange = i.Varvolume
 
-	var xmlItems  = make([]*XmlItem, 6)
-	xmlItems[0] = GetXmlItem("IF")
-	xmlItems[1] = GetXmlItem("IC")
-	xmlItems[2] = GetXmlItem("IH")
-	xmlItems[3] = GetXmlItem("TS")
-	xmlItems[4] = GetXmlItem("TF")
-	xmlItems[5] = GetXmlItem("T")
-	//获取新的指标信息
-	var action string
-	var indexID int
-	var indexKey string
-	var indexCode string
-	var dataTime time.Time
-	for _, xmlItem := range xmlItems {
-		for _, i := range xmlItem.Data {
-			switch i.Value {
-			case "0":
-				action = "_成交量_"
-			case "1":
-				action = "_持买单量_"
-			case "2":
-				action = "_持卖单量_"
-			default:
-				fmt.Println("No data value")
-			}
-			item.IndexName = fmt.Sprintf("%s", i.ShortName+action+i.ContractCode)
-			for {
-				indexCode = fmt.Sprintf("CFFEX%s", time.Now().Format(utils.FormatDateTimeUnSpace)+strconv.Itoa(utils.GetRandInt(1, 100)))
-				if _, ok := indexCodeMap[indexCode]; !ok {
-					indexCodeMap[indexCode] = struct{}{}
-					break
-				}
-			}
-			item.IndexCode = indexCode
-			item.ClassifyName = i.ProductId
-			item.ClassifyType = i.ContractCode
-			item.CreateTime = time.Now()
-			item.ModifyTime = time.Now()
-			item.Frequency = "日度"
-			if existIndex, ok := existIndexMap[item.IndexName]; !ok {
-				newID, err := models.AddBaseFromTradeCffexIndex(item)
-				if err != nil {
-					fmt.Println("insert error:", err)
-				}
-				indexID = int(newID)
-				fmt.Println("new indexID:", indexID)
-				addIndexMap[indexKey] = indexCode
-			} else {
-				indexID = existIndex.BaseFromTradeCffexIndexId
-				indexCode = existIndex.IndexCode
-			}
-			existDataList, err = models.GetBaseFromTradeCffexDataByIndexCode(indexCode)
-			if err != nil && err.Error() != utils.ErrNoRow() {
-				fmt.Println("GetBaseFromTradeCffexDataByIndexCode error:", err)
-				return
-			}
-			for _, v := range existDataList {
-				key := v.IndexCode + v.DataTime
-				existDataMap[key] = v
-			}
-			dataTime, err = time.Parse(utils.FormatDateUnSpace, i.Tradingday)
-			if err != nil {
-				fmt.Println("time.Parse Err:" + err.Error())
-				return
-			}
-			if existDataItem, ok := existDataMap[i.Tradingday]; !ok && indexCode != "" { //新增
+					dataName = item.DealName
+					dataCode = item.DealCode
+				case "1":
+					item.BuyShortName = i.ShortName
+					item.BuyName = fmt.Sprintf("%s", i.ShortName+"_持买单量_"+i.ContractCode)
+					item.BuyCode = cffexIndexCodeGenerator()
+					item.BuyValue = i.Volume
+					item.BuyChange = i.Varvolume
 
-				dataItem.BaseFromTradeCffexIndexId = indexID
-				dataItem.IndexCode = indexCode
-				dataItem.DataTime = i.Tradingday
-				dataItem.Value = i.Volume
-				dataItem.AddCutValue = i.Varvolume
-				dataItem.DataTimestamp = dataTime.UnixNano() / 1e6
-				dataItem.CreateTime = time.Now()
-				dataItem.ModifyTime = time.Now()
-				_, err = models.AddBaseFromTradeCffexData(dataItem)
-				if err != nil {
-					fmt.Println("data insert err:", err)
-					return
+					dataName = item.BuyName
+					dataCode = item.BuyCode
+				case "2":
+					item.SoldShortName = i.ShortName
+					item.SoldName = fmt.Sprintf("%s", i.ShortName+"_持卖单量_"+i.ContractCode)
+					item.SoldCode = cffexIndexCodeGenerator()
+					item.SoldValue = i.Volume
+					item.SoldChange = i.Varvolume
+
+					dataName = item.SoldName
+					dataCode = item.SoldCode
+				default:
+					fmt.Println("No data value")
 				}
-				existDataMap[indexCode+i.Tradingday] = dataItem
-			} else {
-				if existDataItem != nil && existDataItem.Value != i.Volume { //修改数据
-					err = models.ModifyBaseFromTradeIneDataData(i.Volume, i.Varvolume, existDataItem.BaseFromTradeCffexDataId)
+				item.Frequency = "日度"
+				item.ClassifyName = i.ProductId
+				item.ClassifyType = i.ContractCode
+				item.CreateTime = time.Now()
+				item.ModifyTime = time.Now()
+				item.DataTime = i.Tradingday
+
+				valueMap := map[string]int{"0": item.DealValue, "1": item.BuyValue, "2": item.SoldValue}
+				//检查在actionCodeMap中是否已经有了
+				indexKey = fmt.Sprintf("%d", item.Rank) + item.ClassifyType + item.DataTime
+				if existMap, ok := cffexActionCodeMap[indexKey]; !ok {
+					//没有，新增
+					newID, err := models.AddBaseFromTradeCffexIndex(item)
+					if err != nil {
+						fmt.Println("insert error:", err)
+					}
+					fmt.Println("insert new indexID:", newID)
+					existMap = make(map[string]int)
+					existMap[i.Value] = i.Volume
+					existMap["id"] = int(newID)
+					cffexActionCodeMap[indexKey] = existMap
+				} else if existMap != nil && existMap[i.Value] != i.Volume {
+					//更新
+					var columnList = [5]string{
+						shortNameColum[i.Value],
+						nameColum[i.Value],
+						codeColum[i.Value],
+						valueColum[i.Value],
+						changeColum[i.Value],
+					}
+					var dataList = [5]interface{}{
+						i.ShortName,
+						dataName,
+						dataCode,
+						valueMap[i.Value],
+						i.Varvolume,
+					}
+					err := models.ModifyBaseFromTradeCffexIndex(columnList, dataList, existMap["id"])
 					if err != nil {
 						fmt.Println("data update err:", err)
 					}
+					cffexActionCodeMap[indexKey][i.Value] = i.Volume
 				}
 			}
 		}
 	}
-
 }
 
-func GetXmlItem(contract string) *XmlItem {
-	body, err := http.Get(fmt.Sprintf("%s", "http://www.cffex.com.cn/sj/ccpm/202110/18/" + contract + ".xml"))
+func GetXmlItem(url, contract string) *XmlItem {
+	zzUrl := url + contract + ".xml"
+	fmt.Println(zzUrl)
+	body, err := http.Get(zzUrl)
 	if err != nil {
 		fmt.Println("err:", err)
 	}
@@ -152,4 +178,5 @@ func GetXmlItem(contract string) *XmlItem {
 		fmt.Println("xml.Unmarshal:", err.Error())
 	}
 	return xmlItem
+
 }

services/commodity_trade_ine.go

@@ -13,29 +13,45 @@ import (
 
 type position []struct {
 	ContractCode     string `json:"INSTRUMENTID"`
-	ParticipantID3   string `json:"PARTICIPANTID3"`
-	ParticipantID2   string `json:"PARTICIPANTID2"`
-	ParticipantID1   string `json:"PARTICIPANTID1"`
-	ParticipantName3 string `json:"PARTICIPANTABBR3"`
-	ParticipantName2 string `json:"PARTICIPANTABBR2"`
+	ProductSortNo    int    `json:"PRODUCTSORTNO"`
 	Rank             int    `json:"RANK"`
+	ParticipantID1   string `json:"PARTICIPANTID1"`
 	ParticipantName1 string `json:"PARTICIPANTABBR1"`
+	Deal             int    `json:"CJ1"`
+	Change1          int    `json:"CJ1_CHG"`
+	ParticipantID2   string `json:"PARTICIPANTID2"`
+	ParticipantName2 string `json:"PARTICIPANTABBR2"`
 	BuyIn            int    `json:"CJ2"`
-	Deal             string `json:"CJ1"`
-	Change1          string `json:"CJ1_CHG"`
-	Change3          string `json:"CJ3_CHG"`
-	ProductName      string `json:"PRODUCTNAME"`
-	ProductSortNo    int    `json:"PRODUCTSORTNO"`
+	Change2          int    `json:"CJ2_CHG"`
+	ParticipantID3   string `json:"PARTICIPANTID3"`
+	ParticipantName3 string `json:"PARTICIPANTABBR3"`
 	SoldOut          int    `json:"CJ3"`
-	Change2          string `json:"CJ2_CHG"`
+	Change3          int    `json:"CJ3_CHG"`
+	ProductName      string `json:"PRODUCTNAME"`
 }
 
 type message struct {
-	Position   position `json:"o_cursor"`
+	Position   Position `json:"o_cursor"`
+	Length     string   `json:"showlength"`
 	Code       int      `json:"o_code"`
 	Msg        string   `json:"o_msg"`
 	ReportDate string   `json:"report_date"`
 	UpdateDate string   `json:"update_date"`
+	PrintDate  string   `json:"print_date"`
+}
+
+var ineIndexCode string
+var ineIndexCodeMap = make(map[string]struct{})
+
+func IneIndexCodeGenerator() string {
+	for {
+		ineIndexCode = fmt.Sprintf("INE%s", time.Now().Format(utils.FormatDateTimeUnSpace)+strconv.Itoa(utils.GetRandInt(1, 100)))
+		if _, ok := ineIndexCodeMap[ineIndexCode]; !ok {
+			ineIndexCodeMap[ineIndexCode] = struct{}{}
+			break
+		}
+	}
+	return ineIndexCode
 }
 
 // SyncRankingFromIne 上海能源交易中心持单排名
@@ -45,236 +61,66 @@ func SyncRankingFromIne() {
 	if err != nil {
 		fmt.Println("select err:", err)
 	}
-	existIndexMap := make(map[string]*models.BaseFromTradeIneIndex)
-	addIndexMap := make(map[string]string)
-	indexCodeMap := make(map[string]struct{})
-	for _, v := range allIndex {
-		indexKey := v.IndexName
-		existIndexMap[indexKey] = v
-	}
-
-	existDataMap := make(map[string]*models.BaseFromTradeIneData)
 
 	//获取新的指标信息
-	var message message
-	var indexCode string
-	var indexID int
-	var indexKey string
-	var dealItem = new(models.BaseFromTradeIneIndex)
-	var buyItem = new(models.BaseFromTradeIneIndex)
-	var soldItem = new(models.BaseFromTradeIneIndex)
-	var dataItem = new(models.BaseFromTradeIneData)
-	var existDataList []*models.BaseFromTradeIneData
-	var dataTime time.Time
-	var dataKey string
-	body, err := http.Get("http://www.ine.cn/data/dailydata/kx/pm20211028.dat")
-	if err != nil {
-		fmt.Println("err:", err)
-	}
-	err = json.Unmarshal(body, &message)
-	var position = message.Position
-	var tradeDate = message.ReportDate
-	//处理指标
-	for _, p := range position {
-		if p.ParticipantName1 != "" {
-			//成交量
-			dealItem.IndexName = strings.Replace(fmt.Sprintf("%s", p.ParticipantName1+"_成交量"+"_"+p.ProductName+"_"+p.ContractCode), " ", "", -1)
-			for {
-				indexCode = fmt.Sprintf("INE%s", time.Now().Format(utils.FormatDateTimeUnSpace)+strconv.Itoa(utils.GetRandInt(1, 100)))
-				if _, ok := indexCodeMap[indexCode]; !ok {
-					indexCodeMap[indexCode] = struct{}{}
-					break
-				}
-			}
-			dealItem.IndexCode = indexCode
-			dealItem.ClassifyName = strings.Replace(p.ProductName, " ", "", -1)
-			dealItem.ClassifyType = strings.Replace(p.ContractCode, " ", "", -1)
-			dealItem.CreateTime = time.Now()
-			dealItem.Frequency = "日度"
-			dealItem.ModifyTime = time.Now()
-			if existIndex, ok := existIndexMap[dealItem.IndexName]; !ok {
-				newID, err := models.AddBaseFromTradeIneIndex(dealItem)
-				if err != nil {
-					fmt.Println("insert error:", err)
-				}
-				indexID = int(newID)
-				fmt.Println("new indexID:", indexID)
-				addIndexMap[indexKey] = indexCode
-			} else {
-				indexID = existIndex.BaseFromTradeIneIndexId
-				indexCode = existIndex.IndexCode
-			}
-			existDataList, err = models.GetBaseFromTradeIneDataByIndexCode(indexCode)
-			if err != nil && err.Error() != utils.ErrNoRow() {
-				fmt.Println("GetBaseFromTradeIneDataByIndexCode error:", err)
-				return
-			}
-			for _, v := range existDataList {
-				key := v.IndexCode + v.DataTime
-				existDataMap[key] = v
-			}
-			dataTime, err = time.Parse(utils.FormatDateUnSpace, tradeDate)
-			if err != nil {
-				fmt.Println("time.Parse Err:" + err.Error())
-				return
-			}
-			dataKey = indexCode + dataTime.Format(utils.FormatDate)
-			if existDataItem, ok := existDataMap[dataKey]; !ok && indexCode != "" { //新增
-
-				dataItem.BaseFromTradeIneIndexId = indexID
-				dataItem.IndexCode = indexCode
-				dataItem.DataTime = tradeDate
-				dataItem.Value = p.Deal
-				dataItem.AddCutValue = p.Change1
-				dataItem.DataTimestamp = dataTime.UnixNano() / 1e6
-				dataItem.CreateTime = time.Now()
-				dataItem.ModifyTime = time.Now()
-				_, err = models.AddBaseFromTradeIneData(dataItem)
-				if err != nil {
-					fmt.Println("data insert err:", err)
-					return
-				}
-				existDataMap[indexCode+tradeDate] = dataItem
-			} else {
-				if existDataItem != nil && existDataItem.Value != p.Deal { //修改数据
-					err = models.ModifyBaseFromTradeIneDataData(p.Deal, p.Change1, existDataItem.BaseFromTradeIneDataId)
-					if err != nil {
-						fmt.Println("data update err:", err)
-					}
-				}
-			}
+	for i := 180; i > 0; i-- {
+		var message Message
+		var item = new(models.BaseFromTradeIneIndex)
+		zzUrl := "http://www.ine.com.cn/data/dailydata/kx/pm%s.dat"
+		date := time.Now().AddDate(0, 0, -i)
+		dateStr := date.Format(utils.FormatDateUnSpace)
+		zzUrl = fmt.Sprintf(zzUrl, dateStr)
+		fmt.Println(zzUrl)
+		body, err := http.Get(zzUrl)
+		if err != nil {
+			fmt.Println("err:", err)
 		}
+		err = json.Unmarshal(body, &message)
+		var position = message.Position
+		var tradeDate = message.ReportDate
+		existIndexMap := make(map[string]*models.BaseFromTradeIneIndex)
+		for _, v := range allIndex {
+			indexKey := v.DealName + v.BuyName + v.SoldName + tradeDate
+			existIndexMap[indexKey] = v
+		}
+		var itemVerifyCode int
+		//处理指标
+		for _, p := range position {
+			if p.Rank > 0 && p.Rank < 40 && p.ParticipantName1 != "" {
+				//成交量
+				item.Rank = p.Rank
+				item.DealShortName = p.ParticipantName1
+				item.BuyShortName = p.ParticipantName2
+				item.SoldShortName = p.ParticipantName3
+				item.DealName = strings.Replace(fmt.Sprintf("%s", p.ParticipantName1+"_成交量"+"_"+p.ProductName+"_"+p.ContractCode), " ", "", -1)
+				item.BuyName = strings.Replace(fmt.Sprintf("%s", p.ParticipantName2+"_持买单量"+"_"+p.ProductName+"_"+p.ContractCode), " ", "", -1)
+				item.SoldName = strings.Replace(fmt.Sprintf("%s", p.ParticipantName3+"_持卖单量"+"_"+p.ProductName+"_"+p.ContractCode), " ", "", -1)
+				item.DealCode = IneIndexCodeGenerator()
+				item.BuyCode = IneIndexCodeGenerator()
+				item.SoldCode = IneIndexCodeGenerator()
+				item.DealValue = p.Deal
+				item.DealChange = p.Change1
+				item.BuyValue = p.BuyIn
+				item.BuyChange = p.Change2
+				item.SoldValue = p.SoldOut
+				item.SoldChange = p.Change3
+				item.ClassifyName = strings.Replace(p.ProductName, " ", "", -1)
+				item.ClassifyType = strings.Replace(p.ContractCode, " ", "", -1)
+				item.Frequency = "日度"
+				item.CreateTime = time.Now()
+				item.ModifyTime = time.Now()
+				item.DataTime = tradeDate
 
-		if p.ParticipantName2 != "" {
-			//持买单量
-			buyItem.IndexName = strings.Replace(fmt.Sprintf("%s", p.ParticipantName2+"_持买单量"+"_"+p.ProductName+"_"+p.ContractCode), " ", "", -1)
-			for {
-				indexCode = fmt.Sprintf("INE%s", time.Now().Format(utils.FormatDateTimeUnSpace)+strconv.Itoa(utils.GetRandInt(1, 100)))
-				if _, ok := indexCodeMap[indexCode]; !ok {
-					indexCodeMap[indexCode] = struct{}{}
-					break
-				}
-			}
-			buyItem.IndexCode = indexCode
-			buyItem.ClassifyName = strings.Replace(p.ProductName, " ", "", -1)
-			buyItem.ClassifyType = strings.Replace(p.ContractCode, " ", "", -1)
-			buyItem.CreateTime = time.Now()
-			buyItem.Frequency = "日度"
-			buyItem.ModifyTime = time.Now()
-			if existIndex, ok := existIndexMap[buyItem.IndexName]; !ok {
-				newID, err := models.AddBaseFromTradeIneIndex(buyItem)
-				if err != nil {
-					fmt.Println("insert error:", err)
-				}
-				indexID = int(newID)
-				fmt.Println("new indexID:", indexID)
-				addIndexMap[indexKey] = indexCode
-			} else {
-				indexID = existIndex.BaseFromTradeIneIndexId
-				indexCode = existIndex.IndexCode
-			}
-			existDataList, err = models.GetBaseFromTradeIneDataByIndexCode(indexCode)
-			if err != nil && err.Error() != utils.ErrNoRow() {
-				return
-			}
-			for _, v := range existDataList {
-				key := v.IndexCode + v.DataTime
-				existDataMap[key] = v
-			}
-			dataTime, err := time.Parse(utils.FormatDateUnSpace, tradeDate)
-			if err != nil {
-				fmt.Println("time.Parse Err:" + err.Error())
-				return
-			}
-			dataKey = indexCode + dataTime.Format(utils.FormatDate)
-			if existDataItem, ok := existDataMap[dataKey]; !ok && indexCode != "" { //新增
-
-				dataItem.BaseFromTradeIneIndexId = indexID
-				dataItem.IndexCode = indexCode
-				dataItem.DataTime = tradeDate
-				dataItem.Value = fmt.Sprintf("%d", p.BuyIn)
-				dataItem.AddCutValue = p.Change2
-				dataItem.DataTimestamp = dataTime.UnixNano() / 1e6
-				dataItem.CreateTime = time.Now()
-				dataItem.ModifyTime = time.Now()
-				_, err = models.AddBaseFromTradeIneData(dataItem)
-				if err != nil {
-					fmt.Println("data insert err:", err)
-					return
-				}
-				existDataMap[indexCode+tradeDate] = dataItem
-			} else {
-				if existDataItem != nil && existDataItem.Value != fmt.Sprintf("%d", p.BuyIn) { //修改数据
-					err = models.ModifyBaseFromTradeIneDataData(fmt.Sprintf("%d", p.BuyIn), p.Change2, existDataItem.BaseFromTradeIneDataId)
+				itemVerifyCode = item.BuyValue + item.DealValue + item.SoldValue
+				if existIndex, ok := existIndexMap[item.DealName+item.BuyName+item.SoldName+tradeDate]; !ok {
+					newID, err := models.AddBaseFromTradeIneIndex(item)
 					if err != nil {
-						fmt.Println("data update err:", err)
+						fmt.Println("insert error:", err)
 					}
-				}
-			}
-		}
-
-		if p.ParticipantName3 != "" {
-			//持卖单量
-			soldItem.IndexName = strings.Replace(fmt.Sprintf("%s", p.ParticipantName3+"_持卖单量"+"_"+p.ProductName+"_"+p.ContractCode), " ", "", -1)
-			for {
-				indexCode = fmt.Sprintf("INE%s", time.Now().Format(utils.FormatDateTimeUnSpace)+strconv.Itoa(utils.GetRandInt(1, 100)))
-				if _, ok := indexCodeMap[indexCode]; !ok {
-					indexCodeMap[indexCode] = struct{}{}
-					break
-				}
-			}
-			soldItem.IndexCode = indexCode
-			soldItem.ClassifyName = strings.Replace(p.ProductName, " ", "", -1)
-			soldItem.ClassifyType = strings.Replace(p.ContractCode, " ", "", -1)
-			soldItem.CreateTime = time.Now()
-			soldItem.Frequency = "日度"
-			soldItem.ModifyTime = time.Now()
-			if existIndex, ok := existIndexMap[soldItem.IndexName]; !ok {
-				newID, err := models.AddBaseFromTradeIneIndex(soldItem)
-				if err != nil {
-					fmt.Println("insert error:", err)
-				}
-				indexID = int(newID)
-				fmt.Println("new indexID:", indexID)
-				addIndexMap[indexKey] = indexCode
-			} else {
-				indexID = existIndex.BaseFromTradeIneIndexId
-				indexCode = existIndex.IndexCode
-			}
-			existDataList, err = models.GetBaseFromTradeIneDataByIndexCode(indexCode)
-			if err != nil && err.Error() != utils.ErrNoRow() {
-				fmt.Println("GetBaseFromTradeIneDataByIndexCode error:", err)
-				return
-			}
-			for _, v := range existDataList {
-				key := v.IndexCode + v.DataTime
-				existDataMap[key] = v
-			}
-			dataTime, err = time.Parse(utils.FormatDateUnSpace, tradeDate)
-			if err != nil {
-				fmt.Println("time.Parse Err:" + err.Error())
-				return
-			}
-			dataKey = indexCode + dataTime.Format(utils.FormatDate)
-			if existDataItem, ok := existDataMap[dataKey]; !ok && indexCode != "" { //新增
-
-				dataItem.BaseFromTradeIneIndexId = indexID
-				dataItem.IndexCode = indexCode
-				dataItem.DataTime = tradeDate
-				dataItem.Value = fmt.Sprintf("%d", p.SoldOut)
-				dataItem.AddCutValue = p.Change3
-				dataItem.DataTimestamp = dataTime.UnixNano() / 1e6
-				dataItem.CreateTime = time.Now()
-				dataItem.ModifyTime = time.Now()
-				_, err = models.AddBaseFromTradeIneData(dataItem)
-				if err != nil {
-					fmt.Println("data insert err:", err)
-					return
-				}
-				existDataMap[indexCode+tradeDate] = dataItem
-			} else {
-				if existDataItem != nil && existDataItem.Value != fmt.Sprintf("%d", p.SoldOut) { //修改数据
-					err = models.ModifyBaseFromTradeIneDataData(fmt.Sprintf("%d", p.SoldOut), p.Change3, existDataItem.BaseFromTradeIneDataId)
+					fmt.Println("insert new indexID:", newID)
+				} else if existIndex != nil && itemVerifyCode != (existIndex.DealValue+existIndex.BuyValue+existIndex.SoldValue) {
+					//更新
+					err := models.ModifyBaseFromTradeIneIndex(item.DealValue, item.BuyValue, item.SoldValue, existIndex.BaseFromTradeIneIndexId)
 					if err != nil {
 						fmt.Println("data update err:", err)
 					}

services/commodity_trade_shanghai.go

@@ -40,6 +40,20 @@ type Message struct {
 	PrintDate  string   `json:"print_date"`
 }
 
+var indexCode string
+var indexCodeMap = make(map[string]struct{})
+
+func indexCodeGenerator() string {
+	for {
+		indexCode = fmt.Sprintf("SH%s", time.Now().Format(utils.FormatDateTimeUnSpace)+strconv.Itoa(utils.GetRandInt(1, 100)))
+		if _, ok := indexCodeMap[indexCode]; !ok {
+			indexCodeMap[indexCode] = struct{}{}
+			break
+		}
+	}
+	return indexCode
+}
+
 // SyncRankingFromShangHai 上海商品交易所持单排名
 func SyncRankingFromShangHai() {
 	//获取所有指标信息
@@ -47,242 +61,68 @@ func SyncRankingFromShangHai() {
 	if err != nil {
 		fmt.Println("select err:", err)
 	}
-	existIndexMap := make(map[string]*models.BaseFromTradeShanghaiIndex)
-	addIndexMap := make(map[string]string)
-	indexCodeMap := make(map[string]struct{})
-
-	for _, v := range allIndex {
-		indexKey := v.IndexName
-		existIndexMap[indexKey] = v
-	}
-
-	existDataMap := make(map[string]*models.BaseFromTradeShanghaiData)
 
 	//获取新的指标信息
 	var message Message
-	var indexCode string
-	var indexID int
-	var indexKey string
-	var dealItem = new(models.BaseFromTradeShanghaiIndex)
-	var buyItem = new(models.BaseFromTradeShanghaiIndex)
-	var soldItem = new(models.BaseFromTradeShanghaiIndex)
-	var dataItem = new(models.BaseFromTradeShanghaiData)
-	var existDataList []*models.BaseFromTradeShanghaiData
-	var dataTime time.Time
-	var dataKey string
-	body, err := http.Get("http://www.shfe.com.cn/data/dailydata/kx/pm20211025.dat")
-	if err != nil {
-		fmt.Println("err:", err)
-	}
-	err = json.Unmarshal(body, &message)
-	var position = message.Position
-	var tradeDate = message.ReportDate
-	//处理指标
-	for _, p := range position {
-		//排除标题行和最后一行
-		if p.Rank > 0 {
-			if p.ParticipantName1 != "" {
+	var item = new(models.BaseFromTradeShanghaiIndex)
+	for i := 180; i > 0; i-- {
+		zzUrl := "http://www.shfe.com.cn/data/dailydata/kx/pm%s.dat"
+		date := time.Now().AddDate(0, 0, -i)
+		dateStr := date.Format(utils.FormatDateUnSpace)
+		zzUrl = fmt.Sprintf(zzUrl, dateStr)
+		fmt.Println(zzUrl)
+		body, err := http.Get(zzUrl)
+		if err != nil {
+			fmt.Println("err:", err)
+		}
+		err = json.Unmarshal(body, &message)
+		var position = message.Position
+		var tradeDate = message.ReportDate
+		existIndexMap := make(map[string]*models.BaseFromTradeShanghaiIndex)
+		for _, v := range allIndex {
+			indexKey := v.DealName + v.BuyName + v.SoldName + v.DataTime
+			existIndexMap[indexKey] = v
+		}
+		var itemVerifyCode int
+		//处理指标
+		for _, p := range position {
+			if p.Rank > 0 && p.Rank <40 && p.ParticipantName1 != "" {
 				//成交量
-				dealItem.IndexName = strings.Replace(fmt.Sprintf("%s", p.ParticipantName1+"_成交量"+"_"+p.ProductName+"_"+p.ContractCode), " ", "", -1)
-				indexCode = fmt.Sprintf("SH%s", time.Now().Format(utils.FormatDateTimeUnSpace)+strconv.Itoa(utils.GetRandInt(1, 100)))
-				for {
-					indexCode = fmt.Sprintf("SH%s", time.Now().Format(utils.FormatDateTimeUnSpace)+strconv.Itoa(utils.GetRandInt(1, 100)))
-					if _, ok := indexCodeMap[indexCode]; !ok {
-						indexCodeMap[indexCode] = struct{}{}
-						break
-					}
-				}
-				dealItem.IndexCode = indexCode
-				dealItem.ClassifyName = strings.Replace(p.ProductName, " ", "", -1)
-				dealItem.ClassifyType = strings.Replace(p.ContractCode, " ", "", -1)
-				dealItem.CreateTime = time.Now()
-				dealItem.Frequency = "日度"
-				dealItem.ModifyTime = time.Now()
-				if existIndex, ok := existIndexMap[dealItem.IndexName]; !ok {
-					newID, err := models.AddBaseFromTradeShangHaiIndex(dealItem)
-					if err != nil {
-						fmt.Println("insert error:", err)
-					}
-					indexID = int(newID)
-					fmt.Println("new indexID:", indexID)
-					addIndexMap[indexKey] = indexCode
-				} else {
-					indexID = existIndex.BaseFromTradeShangHaiIndexId
-					indexCode = existIndex.IndexCode
-				}
-				existDataList, err = models.GetBaseFromTradeShangHaiDataByIndexCode(indexCode)
-				if err != nil && err.Error() != utils.ErrNoRow() {
-					fmt.Println("GetBaseFromTradeShangHaiDataByIndexCode error:", err)
-					return
-				}
-				for _, v := range existDataList {
-					key := v.IndexCode + v.DataTime
-					existDataMap[key] = v
-				}
-				dataTime, err = time.Parse(utils.FormatDateUnSpace, tradeDate)
-				if err != nil {
-					fmt.Println("time.Parse Err:" + err.Error())
-					return
-				}
-				dataKey = indexCode + dataTime.Format(utils.FormatDate)
-				if existDataItem, ok := existDataMap[dataKey]; !ok && indexCode != "" { //新增
-
-					dataItem.BaseFromTradeShanghaiIndexId = indexID
-					dataItem.IndexCode = indexCode
-					dataItem.DataTime = tradeDate
-					dataItem.Value = fmt.Sprintf("%d", p.Deal)
-					dataItem.AddCutValue = fmt.Sprintf("%d", p.Change1)
-					dataItem.DataTimestamp = dataTime.UnixNano() / 1e6
-					dataItem.CreateTime = time.Now()
-					dataItem.ModifyTime = time.Now()
-					_, err = models.AddBaseFromTradeShangHaiData(dataItem)
-					if err != nil {
-						fmt.Println("data insert err:", err)
-						return
-					}
-					existDataMap[indexCode+tradeDate] = dataItem
-				} else {
-					if existDataItem != nil && existDataItem.Value != fmt.Sprintf("%d", p.Deal) { //修改数据
-						err = models.ModifyBaseFromTradeShangHaiData(fmt.Sprintf("%d", p.Deal), fmt.Sprintf("%d", p.Change1), existDataItem.BaseFromTradeShangHaiDataId)
-						if err != nil {
-							fmt.Println("data update err:", err)
-						}
-					}
-				}
-			}
-			if p.ParticipantName2 != "" {
-				//持买单量
-				buyItem.IndexName = strings.Replace(fmt.Sprintf("%s", p.ParticipantName2+"_持买单量"+"_"+p.ProductName+"_"+p.ContractCode), " ", "", -1)
-				for {
-					indexCode = fmt.Sprintf("SH%s", time.Now().Format(utils.FormatDateTimeUnSpace)+strconv.Itoa(utils.GetRandInt(1, 100)))
-					if _, ok := indexCodeMap[indexCode]; !ok {
-						indexCodeMap[indexCode] = struct{}{}
-						break
-					}
-				}
-				buyItem.IndexCode = indexCode
-				buyItem.ClassifyName = strings.Replace(p.ProductName, " ", "", -1)
-				buyItem.ClassifyType = strings.Replace(p.ContractCode, " ", "", -1)
-				buyItem.CreateTime = time.Now()
-				buyItem.Frequency = "日度"
-				buyItem.ModifyTime = time.Now()
-				if existIndex, ok := existIndexMap[buyItem.IndexName]; !ok {
-					newID, err := models.AddBaseFromTradeShangHaiIndex(buyItem)
-					if err != nil {
-						fmt.Println("insert error:", err)
-					}
-					indexID = int(newID)
-					fmt.Println("new indexID:", indexID)
-					addIndexMap[indexKey] = indexCode
-				} else {
-					indexID = existIndex.BaseFromTradeShangHaiIndexId
-					indexCode = existIndex.IndexCode
-				}
-				existDataList, err = models.GetBaseFromTradeShangHaiDataByIndexCode(indexCode)
-				if err != nil && err.Error() != utils.ErrNoRow() {
-					return
-				}
-				for _, v := range existDataList {
-					key := v.IndexCode + v.DataTime
-					existDataMap[key] = v
-				}
-				dataTime, err := time.Parse(utils.FormatDateUnSpace, tradeDate)
-				if err != nil {
-					fmt.Println("time.Parse Err:" + err.Error())
-					return
-				}
-				dataKey = indexCode + dataTime.Format(utils.FormatDate)
-				if existDataItem, ok := existDataMap[dataKey]; !ok && indexCode != "" { //新增
+				item.Rank = p.Rank
+				item.DealShortName = p.ParticipantName1
+				item.BuyShortName = p.ParticipantName2
+				item.SoldShortName = p.ParticipantName3
+				item.DealName = strings.Replace(fmt.Sprintf("%s", p.ParticipantName1+"_成交量"+"_"+p.ProductName+"_"+p.ContractCode), " ", "", -1)
+				item.BuyName = strings.Replace(fmt.Sprintf("%s", p.ParticipantName2+"_持买单量"+"_"+p.ProductName+"_"+p.ContractCode), " ", "", -1)
+				item.SoldName = strings.Replace(fmt.Sprintf("%s", p.ParticipantName3+"_持卖单量"+"_"+p.ProductName+"_"+p.ContractCode), " ", "", -1)
+				item.DealCode =  indexCodeGenerator()
+				item.BuyCode =  indexCodeGenerator()
+				item.SoldCode =  indexCodeGenerator()
+				item.DealValue = p.Deal
+				item.DealChange = p.Change1
+				item.BuyValue = p.BuyIn
+				item.BuyChange = p.Change2
+				item.SoldValue = p.SoldOut
+				item.SoldChange = p.Change3
+				item.ClassifyName = strings.Replace(p.ProductName, " ", "", -1)
+				item.ClassifyType = strings.Replace(p.ContractCode, " ", "", -1)
+				item.Frequency = "日度"
+				item.CreateTime = time.Now()
+				item.ModifyTime = time.Now()
+				item.DataTime = tradeDate
 
-					dataItem.BaseFromTradeShanghaiIndexId = indexID
-					dataItem.IndexCode = indexCode
-					dataItem.DataTime = tradeDate
-					dataItem.Value = fmt.Sprintf("%d", p.BuyIn)
-					dataItem.AddCutValue = fmt.Sprintf("%d", p.Change2)
-					dataItem.DataTimestamp = dataTime.UnixNano() / 1e6
-					dataItem.CreateTime = time.Now()
-					dataItem.ModifyTime = time.Now()
-					_, err = models.AddBaseFromTradeShangHaiData(dataItem)
-					if err != nil {
-						fmt.Println("data insert err:", err)
-						return
-					}
-					existDataMap[indexCode+tradeDate] = dataItem
-				} else {
-					if existDataItem != nil && existDataItem.Value != fmt.Sprintf("%d", p.BuyIn) { //修改数据
-						err = models.ModifyBaseFromTradeShangHaiData(fmt.Sprintf("%d", p.BuyIn), fmt.Sprintf("%d", p.Change2), existDataItem.BaseFromTradeShangHaiDataId)
-						if err != nil {
-							fmt.Println("data update err:", err)
-						}
-					}
-				}
-			}
-
-			if p.ParticipantName3 != "" {
-				//持卖单量
-				soldItem.IndexName = strings.Replace(fmt.Sprintf("%s", p.ParticipantName3+"_持卖单量"+"_"+p.ProductName+"_"+p.ContractCode), " ", "", -1)
-				for {
-					indexCode = fmt.Sprintf("SH%s", time.Now().Format(utils.FormatDateTimeUnSpace)+strconv.Itoa(utils.GetRandInt(1, 100)))
-					if _, ok := indexCodeMap[indexCode]; !ok {
-						indexCodeMap[indexCode] = struct{}{}
-						break
-					}
-				}
-				soldItem.IndexCode = indexCode
-				soldItem.ClassifyName = strings.Replace(p.ProductName, " ", "", -1)
-				soldItem.ClassifyType = strings.Replace(p.ContractCode, " ", "", -1)
-				soldItem.CreateTime = time.Now()
-				soldItem.Frequency = "日度"
-				soldItem.ModifyTime = time.Now()
-				if existIndex, ok := existIndexMap[soldItem.IndexName]; !ok {
-					newID, err := models.AddBaseFromTradeShangHaiIndex(soldItem)
+				itemVerifyCode = item.BuyValue + item.DealValue + item.SoldValue
+				if existIndex, ok := existIndexMap[item.DealName + item.BuyName + item.SoldName + item.DataTime]; !ok {
+					newID, err := models.AddBaseFromTradeShangHaiIndex(item)
 					if err != nil {
 						fmt.Println("insert error:", err)
 					}
-					indexID = int(newID)
-					fmt.Println("new indexID:", indexID)
-					addIndexMap[indexKey] = indexCode
-				} else {
-					indexID = existIndex.BaseFromTradeShangHaiIndexId
-					indexCode = existIndex.IndexCode
-				}
-				existDataList, err = models.GetBaseFromTradeShangHaiDataByIndexCode(indexCode)
-				if err != nil && err.Error() != utils.ErrNoRow() {
-					fmt.Println("GetBaseFromTradeShangHaiDataByIndexCode error:", err)
-					return
-				}
-				for _, v := range existDataList {
-					key := v.IndexCode + v.DataTime
-					existDataMap[key] = v
-				}
-				dataTime, err = time.Parse(utils.FormatDateUnSpace, tradeDate)
-				if err != nil {
-					fmt.Println("time.Parse Err:" + err.Error())
-					return
-				}
-				dataKey = indexCode + dataTime.Format(utils.FormatDate)
-				if existDataItem, ok := existDataMap[dataKey]; !ok && indexCode != "" { //新增
-
-					dataItem.BaseFromTradeShanghaiIndexId = indexID
-					dataItem.IndexCode = indexCode
-					dataItem.DataTime = tradeDate
-					dataItem.Value = fmt.Sprintf("%d", p.SoldOut)
-					dataItem.AddCutValue = fmt.Sprintf("%d", p.Change3)
-					dataItem.DataTimestamp = dataTime.UnixNano() / 1e6
-					dataItem.CreateTime = time.Now()
-					dataItem.ModifyTime = time.Now()
-					_, err = models.AddBaseFromTradeShangHaiData(dataItem)
+					fmt.Println("insert new indexID:", newID)
+				} else if existIndex != nil && itemVerifyCode != (existIndex.DealValue+existIndex.BuyValue+existIndex.SoldValue) {
+					//更新
+					err := models.ModifyBaseFromTradeShangHaiIndex(item.DealValue, item.BuyValue, item.SoldValue, existIndex.BaseFromTradeShangHaiIndexId)
 					if err != nil {
-						fmt.Println("data insert err:", err)
-						return
-					}
-					existDataMap[indexCode+tradeDate] = dataItem
-				} else {
-					if existDataItem != nil && existDataItem.Value != fmt.Sprintf("%d", p.SoldOut) { //修改数据
-						err = models.ModifyBaseFromTradeShangHaiData(fmt.Sprintf("%d", p.SoldOut), fmt.Sprintf("%d", p.Change3), existDataItem.BaseFromTradeShangHaiDataId)
-						if err != nil {
-							fmt.Println("data update err:", err)
-						}
+						fmt.Println("data update err:", err)
 					}
 				}
 			}

services/commodity_trade_zhengzhou.go

@@ -13,8 +13,23 @@ import (
 	"github.com/PuerkitoBio/goquery"
 )
 
+var zIndexCode string
+var zIndexCodeMap = make(map[string]struct{})
+var zActionCodeMap = make(map[string]map[string]int)
+
+func zIndexCodeGenerator() string {
+	for {
+		zIndexCode = fmt.Sprintf("Z%s", time.Now().Format(utils.FormatDateTimeUnSpace)+strconv.Itoa(utils.GetRandInt(1, 100)))
+		if _, ok := zIndexCodeMap[zIndexCode]; !ok {
+			zIndexCodeMap[zIndexCode] = struct{}{}
+			break
+		}
+	}
+	return zIndexCode
+}
+
 //郑州商品交易所持单排名
-func SyncRankingFromZhenzhou() {
+func SyncRankingFromZhengzhou() {
 	fmt.Println("start")
 	var err error
 	defer func() {
@@ -23,23 +38,25 @@ func SyncRankingFromZhenzhou() {
 		}
 	}()
 
-	//获取所有指标信息
+	//获取所有指标信息  某一天的
 	allIndex, err := models.GetBaseFromTradeZhengzhouIndexAll()
 	if err != nil {
 		return
 	}
+
 	existIndexMap := make(map[string]*models.BaseFromTradeZhengzhouIndex)
-	addIndexMap := make(map[string]string)
 	for _, v := range allIndex {
-		indexKey := v.IndexName + v.ClassifyName
+		indexKey := v.DealName + v.BuyName + v.SoldName
 		existIndexMap[indexKey] = v
 	}
-	for i := 10; i > 0; i-- {
+	var itemVerifyCode int
+	for i := 180; i > 0; i-- {
 		zzUrl := "http://www.czce.com.cn/cn/DFSStaticFiles/Future/%s/%s/FutureDataHolding.htm"
 		date := time.Now().AddDate(0, 0, -i)
 		year := date.Year()
 		dateStr := date.Format(utils.FormatDateUnSpace)
 		zzUrl = fmt.Sprintf(zzUrl, strconv.Itoa(year), dateStr)
+		fmt.Println(zzUrl)
 		body, err := http.Get(zzUrl)
 		if err != nil {
 			fmt.Println("GetData Err:" + err.Error())
@@ -58,6 +75,7 @@ func SyncRankingFromZhenzhou() {
 		var classifyName, classifyType, tradeDate string
 		var dealSuffix, dealAddCutSuffix, buySuffix, buyAddCutSuffix, sellSuffix, sellAddCutSuffix string
 		var dealVal, dealAddCutVal, buyVal, buyAddCutVal, sellVal, sellAddCutVal string
+		var rank string
 		table.Find("tr").Each(func(i int, tr *goquery.Selection) {
 			var memberShortNameArr []string
 			tds := tr.Find("td")
@@ -82,6 +100,14 @@ func SyncRankingFromZhenzhou() {
 					if tdText == "" {
 						return
 					}
+					//fmt.Println("tdText: ", tdText)
+					if tk == 0 {
+						if !(strings.Contains(tdText, "名次") || strings.Contains(tdText, "合计")){
+							fmt.Println("tk:",tk)
+							rank = tdText
+						}
+					}
+
 					if tk == 1 {
 						if !strings.Contains(tdText, "会员简称") {
 							memberShortNameArr = append(memberShortNameArr, tdText)
@@ -159,106 +185,46 @@ func SyncRankingFromZhenzhou() {
 					classifyType = classifyNameArr[0]
 					classifyName = classifyNameArr[1]
 				}
-				fmt.Println("memberShortNameArr:", memberShortNameArr)
-				for sk, sv := range memberShortNameArr {
-					if sv == "" {
-						continue
-					}
-					var indexKey string
-					if sk == 0 {
-						indexKey = sv + "_" + dealSuffix + classifyName
-					} else if sk == 1 {
-						indexKey = sv + "_" + buySuffix + classifyName
-					} else {
-						indexKey = sv + "_" + sellSuffix + classifyName
-					}
+				fmt.Printf("memberShortNameArr:%s,len:%d\n", memberShortNameArr, len(memberShortNameArr))
+				if len(memberShortNameArr) != 0 {
+					if len(memberShortNameArr[0]) != 2 {
+						item := new(models.BaseFromTradeZhengzhouIndex)
+						item.Rank, _ = strconv.Atoi(rank)
+						item.DealShortName = memberShortNameArr[0]
+						item.DealName = memberShortNameArr[0] + "_" + dealSuffix + "_" + classifyName + "_" + classifyType
+						item.DealCode = zIndexCodeGenerator()
+						item.DealValue, _ = strconv.Atoi(dealVal)
+						item.DealChange, _ = strconv.Atoi(dealAddCutVal)
+						item.BuyShortName = memberShortNameArr[1]
+						item.BuyName = memberShortNameArr[1] + "_" + buySuffix + "_" + classifyName + "_" + classifyType
+						item.BuyCode = zIndexCodeGenerator()
+						item.BuyValue, _ = strconv.Atoi(buyVal)
+						item.BuyChange, _ = strconv.Atoi(buyAddCutVal)
+						item.SoldShortName = memberShortNameArr[2]
+						item.SoldName = memberShortNameArr[2] + "_" + sellSuffix + "_" + classifyName + "_" + classifyType
+						item.SoldCode = zIndexCodeGenerator()
+						item.SoldValue, _ = strconv.Atoi(sellVal)
+						item.SoldChange, _ = strconv.Atoi(sellAddCutVal)
+						item.Frequency = "日度"
+						item.ClassifyName = classifyName
+						item.ClassifyType = classifyType
+						item.CreateTime = time.Now()
+						item.ModifyTime = time.Now()
+						item.DataTime = tradeDate
 
-					fmt.Println("line 28417 indexKey:", indexKey)
-					var indexId int
-					var indexCode string
-					if existIndex, ok := existIndexMap[indexKey]; !ok { //不存在，新增指标
-						if _, addOk := addIndexMap[indexKey]; !addOk {
-							fmt.Println("指标不存在")
-							indexCode = fmt.Sprintf("Z%s", time.Now().Format(utils.FormatDateTimeUnSpace)+strconv.Itoa(utils.GetRandInt(1, 100)))
-							item := new(models.BaseFromTradeZhengzhouIndex)
-							item.IndexCode = indexCode
-							item.ClassifyName = classifyName
-							item.ClassifyType = classifyType
-							if sk == 0 {
-								item.IndexName = sv + "_" + dealSuffix
-							} else if sk == 1 {
-								item.IndexName = sv + "_" + buySuffix
-							} else {
-								item.IndexName = sv + "_" + sellSuffix
-							}
-							item.CreateTime = time.Now()
-							item.Frequency = "日度"
-							item.CreateTime = time.Now()
-							item.ModifyTime = time.Now()
-							newId, err := models.AddBaseFromTradeZhengzhouIndex(item)
+						itemVerifyCode = item.BuyValue + item.DealValue + item.SoldValue
+						if existIndex, ok := existIndexMap[item.DealName + item.BuyName + item.SoldName]; !ok {
+							newID, err := models.AddBaseFromTradeZhengzhouIndex(item)
 							if err != nil {
-								return
+								fmt.Println("insert error:", err)
 							}
-							indexId = int(newId)
-							addIndexMap[indexKey] = indexCode
-							time.Sleep(1 * time.Second)
-						}
-					} else {
-						fmt.Println("指标已经存在:" + existIndex.IndexName)
-						indexId = existIndex.BaseFromTradeZhengzhouIndexId
-						indexCode = existIndex.IndexCode
-					}
-					existDataList, err := models.GetBaseFromTradeZhengzhouDataByIndexCode(indexCode)
-					if err != nil && err.Error() != utils.ErrNoRow() {
-						return
-					}
-					existDataMap := make(map[string]*models.BaseFromTradeZhengzhouData)
-					for _, v := range existDataList {
-						key := v.IndexCode + v.DataTime
-						existDataMap[key] = v
-					}
-					dataKey := indexCode + tradeDate
-					if existDataItem, ok := existDataMap[dataKey]; !ok && indexCode != "" { //新增数据
-						dataTime, err := time.Parse(utils.FormatDate, tradeDate)
-						if err != nil {
-							fmt.Println("time.Parse Err:" + err.Error())
-							return
-						}
-						timestamp := dataTime.UnixNano() / 1e6
-
-						dataItem := new(models.BaseFromTradeZhengzhouData)
-						dataItem.BaseFromTradeZhengzhouIndexId = indexId
-						dataItem.IndexCode = indexCode
-						dataItem.DataTime = tradeDate
-						if sk == 0 {
-							dataItem.Value = dealVal
-							dataItem.AddCutValue = dealAddCutVal
-						} else if sk == 1 {
-							dataItem.Value = buyVal
-							dataItem.AddCutValue = buyAddCutVal
-						} else {
-							dataItem.Value = sellVal
-							dataItem.AddCutValue = sellAddCutVal
-						}
-						dataItem.CreateTime = time.Now()
-						dataItem.ModifyTime = time.Now()
-						dataItem.DataTimestamp = timestamp
-						_, err = models.AddBaseFromTradeZhengzhouData(dataItem)
-						existDataMap[indexCode+tradeDate] = dataItem
-					} else {
-						if existDataItem != nil && existDataItem.Value != dealVal { //修改数据
-							var setVal, setAddCutVal string
-							if sk == 0 {
-								setVal = dealVal
-								setAddCutVal = dealAddCutVal
-							} else if sk == 1 {
-								setVal = buyVal
-								setAddCutVal = buyAddCutVal
-							} else {
-								setVal = sellVal
-								setAddCutVal = sellAddCutVal
+							fmt.Println("insert new indexID:", newID)
+						} else if existIndex != nil && itemVerifyCode != (existIndex.DealValue+existIndex.BuyValue+existIndex.SoldValue) {
+							//更新
+							err := models.ModifyBaseFromTradeZhengzhouIndex(item.DealValue, item.BuyValue, item.SoldValue, existIndex.BaseFromTradeZhengzhouIndexId)
+							if err != nil {
+								fmt.Println("data update err:", err)
 							}
-							models.ModifyBaseFromTradeZhengzhouData(setVal, setAddCutVal, existDataItem.BaseFromTradeZhengzhouDataId)
 						}
 					}
 				}

services/task.go

@@ -7,20 +7,13 @@ import (
 func Task() {
 	fmt.Println("start crawler")
 	//同步郑州商品交易所排名
-	//SyncRankingFromZhenzhou()
+	//SyncRankingFromZhengzhou()
 
 	//SyncRankingFromShangHai()
-	//SyncRankingFromIne()
-	SyncRankingFromCffex()
+	SyncRankingFromIne()
+	//SyncRankingFromCffex()
 
 	//SyncRankingFromDalian2()
 
-	//str:=`品种：苹果AP    日期：2021-10-15`
-	//str=strings.ReplaceAll(str,"    ","#")
-	//strArr:=strings.Split(str,"#")
-	//for k,v:=range strArr{
-	//	fmt.Println(k,v)
-	//}
-
 	fmt.Println("end crawler")
 }