新增数据爬取

models/trade_guangzhou.go

@@ -47,6 +47,7 @@ type BaseFromTradeGuangzhouClassify struct {
 type BaseFromTradeGuangzhouIndex struct {
 	BaseFromTradeGuangzhouIndexId    int       `orm:"column(base_from_trade_guangzhou_index_id);pk"`
 	BaseFromTradeGuangzhouClassifyId int       `description:"分类id"`
+	BaseFromTradeGuangzhouContractId int       `description:"合约id"`
 	IndexCode                        string    `description:"指标编码"`
 	IndexName                        string    `description:"指标名称"`
 	Frequency                        string    `description:"频率"`

services/task.go

@@ -43,7 +43,7 @@ func Task() {
 	task.AddTask("中国煤炭网爬取", refreshCoal)
 	task.AddTask("美联储加息概率爬取", refreshMeeting)
 	//task.AddTask("谷歌出行指数爬取", refreshVisitors)
-	task.AddTask("eia steo报告", syncEiaSteoData)   //每天22点爬一次
+	task.AddTask("eia steo报告", syncEiaSteoData)     //每天22点爬一次
 	task.AddTask("UN年度数据", syncYearComTrade)      //每年一月一号的3点同步
 	task.AddTask("UN月度数据", syncYearMonthComTrade) //每月1号的3点同步
 
@@ -171,16 +171,67 @@ func RefreshChangesVisitorsCovid(cont context.Context) (err error) {
 
 func Task123() {
 	fmt.Println("task start")
-	//n := 6
-	//for i := n; i >= 0; i-- {
-	//	date := time.Now().AddDate(0, -i, 0)
-	//	fmt.Println("date:", date.Format(utils.FormatDate))
-	//	CrawlerFromGuangzhouCongregateDeliveryQuotes(date)
-	//	rn := utils.RangeRand(3, 15)
-	//	fmt.Println("i:"+strconv.Itoa(i)+" date:", date.Format(utils.FormatDate)+";sleep:", strconv.Itoa(int(rn)))
-	//	time.Sleep(time.Duration(rn) * time.Second)
-	//}
-	ctx := new(context.Context)
-	CrawlerIcpi(*ctx)
+	//日成交持仓排名
+	n := 3
+	//广州交易所期货-日行情
+	for i := n; i >= 0; i-- {
+		date := time.Now().AddDate(0, 0, -i)
+		if date.Weekday() == time.Saturday || date.Weekday() == time.Sunday {
+			continue
+		}
+		CrawlerFromGuangzhouDayQuotes(date)
+		rn := utils.RangeRand(10, 60)
+		fmt.Println("i:"+strconv.Itoa(i)+" date:", date.Format(utils.FormatDate)+";sleep:", strconv.Itoa(int(rn)))
+		time.Sleep(time.Duration(rn) * time.Second)
+	}
+	//合约
+	for i := n; i >= 0; i-- {
+		date := time.Now().AddDate(0, 0, -i)
+		if date.Weekday() == time.Saturday || date.Weekday() == time.Sunday {
+			continue
+		}
+		fmt.Println("date:", date.Format(utils.FormatDate))
+		CrawlerFromGuangzhouContract(date)
+		rn := utils.RangeRand(3, 15)
+		fmt.Println("i:"+strconv.Itoa(i)+" date:", date.Format(utils.FormatDate)+";sleep:", strconv.Itoa(int(rn)))
+		time.Sleep(time.Duration(rn) * time.Second)
+	}
+
+	//日成交持仓排名
+	for i := n; i >= 0; i-- {
+		date := time.Now().AddDate(0, 0, -i)
+		if date.Weekday() == time.Saturday || date.Weekday() == time.Sunday {
+			continue
+		}
+		fmt.Println("date:", date.Format(utils.FormatDate))
+		CrawlerFromGuangzhouDealPosiQuotes(date)
+		rn := utils.RangeRand(3, 15)
+		fmt.Println("i:"+strconv.Itoa(i)+" date:", date.Format(utils.FormatDate)+";sleep:", strconv.Itoa(int(rn)))
+		time.Sleep(time.Duration(rn) * time.Second)
+	}
+	//广州交易所期货-仓单日报->开始日期:2023-06-20
+	for i := n; i >= 0; i-- {
+		date := time.Now().AddDate(0, 0, -i)
+		if date.Weekday() == time.Saturday || date.Weekday() == time.Sunday {
+			continue
+		}
+		fmt.Println("date:", date.Format(utils.FormatDate))
+		CrawlerFromGuangzhouWbillWeeklyQuotes(date)
+		rn := utils.RangeRand(3, 15)
+		fmt.Println("i:"+strconv.Itoa(i)+" date:", date.Format(utils.FormatDate)+";sleep:", strconv.Itoa(int(rn)))
+		time.Sleep(time.Duration(rn) * time.Second)
+	}
+
+	//一次性交割卖方仓单查询
+	for i := n; i >= 0; i-- {
+		date := time.Now().AddDate(0, -i, 0)
+		fmt.Println("date:", date.Format(utils.FormatDate))
+		CrawlerFromGuangzhouCongregateDeliveryQuotes(date)
+		rn := utils.RangeRand(3, 15)
+		fmt.Println("i:"+strconv.Itoa(i)+" date:", date.Format(utils.FormatDate)+";sleep:", strconv.Itoa(int(rn)))
+		time.Sleep(time.Duration(rn) * time.Second)
+	}
+	//更新指标日期
+	CrawlerFromGuangzhouIndexMinAndMaxDate()
 	fmt.Println("task end")
 }

services/trade_guangzhou.go

@@ -116,144 +116,237 @@ func CrawlerFromGuangzhouDayQuotes(date time.Time) {
 		} else {
 			classifyId = classifyItem.BaseFromTradeGuangzhouClassifyId
 		}
-		if dv.DelivMonth == "" {
+		if dv.DelivMonth == "" && !strings.Contains(dv.Variety, "小计") {
 			continue
 		}
-		//指标编码
-		//开盘价
-		openIndexCode := dv.VarietyOrder + dv.DelivMonth + "open"
-		//最高价
-		highIndexCode := dv.VarietyOrder + dv.DelivMonth + "high"
-		//最低价
-		lowIndexCode := dv.VarietyOrder + dv.DelivMonth + "low"
-		//收盘价
-		closeIndexCode := dv.VarietyOrder + dv.DelivMonth + "close"
-		//前结算价
-		lastClearIndexCode := dv.VarietyOrder + dv.DelivMonth + "lastClear"
-		//结算价
-		clearPriceIndexCode := dv.VarietyOrder + dv.DelivMonth + "clearPrice"
-		//涨跌
-		diffIndexCode := dv.VarietyOrder + dv.DelivMonth + "diff"
-		//涨跌1
-		diff1IndexCode := dv.VarietyOrder + dv.DelivMonth + "diff1"
-		//成交量
-		volumeIndexCode := dv.VarietyOrder + dv.DelivMonth + "volumn"
-		//持仓量
-		openInterestIndexCode := dv.VarietyOrder + dv.DelivMonth + "openInterest"
-		//持仓量变化
-		diffIChangeIndexCode := dv.VarietyOrder + dv.DelivMonth + "diffI"
-		//成交额
-		turnoverIndexCode := dv.VarietyOrder + dv.DelivMonth + "turnover"
-
-		handleIndexCodeArr := []string{openIndexCode, highIndexCode, lowIndexCode, closeIndexCode, lastClearIndexCode, clearPriceIndexCode, diffIndexCode, diff1IndexCode, volumeIndexCode, openInterestIndexCode, diffIChangeIndexCode, turnoverIndexCode}
-		prefxCodeArr := []string{"open", "high", "low", "close", "lastClear", "clearPrice", "diff", "diff1", "volumn", "openInterest", "diffI", "turnover"}
-		handleIndexNameArr := []string{"开盘价", "最高价", "最低价", "收盘价", "前结算价", "结算价", "涨跌", "涨跌1", "成交量", "持仓量", "持仓量变化", "成交额"}
-
-		for ck, cv := range handleIndexCodeArr {
-			var indexId int
-			handleIndexCode := cv
-
-			if indexItem, ok := indexMap[handleIndexCode]; !ok {
-				indexName := dv.Variety + dv.DelivMonth + handleIndexNameArr[ck]
-
-				var unit string
-				if strings.Contains(indexName, "价") || strings.Contains(indexName, "涨跌") {
-					unit = "元/吨"
-				} else if strings.Contains(indexName, "量") {
-					unit = "手"
-				} else if strings.Contains(indexName, "成交额") {
-					unit = "万元"
-				}
 
-				gzObj = &models.BaseFromTradeGuangzhouIndex{
-					BaseFromTradeGuangzhouClassifyId: classifyId,
-					IndexCode:                        handleIndexCode,
-					IndexName:                        indexName,
-					Frequency:                        "日度",
-					Unit:                             unit,
-					CreateTime:                       time.Now(),
-					ModifyTime:                       time.Now(),
+		if !strings.Contains(dv.Variety, "小计") {
+			//开盘价
+			openIndexCode := dv.VarietyOrder + dv.DelivMonth + "open"
+			//最高价
+			highIndexCode := dv.VarietyOrder + dv.DelivMonth + "high"
+			//最低价
+			lowIndexCode := dv.VarietyOrder + dv.DelivMonth + "low"
+			//收盘价
+			closeIndexCode := dv.VarietyOrder + dv.DelivMonth + "close"
+			//前结算价
+			lastClearIndexCode := dv.VarietyOrder + dv.DelivMonth + "lastClear"
+			//结算价
+			clearPriceIndexCode := dv.VarietyOrder + dv.DelivMonth + "clearPrice"
+			//涨跌
+			diffIndexCode := dv.VarietyOrder + dv.DelivMonth + "diff"
+			//涨跌1
+			diff1IndexCode := dv.VarietyOrder + dv.DelivMonth + "diff1"
+			//成交量
+			volumeIndexCode := dv.VarietyOrder + dv.DelivMonth + "volumn"
+			//持仓量
+			openInterestIndexCode := dv.VarietyOrder + dv.DelivMonth + "openInterest"
+			//持仓量变化
+			diffIChangeIndexCode := dv.VarietyOrder + dv.DelivMonth + "diffI"
+			//成交额
+			turnoverIndexCode := dv.VarietyOrder + dv.DelivMonth + "turnover"
+
+			handleIndexCodeArr := []string{openIndexCode, highIndexCode, lowIndexCode, closeIndexCode, lastClearIndexCode, clearPriceIndexCode, diffIndexCode, diff1IndexCode, volumeIndexCode, openInterestIndexCode, diffIChangeIndexCode, turnoverIndexCode}
+			prefxCodeArr := []string{"open", "high", "low", "close", "lastClear", "clearPrice", "diff", "diff1", "volumn", "openInterest", "diffI", "turnover"}
+			handleIndexNameArr := []string{"开盘价", "最高价", "最低价", "收盘价", "前结算价", "结算价", "涨跌", "涨跌1", "成交量", "持仓量", "持仓量变化", "成交额"}
+
+			for ck, cv := range handleIndexCodeArr {
+				var indexId int
+				handleIndexCode := cv
+
+				if indexItem, ok := indexMap[handleIndexCode]; !ok {
+					indexName := dv.Variety + dv.DelivMonth + handleIndexNameArr[ck]
+
+					var unit string
+					if strings.Contains(indexName, "价") || strings.Contains(indexName, "涨跌") {
+						unit = "元/吨"
+					} else if strings.Contains(indexName, "量") {
+						unit = "手"
+					} else if strings.Contains(indexName, "成交额") {
+						unit = "万元"
+					}
+
+					gzObj = &models.BaseFromTradeGuangzhouIndex{
+						BaseFromTradeGuangzhouClassifyId: classifyId,
+						IndexCode:                        handleIndexCode,
+						IndexName:                        indexName,
+						Frequency:                        "日度",
+						Unit:                             unit,
+						CreateTime:                       time.Now(),
+						ModifyTime:                       time.Now(),
+					}
+					lastId, err := gzObj.AddBaseFromTradeGuangzhouIndex()
+					if err != nil {
+						fmt.Println("AddBaseFromTradeGuangzhouIndex Err:" + err.Error())
+						return
+					}
+					indexId = int(lastId)
+				} else {
+					indexId = indexItem.BaseFromTradeGuangzhouIndexId
 				}
-				lastId, err := gzObj.AddBaseFromTradeGuangzhouIndex()
-				if err != nil {
-					fmt.Println("AddBaseFromTradeGuangzhouIndex Err:" + err.Error())
-					return
+
+				if dataItem, ok := dataMap[handleIndexCode]; !ok {
+					dataItem = new(models.BaseFromTradeGuangzhouData)
+					dataItem.BaseFromTradeGuangzhouIndexId = indexId
+					dataItem.IndexCode = handleIndexCode
+					dataItem.DataTime = dateStr
+					switch prefxCodeArr[ck] {
+					case "open":
+						dataItem.Value = float64(dv.Open)
+					case "high":
+						dataItem.Value = float64(dv.High)
+					case "low":
+						dataItem.Value = float64(dv.Low)
+					case "close":
+						dataItem.Value = float64(dv.Close)
+					case "lastClear":
+						dataItem.Value = float64(dv.LastClear)
+					case "clearPrice":
+						dataItem.Value = float64(dv.ClearPrice)
+					case "diff":
+						dataItem.Value = float64(dv.Diff)
+					case "diff1":
+						dataItem.Value = float64(dv.Diff1)
+					case "volumn":
+						dataItem.Value = float64(dv.Volumn)
+					case "openInterest":
+						dataItem.Value = float64(dv.OpenInterest)
+					case "diffI":
+						dataItem.Value = float64(dv.DiffI)
+					case "turnover":
+						dataItem.Value = dv.Turnover
+					}
+					dataItem.CreateTime = time.Now()
+					dataItem.ModifyTime = time.Now()
+					addDataList = append(addDataList, dataItem)
+				} else {
+					var getValue float64
+					switch prefxCodeArr[ck] {
+					case "open":
+						getValue = float64(dv.Open)
+					case "high":
+						getValue = float64(dv.High)
+					case "low":
+						getValue = float64(dv.Low)
+					case "close":
+						getValue = float64(dv.Close)
+					case "lastClear":
+						getValue = float64(dv.LastClear)
+					case "clearPrice":
+						getValue = float64(dv.ClearPrice)
+					case "diff":
+						getValue = float64(dv.Diff)
+					case "diff1":
+						getValue = float64(dv.Diff1)
+					case "volumn":
+						getValue = float64(dv.Volumn)
+					case "openInterest":
+						getValue = float64(dv.OpenInterest)
+					case "diffI":
+						getValue = float64(dv.DiffI)
+					case "turnover":
+						getValue = dv.Turnover
+					}
+					if dataItem.Value != getValue {
+						dataItem.BaseFromTradeGuangzhouDataId = dataItem.BaseFromTradeGuangzhouDataId
+						dataItem.BaseFromTradeGuangzhouIndexId = indexId
+						dataItem.IndexCode = handleIndexCode
+						dataItem.DataTime = dateStr
+						dataItem.Value = getValue
+						dataItem.ModifyTime = time.Now()
+						updateDataList = append(updateDataList, dataItem)
+					}
 				}
-				indexId = int(lastId)
-			} else {
-				indexId = indexItem.BaseFromTradeGuangzhouIndexId
 			}
+		} else {
+			verietyPy := utils.ChineseToPinyinInitials("日行情" + dv.Variety)
+			//成交量
+			volumeIndexCode := dv.VarietyOrder + verietyPy + "volumn"
+			//持仓量
+			openInterestIndexCode := dv.VarietyOrder + verietyPy + "openInterest"
+			//持仓量变化
+			diffIChangeIndexCode := dv.VarietyOrder + verietyPy + "diffI"
+			//成交额
+			turnoverIndexCode := dv.VarietyOrder + verietyPy + "turnover"
+
+			handleIndexCodeArr := []string{volumeIndexCode, openInterestIndexCode, diffIChangeIndexCode, turnoverIndexCode}
+			prefxCodeArr := []string{"volumn", "openInterest", "diffI", "turnover"}
+			handleIndexNameArr := []string{"成交量", "持仓量", "持仓量变化", "成交额"}
+
+			for ck, cv := range handleIndexCodeArr {
+				var indexId int
+				handleIndexCode := cv
+
+				if indexItem, ok := indexMap[handleIndexCode]; !ok {
+					indexName := "日行情" + dv.Variety + handleIndexNameArr[ck]
+
+					var unit string
+					if strings.Contains(indexName, "价") || strings.Contains(indexName, "涨跌") {
+						unit = "元/吨"
+					} else if strings.Contains(indexName, "量") {
+						unit = "手"
+					} else if strings.Contains(indexName, "成交额") {
+						unit = "万元"
+					}
 
-			if dataItem, ok := dataMap[handleIndexCode]; !ok {
-				dataItem = new(models.BaseFromTradeGuangzhouData)
-				dataItem.BaseFromTradeGuangzhouIndexId = indexId
-				dataItem.IndexCode = handleIndexCode
-				dataItem.DataTime = dateStr
-				switch prefxCodeArr[ck] {
-				case "open":
-					dataItem.Value = float64(dv.Open)
-				case "high":
-					dataItem.Value = float64(dv.High)
-				case "low":
-					dataItem.Value = float64(dv.Low)
-				case "close":
-					dataItem.Value = float64(dv.Close)
-				case "lastClear":
-					dataItem.Value = float64(dv.LastClear)
-				case "clearPrice":
-					dataItem.Value = float64(dv.ClearPrice)
-				case "diff":
-					dataItem.Value = float64(dv.Diff)
-				case "diff1":
-					dataItem.Value = float64(dv.Diff1)
-				case "volumn":
-					dataItem.Value = float64(dv.Volumn)
-				case "openInterest":
-					dataItem.Value = float64(dv.OpenInterest)
-				case "diffI":
-					dataItem.Value = float64(dv.DiffI)
-				case "turnover":
-					dataItem.Value = dv.Turnover
-				}
-				dataItem.CreateTime = time.Now()
-				dataItem.ModifyTime = time.Now()
-				addDataList = append(addDataList, dataItem)
-			} else {
-				var getValue float64
-				switch prefxCodeArr[ck] {
-				case "open":
-					getValue = float64(dv.Open)
-				case "high":
-					getValue = float64(dv.High)
-				case "low":
-					getValue = float64(dv.Low)
-				case "close":
-					getValue = float64(dv.Close)
-				case "lastClear":
-					getValue = float64(dv.LastClear)
-				case "clearPrice":
-					getValue = float64(dv.ClearPrice)
-				case "diff":
-					getValue = float64(dv.Diff)
-				case "diff1":
-					getValue = float64(dv.Diff1)
-				case "volumn":
-					getValue = float64(dv.Volumn)
-				case "openInterest":
-					getValue = float64(dv.OpenInterest)
-				case "diffI":
-					getValue = float64(dv.DiffI)
-				case "turnover":
-					getValue = dv.Turnover
+					gzObj = &models.BaseFromTradeGuangzhouIndex{
+						BaseFromTradeGuangzhouClassifyId: classifyId,
+						IndexCode:                        handleIndexCode,
+						IndexName:                        indexName,
+						Frequency:                        "日度",
+						Unit:                             unit,
+						CreateTime:                       time.Now(),
+						ModifyTime:                       time.Now(),
+					}
+					lastId, err := gzObj.AddBaseFromTradeGuangzhouIndex()
+					if err != nil {
+						fmt.Println("AddBaseFromTradeGuangzhouIndex Err:" + err.Error())
+						return
+					}
+					indexId = int(lastId)
+				} else {
+					indexId = indexItem.BaseFromTradeGuangzhouIndexId
 				}
-				if dataItem.Value != getValue {
-					dataItem.BaseFromTradeGuangzhouDataId = dataItem.BaseFromTradeGuangzhouDataId
+
+				if dataItem, ok := dataMap[handleIndexCode]; !ok {
+					dataItem = new(models.BaseFromTradeGuangzhouData)
 					dataItem.BaseFromTradeGuangzhouIndexId = indexId
 					dataItem.IndexCode = handleIndexCode
 					dataItem.DataTime = dateStr
-					dataItem.Value = getValue
+					switch prefxCodeArr[ck] {
+					case "volumn":
+						dataItem.Value = float64(dv.Volumn)
+					case "openInterest":
+						dataItem.Value = float64(dv.OpenInterest)
+					case "diffI":
+						dataItem.Value = float64(dv.DiffI)
+					case "turnover":
+						dataItem.Value = dv.Turnover
+					}
+					dataItem.CreateTime = time.Now()
 					dataItem.ModifyTime = time.Now()
-					updateDataList = append(updateDataList, dataItem)
+					addDataList = append(addDataList, dataItem)
+				} else {
+					var getValue float64
+					switch prefxCodeArr[ck] {
+					case "volumn":
+						getValue = float64(dv.Volumn)
+					case "openInterest":
+						getValue = float64(dv.OpenInterest)
+					case "diffI":
+						getValue = float64(dv.DiffI)
+					case "turnover":
+						getValue = dv.Turnover
+					}
+					if dataItem.Value != getValue {
+						dataItem.BaseFromTradeGuangzhouDataId = dataItem.BaseFromTradeGuangzhouDataId
+						dataItem.BaseFromTradeGuangzhouIndexId = indexId
+						dataItem.IndexCode = handleIndexCode
+						dataItem.DataTime = dateStr
+						dataItem.Value = getValue
+						dataItem.ModifyTime = time.Now()
+						updateDataList = append(updateDataList, dataItem)
+					}
 				}
 			}
 		}
@@ -442,30 +535,37 @@ func CrawlerFromGuangzhouDealPosiQuotes(date time.Time) {
 				addDataList := make([]*models.BaseFromTradeGuangzhouData, 0)
 				updateDataList := make([]*models.BaseFromTradeGuangzhouData, 0)
 				for _, v := range dealPosiQuotes.Data {
-					if v.Abbr == "总计" {
-						continue
-					}
 					indexCode := utils.ChineseToPinyin(v.Abbr)
-					if indexCode == "" {
-						continue
-					}
 					var indexName string
 					switch pv {
 					case 1:
 						indexName = v.Abbr + "_" + cv.Contract + "_" + "成交量"
 						indexCode = indexCode + cv.Contract + "deal"
+						if v.ContractId == "总计" {
+							indexCode = utils.ChineseToPinyinInitials("日成交持仓排名总计") + cv.Contract + "deal"
+							indexName = "日成交持仓排名_" + cv.Contract + "_成交量总计"
+						}
 					case 2:
 						indexName = v.Abbr + "_" + cv.Contract + "_" + "持买单量"
 						indexCode = indexCode + cv.Contract + "buy"
+						if v.ContractId == "总计" {
+							indexCode = utils.ChineseToPinyinInitials("日成交持仓排名总计") + cv.Contract + "buy"
+							indexName = "日成交持仓排名_" + cv.Contract + "_持买单量总计"
+						}
 					case 3:
 						indexName = v.Abbr + "_" + cv.Contract + "_" + "持卖单量"
 						indexCode = indexCode + cv.Contract + "sold"
+						if v.ContractId == "总计" {
+							indexCode = utils.ChineseToPinyinInitials("日成交持仓排名总计") + cv.Contract + "sold"
+							indexName = "日成交持仓排名_" + cv.Contract + "_持卖单量总计"
+						}
 					}
 
 					var indexId int
 					if indexItem, ok := indexMap[indexCode]; !ok {
 						gzObj = &models.BaseFromTradeGuangzhouIndex{
 							BaseFromTradeGuangzhouClassifyId: cv.BaseFromTradeGuangzhouClassifyId,
+							BaseFromTradeGuangzhouContractId: cv.BaseFromTradeGuangzhouContractId,
 							IndexCode:                        indexCode,
 							IndexName:                        indexName,
 							Frequency:                        "日度",
@@ -606,75 +706,143 @@ func CrawlerFromGuangzhouWbillWeeklyQuotes(date time.Time) {
 		addDataList := make([]*models.BaseFromTradeGuangzhouData, 0)
 		updateDataList := make([]*models.BaseFromTradeGuangzhouData, 0)
 		for _, v := range wbillQty.Data {
-			if v.WhAbbr == "总计" {
+			if v.Variety == "总计" {
 				continue
 			}
-			indexCode := utils.ChineseToPinyinInitials(v.WhAbbr)
-			if indexCode == "" {
-				continue
-			}
-			indexCode = v.VarietyOrder + indexCode
-			indexCode += "cdl"
-			indexName := v.Variety + v.WhAbbr + "仓单量"
-
-			var indexId int
-			if indexItem, ok := indexMap[indexCode]; !ok {
-				var classifyId int
-				if classifyItem, ok := classifyMap[v.VarietyOrder]; !ok {
-					classifyItem = new(models.BaseFromTradeGuangzhouClassify)
-					classifyItem.ClassifyCode = v.VarietyOrder
-					classifyItem.ClassifyName = v.Variety
-					classifyItem.ParentId = 3
-					classifyItem.ModifyTime = time.Now()
-					classifyItem.CreateTime = time.Now()
-					classifyLastId, err := gzObj.AddBaseFromTradeGuangzhouClassify(classifyItem)
+			if !strings.Contains(v.Variety, "小计") {
+				indexCode := utils.ChineseToPinyinInitials(v.WhAbbr)
+				if indexCode == "" {
+					continue
+				}
+				indexCode = v.VarietyOrder + indexCode
+				indexCode += "cdl"
+				indexName := v.Variety + v.WhAbbr + "仓单量"
+
+				var indexId int
+				if indexItem, ok := indexMap[indexCode]; !ok {
+					var classifyId int
+					if classifyItem, ok := classifyMap[v.VarietyOrder]; !ok {
+						classifyItem = new(models.BaseFromTradeGuangzhouClassify)
+						classifyItem.ClassifyCode = v.VarietyOrder
+						classifyItem.ClassifyName = v.Variety
+						classifyItem.ParentId = 3
+						classifyItem.ModifyTime = time.Now()
+						classifyItem.CreateTime = time.Now()
+						classifyLastId, err := gzObj.AddBaseFromTradeGuangzhouClassify(classifyItem)
+						if err != nil {
+							fmt.Println("AddBaseFromTradeGuangzhouClassify err:" + err.Error())
+							return
+						}
+						classifyId = int(classifyLastId)
+					} else {
+						classifyId = classifyItem.BaseFromTradeGuangzhouClassifyId
+					}
+					gzObj = &models.BaseFromTradeGuangzhouIndex{
+						BaseFromTradeGuangzhouClassifyId: classifyId,
+						IndexCode:                        indexCode,
+						IndexName:                        indexName,
+						Frequency:                        "日度",
+						Unit:                             "手",
+						CreateTime:                       time.Now(),
+						ModifyTime:                       time.Now(),
+					}
+					lastId, err := gzObj.AddBaseFromTradeGuangzhouIndex()
 					if err != nil {
-						fmt.Println("AddBaseFromTradeGuangzhouClassify err:" + err.Error())
+						fmt.Println("AddBaseFromTradeGuangzhouIndex Err:" + err.Error())
 						return
 					}
-					classifyId = int(classifyLastId)
+					indexId = int(lastId)
 				} else {
-					classifyId = classifyItem.BaseFromTradeGuangzhouClassifyId
+					indexId = indexItem.BaseFromTradeGuangzhouIndexId
 				}
-				gzObj = &models.BaseFromTradeGuangzhouIndex{
-					BaseFromTradeGuangzhouClassifyId: classifyId,
-					IndexCode:                        indexCode,
-					IndexName:                        indexName,
-					Frequency:                        "日度",
-					Unit:                             "手",
-					CreateTime:                       time.Now(),
-					ModifyTime:                       time.Now(),
-				}
-				lastId, err := gzObj.AddBaseFromTradeGuangzhouIndex()
-				if err != nil {
-					fmt.Println("AddBaseFromTradeGuangzhouIndex Err:" + err.Error())
-					return
+				//判断数据是否存在
+				if dataItem, ok := dataMap[indexCode]; !ok {
+					dataItem = new(models.BaseFromTradeGuangzhouData)
+					dataItem.BaseFromTradeGuangzhouIndexId = indexId
+					dataItem.IndexCode = indexCode
+					dataItem.DataTime = dateStr
+					dataItem.Value = float64(v.WbillQty)
+					dataItem.QtySub = float64(v.RegWbillQty)
+					dataItem.CreateTime = time.Now()
+					dataItem.ModifyTime = time.Now()
+					addDataList = append(addDataList, dataItem)
+				} else {
+					if dataItem.Value != float64(v.WbillQty) {
+						dataItem.BaseFromTradeGuangzhouDataId = dataItem.BaseFromTradeGuangzhouDataId
+						dataItem.BaseFromTradeGuangzhouIndexId = indexId
+						dataItem.IndexCode = indexCode
+						dataItem.DataTime = dateStr
+						dataItem.Value = float64(v.WbillQty)
+						dataItem.QtySub = float64(v.RegWbillQty)
+						dataItem.ModifyTime = time.Now()
+						updateDataList = append(updateDataList, dataItem)
+					}
 				}
-				indexId = int(lastId)
 			} else {
-				indexId = indexItem.BaseFromTradeGuangzhouIndexId
-			}
-			//判断数据是否存在
-			if dataItem, ok := dataMap[indexCode]; !ok {
-				dataItem = new(models.BaseFromTradeGuangzhouData)
-				dataItem.BaseFromTradeGuangzhouIndexId = indexId
-				dataItem.IndexCode = indexCode
-				dataItem.DataTime = dateStr
-				dataItem.Value = float64(v.WbillQty)
-				dataItem.QtySub = float64(v.RegWbillQty)
-				dataItem.CreateTime = time.Now()
-				dataItem.ModifyTime = time.Now()
-				addDataList = append(addDataList, dataItem)
-			} else {
-				if dataItem.Value != float64(v.WbillQty) {
-					dataItem.BaseFromTradeGuangzhouDataId = dataItem.BaseFromTradeGuangzhouDataId
+				indexCode := utils.ChineseToPinyinInitials("仓单日报_" + v.Variety)
+				indexCode = v.VarietyOrder + indexCode
+				indexCode += "cdl"
+				indexName := "仓单日报_" + v.Variety + "_仓单量"
+
+				var indexId int
+				if indexItem, ok := indexMap[indexCode]; !ok {
+					var classifyId int
+					if classifyItem, ok := classifyMap[v.VarietyOrder]; !ok {
+						classifyItem = new(models.BaseFromTradeGuangzhouClassify)
+						classifyItem.ClassifyCode = v.VarietyOrder
+						classifyItem.ClassifyName = v.Variety
+						classifyItem.ParentId = 3
+						classifyItem.ModifyTime = time.Now()
+						classifyItem.CreateTime = time.Now()
+						classifyLastId, err := gzObj.AddBaseFromTradeGuangzhouClassify(classifyItem)
+						if err != nil {
+							fmt.Println("AddBaseFromTradeGuangzhouClassify err:" + err.Error())
+							return
+						}
+						classifyId = int(classifyLastId)
+					} else {
+						classifyId = classifyItem.BaseFromTradeGuangzhouClassifyId
+					}
+					gzObj = &models.BaseFromTradeGuangzhouIndex{
+						BaseFromTradeGuangzhouClassifyId: classifyId,
+						IndexCode:                        indexCode,
+						IndexName:                        indexName,
+						Frequency:                        "日度",
+						Unit:                             "手",
+						CreateTime:                       time.Now(),
+						ModifyTime:                       time.Now(),
+					}
+					lastId, err := gzObj.AddBaseFromTradeGuangzhouIndex()
+					if err != nil {
+						fmt.Println("AddBaseFromTradeGuangzhouIndex Err:" + err.Error())
+						return
+					}
+					indexId = int(lastId)
+				} else {
+					indexId = indexItem.BaseFromTradeGuangzhouIndexId
+				}
+				//判断数据是否存在
+				if dataItem, ok := dataMap[indexCode]; !ok {
+					dataItem = new(models.BaseFromTradeGuangzhouData)
 					dataItem.BaseFromTradeGuangzhouIndexId = indexId
 					dataItem.IndexCode = indexCode
 					dataItem.DataTime = dateStr
 					dataItem.Value = float64(v.WbillQty)
 					dataItem.QtySub = float64(v.RegWbillQty)
+					dataItem.CreateTime = time.Now()
 					dataItem.ModifyTime = time.Now()
-					updateDataList = append(updateDataList, dataItem)
+					addDataList = append(addDataList, dataItem)
+				} else {
+					if dataItem.Value != float64(v.WbillQty) {
+						dataItem.BaseFromTradeGuangzhouDataId = dataItem.BaseFromTradeGuangzhouDataId
+						dataItem.BaseFromTradeGuangzhouIndexId = indexId
+						dataItem.IndexCode = indexCode
+						dataItem.DataTime = dateStr
+						dataItem.Value = float64(v.WbillQty)
+						dataItem.QtySub = float64(v.RegWbillQty)
+						dataItem.ModifyTime = time.Now()
+						updateDataList = append(updateDataList, dataItem)
+					}
 				}
 			}
 		}