大连数据爬取修改

models/base_from_trade_dalian.go

@@ -5,32 +5,43 @@ import (
 	"time"
 )
 
-type BaseFromTradeDalianIndex struct {
-	BaseFromTradeDalianIndexId int `orm:"column(base_from_trade_dalian_index_id);pk"`
-	IndexName                  string
-	IndexCode                  string
-	Frequency                  string
-	StartDate                  time.Time
-	EndDate                    time.Time
-	CreateTime                 time.Time
-	ModifyTime                 time.Time
-	ClassifyName               string
-	ClassifyType               string
-}
+//type BaseFromTradeDalianIndex struct {
+//	BaseFromTradeDalianIndexId int `orm:"column(base_from_trade_dalian_index_id);pk"`
+//	IndexName                  string
+//	IndexCode                  string
+//	Frequency                  string
+//	StartDate                  time.Time
+//	EndDate                    time.Time
+//	CreateTime                 time.Time
+//	ModifyTime                 time.Time
+//	ClassifyName               string
+//	ClassifyType               string
+//}
 
-type BaseFromTradeDalianIndexItem struct {
-	BaseFromTradeDalianIndexId int `orm:"column(base_from_trade_dalian_index_id);pk"`
-	IndexName                  string
-	IndexCode                  string
-	Frequency                  string
-	Value                      string
-	AddCutValue                string
-	ClassifyName               string
-	ClassifyType               string
-	StartDate                  time.Time
-	EndDate                    time.Time
-	CreateTime                 time.Time
-	ModifyTime                 time.Time
+type BaseFromTradeDalianIndex struct {
+	BaseFromTradeDalianIndexId int       `orm:"column(base_from_trade_dalian_index_id);pk"`
+	Rank                       string    `description:"排名"`
+	DealShortName              string    `description:"成交量公司简称"`
+	DealName                   string    `description:"成交量指标名称"`
+	DealCode                   string    `description:"成交量指标编码"`
+	DealValue                  string    `description:"成交量"`
+	DealChange                 string    `description:"成交变化量"`
+	BuyShortName               string    `description:"成交量公司简称"`
+	BuyName                    string    `description:"持买单量指标名称"`
+	BuyCode                    string    `description:"持买单量指标编码"`
+	BuyValue                   string    `description:"持买单量"`
+	BuyChange                  string    `description:"持买单量变化量"`
+	SoldShortName              string    `description:"成交量公司简称"`
+	SoldName                   string    `description:"持买单量指标名称"`
+	SoldCode                   string    `description:"持买单量指标编码"`
+	SoldValue                  string    `description:"持买单量"`
+	SoldChange                 string    `description:"持买单量变化量"`
+	Frequency                  string    `description:"频度"`
+	ClassifyName               string    `description:"分类名称"`
+	ClassifyType               string    `description:"分类名称下的类型"`
+	CreateTime                 time.Time `description:"插入时间"`
+	ModifyTime                 time.Time `description:"修改时间"`
+	DataTime                   time.Time `description:"数据日期"`
 }
 
 func AddBaseFromTradeDalianIndex(item *BaseFromTradeDalianIndex) (lastId int64, err error) {

services/commodity_trade_dalian.go

@@ -109,7 +109,7 @@ func SyncRankingFromDalian() {
 					fmt.Println(err)
 					return
 				}
-				err = DoHtml(string(body), v2, "", exitProductMap[k], time.Now())
+				err = DoHtml(string(body), v2, exitProductMap[k], time.Now())
 				//fmt.Println(err)
 				//fmt.Println("解析：", v2, exitProductMap[k])
 			}
@@ -718,169 +718,82 @@ func DoSearch(body string) (exitProductMaps, exitContractIdMaps, varietyArrMaps
 }
 
 //处理解析Html
-func DoHtml(body, name, carietyCode, contractId string, timeDate time.Time) (err error) {
+func DoHtml(body, name, contractId string, dateTime time.Time) (err error) {
 	str := body
 	doc, err := goquery.NewDocumentFromReader(strings.NewReader(str))
 	if err != nil {
 		log.Fatal(err)
 	}
 	table := doc.Find("table")
-	var classifyName, tradeDate string
-	var indexCode string
-	var dealSuffix, dealZjSuffix1, dealZjSuffix2, dealZjSuffix3, buySuffix, sellSuffix, userName1, userName2, userName3 string
-
+	var rank, shortName, dealValue, dealChange, buyName, buyValue, buyChange, soldName, soldValue, soldChange string
 	table.Find("tr").Each(func(i int, tr *goquery.Selection) {
 		tds := tr.Find("td")
-		//fmt.Println( tds.Length(),"长度：",i)
+		//fmt.Println(tds.Length(), "长度：", i)
 		if tds.Length() == 0 || tds.Length() == 7 || i == 23 {
 			tdText := tds.Text()
 			utils.FileLog.Info(tdText)
-			if tdText != "" {
-				tdTextArr := strings.Split(tdText, "    ")
-				for k, v := range tdTextArr {
-					//fmt.Println(k, v)
-					if k == 0 {
-						classifyName = v
-					} else {
-						tradeDate = v
-					}
-				}
-			}
 		} else {
-			var items []*models.BaseFromTradeDalianIndexItem
-			item := new(models.BaseFromTradeDalianIndexItem)
-			item2 := new(models.BaseFromTradeDalianIndexItem)
-			item3 := new(models.BaseFromTradeDalianIndexItem)
+			item := new(models.BaseFromTradeDalianIndex)
 			tds.Each(func(tk int, td *goquery.Selection) {
 				tdText := td.Text()
-				//fmt.Println(tk,tdText)
 				if tk == 0 { //名次
-
+					rank = tdText
 				}
 				if tk == 1 { //会员简称
-					userName1 = tdText
+					shortName = tdText
 				}
 				if tk == 2 { //成交量
-					//dealSuffix = tdText
-					dealSuffix = strings.Replace(tdText, ",", "", -1)
-					fmt.Println("成交量", dealSuffix, "原文本", tdText)
+					dealValue = strings.Replace(tdText, ",", "", -1)
 				}
 				if tk == 3 { //增减
-					dealZjSuffix1 = strings.Replace(tdText, ",", "", -1)
-
+					dealChange = strings.Replace(tdText, ",", "", -1)
 				}
-
 				if tk == 5 { //会员简称
-					userName2 = tdText
+					buyName = tdText
 				}
 				if tk == 6 { //持买单量
-					buySuffix = strings.Replace(tdText, ",", "", -1)
+					buyValue = strings.Replace(tdText, ",", "", -1)
 				}
 				if tk == 7 { //增减
-					dealZjSuffix2 = strings.Replace(tdText, ",", "", -1)
+					buyChange = strings.Replace(tdText, ",", "", -1)
 				}
 
 				if tk == 9 { //会员简称
-					userName3 = tdText
+					soldName = tdText
 				}
-				if tk == 10 { //持买单量
-					sellSuffix = strings.Replace(tdText, ",", "", -1)
+				if tk == 10 { //持卖单量
+					soldValue = strings.Replace(tdText, ",", "", -1)
 				}
 				if tk == 11 { //增减
-					dealZjSuffix3 = strings.Replace(tdText, ",", "", -1)
+					soldChange = strings.Replace(tdText, ",", "", -1)
 				}
 			})
-			//fmt.Println(dealSuffix, "+", "+", "")
-			item.Frequency = contractId
+
+			item.Rank = rank
+			item.DealShortName = shortName
+			item.DealName = shortName + "_成交量_" + contractId
+			item.DealCode = fmt.Sprintf("D%s", time.Now().Format(utils.FormatDateTimeUnSpace)+strconv.Itoa(utils.GetRandInt(1, 100)))
+			item.DealValue = dealValue
+			item.BuyShortName = buyName
+			item.BuyName = buyName + "_持买单量_" + contractId
+			item.BuyCode = fmt.Sprintf("D%s", time.Now().Format(utils.FormatDateTimeUnSpace)+strconv.Itoa(utils.GetRandInt(1, 100)))
+			item.BuyValue = buyValue
+			item.BuyChange = buyChange
+			item.SoldShortName = soldName
+			item.SoldName = soldName + "_持买单量_" + contractId
+			item.SoldCode = fmt.Sprintf("D%s", time.Now().Format(utils.FormatDateTimeUnSpace)+strconv.Itoa(utils.GetRandInt(1, 100)))
+			item.SoldValue = soldValue
+			item.SoldChange = soldChange
+			item.Frequency = "日度"
 			item.ClassifyName = name
-			item.ClassifyType = carietyCode
-			item.Value = dealSuffix
-			item.AddCutValue = dealZjSuffix1
+			item.ClassifyType = contractId
 			item.CreateTime = time.Now()
 			item.ModifyTime = time.Now()
-			item.IndexName = userName1 + "_成交量_" + contractId
-			indexCode = fmt.Sprintf("D%s", time.Now().Format(utils.FormatDateTimeUnSpace)+strconv.Itoa(utils.GetRandInt(1, 100)))
-			item.IndexCode = indexCode
-			items = append(items, item)
-
-			item2.Frequency = contractId
-			item2.ClassifyName = name
-			item2.ClassifyType = carietyCode
-			item2.Value = buySuffix
-			item2.AddCutValue = dealZjSuffix2
-			item2.CreateTime = time.Now()
-			item2.ModifyTime = time.Now()
-			item2.IndexName = userName2 + "_持买单量_" + contractId
-			indexCode = fmt.Sprintf("D%s", time.Now().Format(utils.FormatDateTimeUnSpace)+strconv.Itoa(utils.GetRandInt(1, 100)))
-			item2.IndexCode = indexCode
-			items = append(items, item2)
-
-			item3.Frequency = contractId
-			item3.ClassifyName = name
-			item3.ClassifyType = carietyCode
-			item3.Value = sellSuffix
-			item3.AddCutValue = dealZjSuffix3
-			item3.CreateTime = time.Now()
-			item3.ModifyTime = time.Now()
-			item3.IndexName = userName1 + "_持卖单量_" + contractId
-			indexCode = fmt.Sprintf("D%s", time.Now().Format(utils.FormatDateTimeUnSpace)+strconv.Itoa(utils.GetRandInt(1, 100)))
-			item3.IndexCode = indexCode
-			items = append(items, item3)
-
-			//添加指标参数
-			for _, v := range items {
-				total, err := models.GetDalianIndexCountByIndexName(v.IndexName)
-				if err != nil {
-					fmt.Println(err)
-					return
-				}
-				if total == 0 {
-					additem := new(models.BaseFromTradeDalianIndex)
-					additem.Frequency = "日度"
-					additem.ClassifyName = v.ClassifyName
-					additem.ClassifyType = v.Frequency
-					additem.CreateTime = time.Now()
-					additem.ModifyTime = time.Now()
-					additem.IndexName = v.IndexName
-					additem.IndexCode = v.IndexCode
-					newId, err := models.AddBaseFromTradeDalianIndex(additem)
-					if err != nil {
-						fmt.Println(err)
-						return
-					}
-					fmt.Println("新增指标参数：", newId, v.IndexName)
-				}
-			}
-
-			//fmt.Println(i,name,carietyCode,contractId, tds.Text())
-			//if name != ""{
-			//	fmt.Println(i,name,code, tds.Text())
-			//}
-			//fmt.Println("len 10:" + tds.Text())
-
-			//添加数据
-			for _, v := range items {
-				detail, err := models.GetBaseFromTradeDalianIndexInfo(v.IndexName)
-				if err != nil {
-					fmt.Println(err, "名称", v.IndexName)
-					return
-				}
-				itemData := new(models.BaseFromTradeDalianData)
-				itemData.IndexCode = detail.IndexCode
-				itemData.BaseFromTradeDalianIndexId = detail.BaseFromTradeDalianIndexId
-				itemData.Value = v.Value
-				itemData.AddCutValue = v.AddCutValue
-				itemData.DataTime = timeDate
-				itemData.CreateTime = time.Now()
-				itemData.ModifyTime = time.Now()
-				itemData.DataTimestamp = strconv.FormatInt(time.Now().UnixNano()/1e6, 10)
-				//fmt.Println(itemData)
-				newId, err := models.AddBaseFromTradeDalianData(itemData)
-				if err != nil {
-					fmt.Println(err)
-					return
-				}
-				fmt.Println("新增数据：", newId, v.IndexName)
+			item.DataTime = dateTime
+			_, err := models.AddBaseFromTradeDalianIndex(item)
+			if err != nil {
+				fmt.Println(err)
+				return
 			}
 		}
 	})
@@ -924,8 +837,8 @@ func SyncRankingFromDalian2() {
 		SearchList{VarietyName: "液化石油气", CarietyCode: "pg", List: []SearchContractId{SearchContractId{ContractId: "pg2111"}, {ContractId: "pg2112"}, {ContractId: "pg2201"}, {ContractId: "pg2202"}}},
 	}
 	//定义爬取时间
-	//endDate := time.Now().AddDate(0, 0, -1).Format(utils.FormatDateTime)
-	endDate := time.Now().Format(utils.FormatDateTime)
+	endDate := time.Now().AddDate(0, 0, -1).Format(utils.FormatDateTime)
+	//endDate := time.Now().Format(utils.FormatDateTime)
 	timeDate := utils.StrTimeToTime(endDate)
 	currDate := timeDate.Format(utils.FormatDateUnSpace)
 	year := timeDate.Year()
@@ -938,7 +851,7 @@ func SyncRankingFromDalian2() {
 		dayStr = strconv.Itoa(day)
 	}
 	monthNum, _ := strconv.Atoi(month)
-	fmt.Println(currDate)
+	//fmt.Println(currDate)
 	month = strconv.Itoa(monthNum - 1) //获取时月份需要减一
 	for _, v := range searchList {
 		for _, v2 := range v.List {
@@ -956,7 +869,9 @@ func SyncRankingFromDalian2() {
 			_ = writer.WriteField("variety_id", v.CarietyCode)
 			_ = writer.WriteField("currDate", currDate)
 			err := writer.Close()
+			fmt.Println(currDate, v.VarietyName, v2.ContractId)
 			if err != nil {
+				utils.FileLog.Info("获取指标失败：" + currDate + v.VarietyName + v2.ContractId)
 				fmt.Println(err)
 				return
 			}
@@ -979,7 +894,7 @@ func SyncRankingFromDalian2() {
 				fmt.Println(err)
 				return
 			}
-			err = DoHtml(string(body), v.VarietyName, v.CarietyCode, v2.ContractId, timeDate)
+			err = DoHtml(string(body), v.VarietyName, v2.ContractId, timeDate)
 		}
 	}
 }