大连数据更新

models/base_from_trade_dalian.go

@@ -5,19 +5,6 @@ import (
 	"time"
 )
 
-//type BaseFromTradeDalianIndex struct {
-//	BaseFromTradeDalianIndexId int `orm:"column(base_from_trade_dalian_index_id);pk"`
-//	IndexName                  string
-//	IndexCode                  string
-//	Frequency                  string
-//	StartDate                  time.Time
-//	EndDate                    time.Time
-//	CreateTime                 time.Time
-//	ModifyTime                 time.Time
-//	ClassifyName               string
-//	ClassifyType               string
-//}
-
 type BaseFromTradeDalianIndex struct {
 	BaseFromTradeDalianIndexId int    `orm:"column(base_from_trade_dalian_index_id);pk"`
 	Rank                       string `description:"排名"`
@@ -42,7 +29,6 @@ type BaseFromTradeDalianIndex struct {
 	//CreateTime                 time.Time `description:"插入时间"`
 	//ModifyTime                 time.Time `description:"修改时间"`
 	//DataTime                   time.Time `description:"数据日期"`
-
 	CreateTime string `description:"插入时间"`
 	ModifyTime string `description:"修改时间"`
 	DataTime   string `description:"数据日期"`
@@ -55,24 +41,6 @@ func AddBaseFromTradeDalianIndex(item *BaseFromTradeDalianIndex) (lastId int64,
 	return
 }
 
-//详情
-func GetBaseFromTradeDalianIndexInfo(indexName string) (item *BaseFromTradeDalianIndex, err error) {
-	o := orm.NewOrm()
-	o.Using("data")
-	sql := `SELECT * FROM base_from_trade_dalian_index  WHERE index_name=? `
-	err = o.Raw(sql, indexName).QueryRow(&item)
-	return
-}
-
-//获取数量
-func GetDalianIndexCountByIndexName(indexName string) (count int, err error) {
-	o := orm.NewOrm()
-	o.Using("data")
-	sqlCount := `SELECT COUNT(1) AS count  FROM base_from_trade_dalian_index  WHERE index_name=?`
-	err = o.Raw(sqlCount, indexName).QueryRow(&count)
-	return
-}
-
 type BaseFromTradeDalianData struct {
 	BaseFromTradeDalianDataId  int `orm:"column(base_from_trade_dalian_data_id);pk"`
 	BaseFromTradeDalianIndexId int
@@ -85,13 +53,6 @@ type BaseFromTradeDalianData struct {
 	DataTimestamp              string
 }
 
-func AddBaseFromTradeDalianData(item *BaseFromTradeDalianData) (lastId int64, err error) {
-	o := orm.NewOrm()
-	o.Using("data")
-	lastId, err = o.Insert(item)
-	return
-}
-
 func GetAddSql(item *BaseFromTradeDalianIndex) (addSql string) {
 	addSql += "('" + item.Rank + "','" + item.DealShortName + "','" + item.DealName + "','" + item.DealCode + "','" + item.DealValue + "','" + item.BuyShortName + "','" + item.DealChange + "','" + item.BuyName + "','" + item.BuyCode + "','" + item.BuyValue + "','" + item.BuyChange + "','" + item.SoldShortName + "','" + item.SoldName + "','" + item.SoldCode + "','" + item.SoldValue + "','" + item.SoldChange + "','" + item.Frequency + "','" + item.ClassifyName + "','" + item.ClassifyType + "','" + item.CreateTime + "','" + item.ModifyTime + "','" + item.DataTime + "'),"
 	return
@@ -107,3 +68,20 @@ func RefreshEdbDataByDaLian(addSql string) (err error) {
 	}
 	return
 }
+
+//获取当天数据信息列表
+func GetBaseFromTradeDalianDataList(dateTime string) (items []*BaseFromTradeDalianIndex, err error) {
+	o := orm.NewOrm()
+	o.Using("data")
+	sql := `SELECT * FROM base_from_trade_dalian_index WHERE data_time=?`
+	_, err = o.Raw(sql, dateTime).QueryRows(&items)
+	return
+}
+
+func UpdateBaseFromTradeDalianIndex(item *BaseFromTradeDalianIndex, dataId int) (err error) {
+	o := orm.NewOrm()
+	o.Using("data")
+	sql := `UPDATE base_from_trade_dalian_index SET rank=?,deal_short_name=?,deal_name=?,deal_code=?,deal_value=?,buy_short_name=?,deal_change=?,buy_name=?,buy_code=?,buy_value=?,buy_change=?,sold_short_name=?,sold_name=?,sold_code=?,sold_value=?,sold_change=?,frequency=?,classify_name=?,classify_type=?,modify_time=?,data_time=? WHERE base_from_trade_dalian_index_id=? `
+	_, err = o.Raw(sql, item.Rank, item.DealShortName, item.DealName, item.DealCode, item.DealValue, item.BuyShortName, item.DealChange, item.BuyName, item.BuyCode, item.BuyValue, item.BuyChange, item.SoldShortName, item.SoldName, item.SoldCode, item.SoldValue, item.SoldChange, item.Frequency, item.ClassifyName, item.ClassifyType, item.ModifyTime, item.DataTime, dataId).Exec()
+	return
+}

services/commodity_trade_dalian.go

@@ -15,108 +15,6 @@ import (
 	"time"
 )
 
-//大连商品交易所持单排名
-func SyncRankingFromDalianOld() {
-	fmt.Println("start")
-	var str string
-	exitProductMap, exitContractIdMap, varietyArrMap := DoSearch(str)
-	//fmt.Println(exitProductMap)
-	//fmt.Println(exitContractIdMap)
-	//fmt.Println(varietyArrMap)
-	var varietyNum int
-	for k, v := range varietyArrMap {
-		if k < 3 {
-			if k-1 > varietyNum {
-				url := "http://www.dce.com.cn/publicweb/quotesdata/memberDealPosiQuotes.html"
-				method := "POST"
-				payload := &bytes.Buffer{}
-				writer := multipart.NewWriter(payload)
-				_ = writer.WriteField("memberDealPosiQuotes.variety", v)
-				_ = writer.WriteField("memberDealPosiQuotes.trade_type", "0")
-				_ = writer.WriteField("year", "2021")
-				_ = writer.WriteField("month", "9")
-				_ = writer.WriteField("day", "19")
-				_ = writer.WriteField("contract.variety_id", v)
-				_ = writer.WriteField("currDate", "20211019")
-				err := writer.Close()
-				if err != nil {
-					fmt.Println(err)
-					return
-				}
-				client := &http.Client{}
-				req, err := http.NewRequest(method, url, payload)
-				if err != nil {
-					fmt.Println(err)
-					return
-				}
-				req.Header.Add("Cookie", "JSESSIONID=36ACF02A59227A3854F9D5D5E2FB5F2E; WMONID=R5ojcAIIcx-")
-				req.Header.Set("Content-Type", writer.FormDataContentType())
-				res, err := client.Do(req)
-				if err != nil {
-					fmt.Println(err)
-					return
-				}
-				defer res.Body.Close()
-				body, err := ioutil.ReadAll(res.Body)
-				if err != nil {
-					fmt.Println(err)
-					return
-				}
-				_, exitContractIdMap, _ = DoSearch(string(body))
-				//fmt.Println(exitContractIdMap)
-			}
-		}
-		if k < -1 {
-			//fmt.Println(v)
-			for _, v2 := range exitContractIdMap {
-				//模拟form表单请求
-				url := "http://www.dce.com.cn/publicweb/quotesdata/memberDealPosiQuotes.html"
-				method := "POST"
-				payload := &bytes.Buffer{}
-				writer := multipart.NewWriter(payload)
-				_ = writer.WriteField("memberDealPosiQuotes.variety", v)
-				_ = writer.WriteField("memberDealPosiQuotes.trade_type", "0")
-				_ = writer.WriteField("year", "2021")
-				_ = writer.WriteField("month", "9")
-				_ = writer.WriteField("day", "19")
-				_ = writer.WriteField("contract.contract_id", v2)
-				_ = writer.WriteField("variety_id", v)
-				_ = writer.WriteField("currDate", "20211019")
-				err := writer.Close()
-				if err != nil {
-					fmt.Println(err)
-					return
-				}
-				client := &http.Client{}
-				req, err := http.NewRequest(method, url, payload)
-				if err != nil {
-					fmt.Println(err)
-					return
-				}
-				req.Header.Add("Cookie", "JSESSIONID=36ACF02A59227A3854F9D5D5E2FB5F2E; WMONID=R5ojcAIIcx-")
-				req.Header.Set("Content-Type", writer.FormDataContentType())
-				res, err := client.Do(req)
-				if err != nil {
-					fmt.Println(err)
-					return
-				}
-				defer res.Body.Close()
-				body, err := ioutil.ReadAll(res.Body)
-				if err != nil {
-					fmt.Println(err)
-					return
-				}
-				err = DoHtml(string(body), v2, exitProductMap[k], time.Now())
-				//fmt.Println(err)
-				//fmt.Println("解析：", v2, exitProductMap[k])
-			}
-			varietyNum++
-		}
-	}
-
-	fmt.Println("end")
-}
-
 //处理搜索条件
 func DoSearch(body string) (exitProductMaps, exitContractIdMaps, varietyArrMaps map[int]string) {
 	var str string
@@ -187,7 +85,7 @@ func DoSearch(body string) (exitProductMaps, exitContractIdMaps, varietyArrMaps
 }
 
 //处理解析Html
-func DoHtml(body, name, contractId string, dateTime time.Time) (err error) {
+func DoHtml(body, name, contractId string, dateTime time.Time, listDataMap map[string]int) (err error) {
 	defer func() {
 		if err != nil {
 			fmt.Println("RefreshDataFromDaLian Err:" + err.Error())
@@ -267,8 +165,16 @@ func DoHtml(body, name, contractId string, dateTime time.Time) (err error) {
 			item.CreateTime = time.Now().Format(utils.FormatDateTime)
 			item.ModifyTime = time.Now().Format(utils.FormatDateTime)
 			item.DataTime = dateTime.Format(utils.FormatDate)
-			addSql += models.GetAddSql(item)
-			isAdd = true
+			if val, ok := listDataMap[item.DealShortName+item.ClassifyType+item.DataTime]; !ok {
+				addSql += models.GetAddSql(item)
+				isAdd = true
+			} else {
+				//更新
+				err := models.UpdateBaseFromTradeDalianIndex(item, val)
+				if err != nil {
+					fmt.Println("UpdateBaseFromTradeDalianIndex err:", err)
+				}
+			}
 		}
 	})
 	addSql = strings.TrimRight(addSql, ",")
@@ -299,19 +205,6 @@ func SyncRankingFromDalianDo() {
 	}
 }
 
-func SyncRankingFromDalianDos() (err error) {
-	fmt.Println(66666)
-	defer func() {
-		if err != nil {
-			fmt.Println("RefreshDataFromWind Err:" + err.Error())
-			go utils.SendEmail(utils.APPNAME+"【"+utils.RunMode+"】"+"失败提醒", "RefreshDataFromWind ErrMsg:"+err.Error(), utils.EmailSendToUsers)
-		}
-	}()
-
-	//SyncRankingFromDalian2(1)
-	return err
-}
-
 //大连交易所持单排名
 func SyncRankingFromDalian(dayNum int) (err error) {
 	fmt.Println("start")
@@ -346,6 +239,7 @@ func SyncRankingFromDalian(dayNum int) (err error) {
 		SearchList{VarietyName: "乙二醇", CarietyCode: "eg", List: []SearchContractId{SearchContractId{ContractId: "eg2112"}, {ContractId: "eg2201"}, {ContractId: "eg2202"}, {ContractId: "eg2203"}, {ContractId: "eg2205"}}},
 		SearchList{VarietyName: "液化石油气", CarietyCode: "pg", List: []SearchContractId{SearchContractId{ContractId: "pg2111"}, {ContractId: "pg2112"}, {ContractId: "pg2201"}, {ContractId: "pg2202"}}},
 	}
+
 	//定义爬取时间
 	endDate := time.Now().AddDate(0, 0, -dayNum).Format(utils.FormatDateTime)
 	//endDate := time.Now().Format(utils.FormatDateTime)
@@ -362,6 +256,16 @@ func SyncRankingFromDalian(dayNum int) (err error) {
 	}
 	monthNum, _ := strconv.Atoi(month)
 	month = strconv.Itoa(monthNum - 1) //获取时月份需要减一
+
+	list, err := models.GetBaseFromTradeDalianDataList(timeDate.Format(utils.FormatDate))
+	listDataMap := make(map[string]int)
+	for _, v := range list {
+		listDataMap[v.DealShortName+v.ClassifyType+v.DataTime] = v.BaseFromTradeDalianIndexId
+	}
+	if err != nil {
+		fmt.Println(err)
+		return err
+	}
 	for _, v := range searchList {
 		for _, v2 := range v.List {
 			//模拟form表单请求
@@ -401,7 +305,7 @@ func SyncRankingFromDalian(dayNum int) (err error) {
 				go utils.SendEmail(utils.APPNAME+"【"+utils.RunMode+"】"+"失败提醒", "RefreshDataFromDalian ErrMsg:"+err.Error()+"获取指标失败："+currDate+v.VarietyName+v2.ContractId, utils.EmailSendToUsers)
 				return err
 			}
-			err = DoHtml(string(body), v.VarietyName, v2.ContractId, timeDate)
+			err = DoHtml(string(body), v.VarietyName, v2.ContractId, timeDate, listDataMap)
 			if err != nil {
 				return err
 			}
@@ -422,5 +326,3 @@ func DlIndexCodeGenerator(indexName, suffix string) string {
 	}
 	return ineIndexCode
 }
-
-//rank,deal_short_name,deal_name,deal_code,deal_value,buy_short_name,deal_change,buy_name,buy_code,buy_value,buy_change,sold_short_name,sold_name,sold_code,sold_value,sold_change,frequency,classify_name,classify_type,create_time,modify_time,data_time