大连数据插入方式修改

models/base_from_trade_dalian.go

@@ -19,29 +19,33 @@ import (
 //}
 
 type BaseFromTradeDalianIndex struct {
-	BaseFromTradeDalianIndexId int       `orm:"column(base_from_trade_dalian_index_id);pk"`
-	Rank                       string    `description:"排名"`
-	DealShortName              string    `description:"成交量公司简称"`
-	DealName                   string    `description:"成交量指标名称"`
-	DealCode                   string    `description:"成交量指标编码"`
-	DealValue                  string    `description:"成交量"`
-	DealChange                 string    `description:"成交变化量"`
-	BuyShortName               string    `description:"成交量公司简称"`
-	BuyName                    string    `description:"持买单量指标名称"`
-	BuyCode                    string    `description:"持买单量指标编码"`
-	BuyValue                   string    `description:"持买单量"`
-	BuyChange                  string    `description:"持买单量变化量"`
-	SoldShortName              string    `description:"成交量公司简称"`
-	SoldName                   string    `description:"持买单量指标名称"`
-	SoldCode                   string    `description:"持买单量指标编码"`
-	SoldValue                  string    `description:"持买单量"`
-	SoldChange                 string    `description:"持买单量变化量"`
-	Frequency                  string    `description:"频度"`
-	ClassifyName               string    `description:"分类名称"`
-	ClassifyType               string    `description:"分类名称下的类型"`
-	CreateTime                 time.Time `description:"插入时间"`
-	ModifyTime                 time.Time `description:"修改时间"`
-	DataTime                   time.Time `description:"数据日期"`
+	BaseFromTradeDalianIndexId int    `orm:"column(base_from_trade_dalian_index_id);pk"`
+	Rank                       string `description:"排名"`
+	DealShortName              string `description:"成交量公司简称"`
+	DealName                   string `description:"成交量指标名称"`
+	DealCode                   string `description:"成交量指标编码"`
+	DealValue                  string `description:"成交量"`
+	DealChange                 string `description:"成交变化量"`
+	BuyShortName               string `description:"成交量公司简称"`
+	BuyName                    string `description:"持买单量指标名称"`
+	BuyCode                    string `description:"持买单量指标编码"`
+	BuyValue                   string `description:"持买单量"`
+	BuyChange                  string `description:"持买单量变化量"`
+	SoldShortName              string `description:"成交量公司简称"`
+	SoldName                   string `description:"持买单量指标名称"`
+	SoldCode                   string `description:"持买单量指标编码"`
+	SoldValue                  string `description:"持买单量"`
+	SoldChange                 string `description:"持买单量变化量"`
+	Frequency                  string `description:"频度"`
+	ClassifyName               string `description:"分类名称"`
+	ClassifyType               string `description:"分类名称下的类型"`
+	//CreateTime                 time.Time `description:"插入时间"`
+	//ModifyTime                 time.Time `description:"修改时间"`
+	//DataTime                   time.Time `description:"数据日期"`
+
+	CreateTime string `description:"插入时间"`
+	ModifyTime string `description:"修改时间"`
+	DataTime   string `description:"数据日期"`
 }
 
 func AddBaseFromTradeDalianIndex(item *BaseFromTradeDalianIndex) (lastId int64, err error) {
@@ -87,3 +91,19 @@ func AddBaseFromTradeDalianData(item *BaseFromTradeDalianData) (lastId int64, er
 	lastId, err = o.Insert(item)
 	return
 }
+
+func GetAddSql(item *BaseFromTradeDalianIndex) (addSql string) {
+	addSql += "('" + item.Rank + "','" + item.DealShortName + "','" + item.DealName + "','" + item.DealCode + "','" + item.DealValue + "','" + item.BuyShortName + "','" + item.DealChange + "','" + item.BuyName + "','" + item.BuyCode + "','" + item.BuyValue + "','" + item.BuyChange + "','" + item.SoldShortName + "','" + item.SoldName + "','" + item.SoldCode + "','" + item.SoldValue + "','" + item.SoldChange + "','" + item.Frequency + "','" + item.ClassifyName + "','" + item.ClassifyType + "','" + item.CreateTime + "','" + item.ModifyTime + "','" + item.DataTime + "'),"
+	return
+}
+
+//刷新大连指标数据
+func RefreshEdbDataByDaLian(addSql string) (err error) {
+	o := orm.NewOrm()
+	o.Using("data")
+	_, err = o.Raw(addSql).Exec()
+	if err != nil {
+		return err
+	}
+	return
+}

models/db.go

@@ -32,7 +32,7 @@ func init() {
 		new(BaseFromTradeIneIndex),
 		new(BaseFromTradeCffexIndex),
 		//
-		//new(BaseFromTradeDalianIndex),
+		new(BaseFromTradeDalianIndex),
 		//new(BaseFromTradeDalianData),
 	)
 }

services/commodity_trade_dalian.go

@@ -5,21 +5,18 @@ import (
 	"fmt"
 	"github.com/PuerkitoBio/goquery"
 	"hongze/hongze_data_crawler/models"
+	"hongze/hongze_data_crawler/utils"
 	"io/ioutil"
+	"log"
 	"mime/multipart"
 	"net/http"
 	"strconv"
-	"time"
-
-	//"golang.org/x/net/html"
-	"hongze/hongze_data_crawler/utils"
-	//"rdluck_tools/http"
-	"log"
 	"strings"
+	"time"
 )
 
-//郑州商品交易所持单排名
-func SyncRankingFromDalian() {
+//大连商品交易所持单排名
+func SyncRankingFromDalianOld() {
 	fmt.Println("start")
 	var str string
 	exitProductMap, exitContractIdMap, varietyArrMap := DoSearch(str)
@@ -126,535 +123,7 @@ func DoSearch(body string) (exitProductMaps, exitContractIdMaps, varietyArrMaps
 	if body != "" {
 		str = body
 	} else {
-		str = `<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
-<html>
- <head> 
-  <meta http-equiv="Content-Type" content="text/html; charset=utf-8" /> 
-  <meta http-equiv="X-UA-Compatible" content="IE=8" /> 
-  <title>大商所-行情/数据-日成交持仓排名</title> 
-  <link type="text/css" href="/publicweb/css/main.css" rel="stylesheet" /> 
-  <link type="text/css" href="/publicweb/css/calendar.css" rel="stylesheet" /> 
-  <script type="text/javascript" src="/publicweb/js/jquery-1.7.2.min.js"></script> 
-  <script type="text/javascript" src="/publicweb/js/calendar.js"></script> 
-  <script type="text/javascript" src="/publicweb/js/staUtil.js"></script> 
-  <script type="text/javascript" src="/publicweb/js/common.js"></script> 
-  <script type="text/javascript">
-        // 品种赋值
-        function setVariety(variety){
-            // 设置品种
-            $("#variety").val(variety);
-            // 设置品种下的合约
-            $("#variety_id").val(variety);
-            $("#contract_id").val("all");
-            $("#memberDealPosiQuotesForm").submit();
-        }
-        // 类型赋值
-        function setTrade_type(trade_type){
-            $("#trade_type").val(trade_type);
-            $("#memberDealPosiQuotesForm").submit();
-        }
-        // 合约赋值
-        function setContract_id(contract_id){
-            $("#contract_id").val(contract_id);
-            $("#memberDealPosiQuotesForm").submit();
-        }
-
-        // 导出查询结果
-        function exportData(type){
-
-            $("#exportFlag").val(type);
-            $("#exportForm").submit();
-        }
-
-        //批量导出查询结果
-        function batchExportData(type){
-
-            // 启用遮罩效果，暂时保留
-            // showBg('dialog','dialog_content');
-
-            $("#batchExportFlag").val(type);
-
-            // 批量下载提示信息
-            if(confirm("您将批量下载当日成交持仓排名数据！")){
-                $("#batchExportForm").submit();
-            }
-        }
-
-    </script> 
- </head> 
- <body> 
-  <form action="/publicweb/quotesdata/memberDealPosiQuotes.html" name="memberDealPosiQuotesForm" id="memberDealPosiQuotesForm" method="post"> 
-   <input type="hidden" id="variety" name="memberDealPosiQuotes.variety" value="a" /> 
-   <input type="hidden" id="trade_type" name="memberDealPosiQuotes.trade_type" value="0" /> 
-   <input type="hidden" name="year" id="year" value="2021" /> 
-   <input type="hidden" name="month" id="month" value="9" /> 
-   <input type="hidden" name="day" id="day" value="19" /> 
-   <input type="hidden" id="contract_id" name="contract.contract_id" value="a2111" /> 
-   <input type="hidden" id="variety_id" name="contract.variety_id" value="a" /> 
-   <input type="hidden" id="currDate" value="20211019" /> 
-   <div class="tradeArea"> 
-    <div class="tradeSel"> 
-     <div id="cal" class="selBox"> 
-     </div> 
-     <!-- 品种选择区域 --> 
-     <div class="selBox"> 
-      <div> 
-       <ul class="keyWord clearfix"> 
-        <li class="keyWord_100"> <input type="radio" name="" value="" onclick="javascript:setVariety('a');" checked="" />豆一 </li> 
-        <li class="keyWord_100"> <input type="radio" name="" value="" onclick="javascript:setVariety('b');" />豆二 </li> 
-        <li class="keyWord_100"> <input type="radio" name="" value="" onclick="javascript:setVariety('m');" />豆粕 </li> 
-        <li class="keyWord_100"> <input type="radio" name="" value="" onclick="javascript:setVariety('y');" />豆油 </li> 
-        <li class="keyWord_100"> <input type="radio" name="" value="" onclick="javascript:setVariety('p');" />棕榈油 </li> 
-        <li class="keyWord_100"> <input type="radio" name="" value="" onclick="javascript:setVariety('c');" />玉米 </li> 
-        <li class="keyWord_100"> <input type="radio" name="" value="" onclick="javascript:setVariety('cs');" />玉米淀粉 </li> 
-        <li class="keyWord_100"> <input type="radio" name="" value="" onclick="javascript:setVariety('jd');" />鸡蛋 </li> 
-        <li class="keyWord_100"> <input type="radio" name="" value="" onclick="javascript:setVariety('rr');" />粳米 </li> 
-        <li class="keyWord_100"> <input type="radio" name="" value="" onclick="javascript:setVariety('fb');" />纤维板 </li> 
-        <li class="keyWord_100"> <input type="radio" name="" value="" onclick="javascript:setVariety('bb');" />胶合板 </li> 
-        <li class="keyWord_100"> <input type="radio" name="" value="" onclick="javascript:setVariety('lh');" />生猪 </li> 
-       </ul> 
-       <ul class="keyWord_x"></ul> 
-       <ul class="keyWord clearfix"> 
-        <li class="keyWord_100"> <input type="radio" name="" value="" onclick="javascript:setVariety('l');" />聚乙烯 </li> 
-        <li class="keyWord_100"> <input type="radio" name="" value="" onclick="javascript:setVariety('v');" />聚氯乙烯 </li> 
-        <li class="keyWord_100"> <input type="radio" name="" value="" onclick="javascript:setVariety('pp');" />聚丙烯 </li> 
-        <li class="keyWord_100"> <input type="radio" name="" value="" onclick="javascript:setVariety('eb');" />苯乙烯 </li> 
-        <li class="keyWord_100"> <input type="radio" name="" value="" onclick="javascript:setVariety('j');" />焦炭 </li> 
-        <li class="keyWord_100"> <input type="radio" name="" value="" onclick="javascript:setVariety('jm');" />焦煤 </li> 
-        <li class="keyWord_100"> <input type="radio" name="" value="" onclick="javascript:setVariety('i');" />铁矿石 </li> 
-        <li class="keyWord_100"> <input type="radio" name="" value="" onclick="javascript:setVariety('eg');" />乙二醇 </li> 
-        <li class="keyWord_100"> <input type="radio" name="" value="" onclick="javascript:setVariety('pg');" />液化石油气 </li> 
-       </ul> 
-      </div> 
-     </div> 
-     <!-- 合约选择区域 --> 
-     <div class="selBox"> 
-      <div> 
-       <!-- modify by wenhongxue  20190104  取消页面"全部"的选择，默认选中第一个合约 --> 
-       <ul class="keyWord clearfix"> 
-        <li class="keyWord_65"> <input type="radio" name="contract" value="" onclick="javascript:setContract_id('a2111');" checked="" />a2111 </li> 
-        <li class="keyWord_65"> <input type="radio" name="contract" value="" onclick="javascript:setContract_id('a2201');" />a2201 </li> 
-        <li class="keyWord_65"> <input type="radio" name="contract" value="" onclick="javascript:setContract_id('a2203');" />a2203 </li> 
-       </ul> 
-      </div> 
-     </div> 
-     <div class="selBox noBorder"> 
-      <div> 
-       <ul class="keyWord02 clearfix"> 
-        <li><a href="javascript:setTrade_type('0');" class="currentKey">期货</a></li> 
-        <li><a href="javascript:setTrade_type('1');javascript:setVariety('m');">期权</a></li> 
-       </ul> 
-      </div> 
-     </div> 
-    </div> 
-    <div class="tradeResult02"> 
-     <p>大连商品交易所 日成交持仓排名</p> 
-     <p><span> 合约代码：a2111 &nbsp;&nbsp;&nbsp;&nbsp;查询日期：20211019</span></p> 
-     <ul class="btns clearfix"> 
-      <li><a href="javascript:print();"><img src="/publicweb/images/common/pic_icon03.gif" alt="打印" />打印</a></li> 
-      <li><a href="javascript:exportData('excel')"><img src="/publicweb/images/common/pic_icon02.gif" alt="导出表格" />导出表格</a></li> 
-      <li><a href="javascript:exportData('txt')"><img src="/publicweb/images/common/pic_icon01.gif" alt="导出文本" />导出文本</a></li> 
-      <li><a href="javascript:batchExportData('batch')"><img src="/publicweb/images/common/pic_icon03.gif" alt="批量下载" />批量下载</a></li> 
-     </ul> 
-     <div class="dataWrapper_member_deal_posi" id="printData"> 
-      <div class="dataArea"> 
-       <table cellpadding="0" cellspacing="0"> 
-        <tbody>
-         <tr> 
-          <th> 会员类别</th> 
-          <th> 总成交量 </th> 
-          <th> 增减</th> 
-          <th> 总持买单量</th> 
-          <th> 增减</th> 
-          <th> 总持卖单量</th> 
-          <th> 增减</th> 
-         </tr> 
-         <!-- 汇总 --> 
-         <tr> 
-          <td align="center">期货公司会员</td> 
-          <td class="td-right">116,526</td> 
-          <td class="td-right">-28,029</td> 
-          <td class="td-right">25,831</td> 
-          <td class="td-right">-3,794</td> 
-          <td class="td-right">27,066</td> 
-          <td class="td-right">-2,978</td> 
-         </tr> 
-        </tbody>
-       </table> 
-       <table cellpadding="0" cellspacing="0"> 
-        <tbody>
-         <tr> 
-          <th> 名次</th> 
-          <th> 会员简称 </th> 
-          <th> 成交量</th> 
-          <th> 增减</th> 
-          <th> 名次</th> 
-          <th> 会员简称</th> 
-          <th> 持买单量</th> 
-          <th> 增减</th> 
-          <th> 名次</th> 
-          <th> 会员简称</th> 
-          <th> 持卖单量</th> 
-          <th> 增减</th> 
-         </tr> 
-         <!-- 列表内容 --> 
-         <tr> 
-          <td>1</td> 
-          <td>华泰期货</td> 
-          <td class="td-right">19,087</td> 
-          <td class="td-right">-5,243</td> 
-          <td>1</td> 
-          <td>国泰君安</td> 
-          <td class="td-right">3,774</td> 
-          <td class="td-right">-762</td> 
-          <td>1</td> 
-          <td>一德期货</td> 
-          <td class="td-right">6,598</td> 
-          <td class="td-right">71</td> 
-         </tr> 
-         <tr> 
-          <td>2</td> 
-          <td>中信期货</td> 
-          <td class="td-right">15,920</td> 
-          <td class="td-right">-3,848</td> 
-          <td>2</td> 
-          <td>东证期货</td> 
-          <td class="td-right">3,590</td> 
-          <td class="td-right">-458</td> 
-          <td>2</td> 
-          <td>招商期货</td> 
-          <td class="td-right">3,055</td> 
-          <td class="td-right">-9</td> 
-         </tr> 
-         <tr> 
-          <td>3</td> 
-          <td>东证期货</td> 
-          <td class="td-right">13,487</td> 
-          <td class="td-right">-6,777</td> 
-          <td>3</td> 
-          <td>招商期货</td> 
-          <td class="td-right">3,339</td> 
-          <td class="td-right">-148</td> 
-          <td>3</td> 
-          <td>申银万国</td> 
-          <td class="td-right">2,403</td> 
-          <td class="td-right">-12</td> 
-         </tr> 
-         <tr> 
-          <td>4</td> 
-          <td>国泰君安</td> 
-          <td class="td-right">12,689</td> 
-          <td class="td-right">1,725</td> 
-          <td>4</td> 
-          <td>华安期货</td> 
-          <td class="td-right">2,250</td> 
-          <td class="td-right">106</td> 
-          <td>4</td> 
-          <td>浙商期货</td> 
-          <td class="td-right">2,257</td> 
-          <td class="td-right">-24</td> 
-         </tr> 
-         <tr> 
-          <td>5</td> 
-          <td>国富期货</td> 
-          <td class="td-right">7,395</td> 
-          <td class="td-right">-2,569</td> 
-          <td>5</td> 
-          <td>一德期货</td> 
-          <td class="td-right">1,840</td> 
-          <td class="td-right">1,092</td> 
-          <td>5</td> 
-          <td>国富期货</td> 
-          <td class="td-right">1,887</td> 
-          <td class="td-right">-276</td> 
-         </tr> 
-         <tr> 
-          <td>6</td> 
-          <td>海通期货</td> 
-          <td class="td-right">6,225</td> 
-          <td class="td-right">-3,426</td> 
-          <td>6</td> 
-          <td>中信期货</td> 
-          <td class="td-right">1,802</td> 
-          <td class="td-right">-1,541</td> 
-          <td>6</td> 
-          <td>中州期货</td> 
-          <td class="td-right">1,716</td> 
-          <td class="td-right">-50</td> 
-         </tr> 
-         <tr> 
-          <td>7</td> 
-          <td>广发期货</td> 
-          <td class="td-right">5,722</td> 
-          <td class="td-right">744</td> 
-          <td>7</td> 
-          <td>申银万国</td> 
-          <td class="td-right">1,018</td> 
-          <td class="td-right">-151</td> 
-          <td>7</td> 
-          <td>海通期货</td> 
-          <td class="td-right">1,151</td> 
-          <td class="td-right">144</td> 
-         </tr> 
-         <tr> 
-          <td>8</td> 
-          <td>兴证期货</td> 
-          <td class="td-right">4,647</td> 
-          <td class="td-right">-3,405</td> 
-          <td>8</td> 
-          <td>海通期货</td> 
-          <td class="td-right">1,014</td> 
-          <td class="td-right">-119</td> 
-          <td>8</td> 
-          <td>中信期货</td> 
-          <td class="td-right">1,015</td> 
-          <td class="td-right">-363</td> 
-         </tr> 
-         <tr> 
-          <td>9</td> 
-          <td>光大期货</td> 
-          <td class="td-right">3,714</td> 
-          <td class="td-right">-804</td> 
-          <td>9</td> 
-          <td>永安期货</td> 
-          <td class="td-right">727</td> 
-          <td class="td-right">82</td> 
-          <td>9</td> 
-          <td>银河期货</td> 
-          <td class="td-right">979</td> 
-          <td class="td-right">133</td> 
-         </tr> 
-         <tr> 
-          <td>10</td> 
-          <td>华闻期货</td> 
-          <td class="td-right">3,404</td> 
-          <td class="td-right">503</td> 
-          <td>10</td> 
-          <td>光大期货</td> 
-          <td class="td-right">695</td> 
-          <td class="td-right">169</td> 
-          <td>10</td> 
-          <td>宏源期货</td> 
-          <td class="td-right">785</td> 
-          <td class="td-right">-323</td> 
-         </tr> 
-         <tr> 
-          <td>11</td> 
-          <td>东方财富</td> 
-          <td class="td-right">2,988</td> 
-          <td class="td-right">698</td> 
-          <td>11</td> 
-          <td>华泰期货</td> 
-          <td class="td-right">691</td> 
-          <td class="td-right">-550</td> 
-          <td>11</td> 
-          <td>新湖期货</td> 
-          <td class="td-right">715</td> 
-          <td class="td-right">-382</td> 
-         </tr> 
-         <tr> 
-          <td>12</td> 
-          <td>一德期货</td> 
-          <td class="td-right">2,823</td> 
-          <td class="td-right">-1,270</td> 
-          <td>12</td> 
-          <td>金元期货</td> 
-          <td class="td-right">688</td> 
-          <td class="td-right">-278</td> 
-          <td>12</td> 
-          <td>格林大华</td> 
-          <td class="td-right">694</td> 
-          <td class="td-right">-160</td> 
-         </tr> 
-         <tr> 
-          <td>13</td> 
-          <td>申银万国</td> 
-          <td class="td-right">2,809</td> 
-          <td class="td-right">-966</td> 
-          <td>13</td> 
-          <td>银河期货</td> 
-          <td class="td-right">667</td> 
-          <td class="td-right">-206</td> 
-          <td>13</td> 
-          <td>东证期货</td> 
-          <td class="td-right">640</td> 
-          <td class="td-right">-457</td> 
-         </tr> 
-         <tr> 
-          <td>14</td> 
-          <td>中原期货</td> 
-          <td class="td-right">2,515</td> 
-          <td class="td-right">232</td> 
-          <td>14</td> 
-          <td>东航期货</td> 
-          <td class="td-right">663</td> 
-          <td class="td-right">-31</td> 
-          <td>14</td> 
-          <td>国投安信</td> 
-          <td class="td-right">564</td> 
-          <td class="td-right">12</td> 
-         </tr> 
-         <tr> 
-          <td>15</td> 
-          <td>徽商期货</td> 
-          <td class="td-right">2,503</td> 
-          <td class="td-right">960</td> 
-          <td>15</td> 
-          <td>瑞达期货</td> 
-          <td class="td-right">589</td> 
-          <td class="td-right">-509</td> 
-          <td>15</td> 
-          <td>鲁证期货</td> 
-          <td class="td-right">531</td> 
-          <td class="td-right">-168</td> 
-         </tr> 
-         <tr> 
-          <td>16</td> 
-          <td>南华期货</td> 
-          <td class="td-right">2,502</td> 
-          <td class="td-right">-147</td> 
-          <td>16</td> 
-          <td>兴证期货</td> 
-          <td class="td-right">583</td> 
-          <td class="td-right">-650</td> 
-          <td>16</td> 
-          <td>瑞达期货</td> 
-          <td class="td-right">527</td> 
-          <td class="td-right">-487</td> 
-         </tr> 
-         <tr> 
-          <td>17</td> 
-          <td>鲁证期货</td> 
-          <td class="td-right">2,275</td> 
-          <td class="td-right">-2,825</td> 
-          <td>17</td> 
-          <td>鲁证期货</td> 
-          <td class="td-right">563</td> 
-          <td class="td-right">-11</td> 
-          <td>17</td> 
-          <td>华泰期货</td> 
-          <td class="td-right">425</td> 
-          <td class="td-right">3</td> 
-         </tr> 
-         <tr> 
-          <td>18</td> 
-          <td>银河期货</td> 
-          <td class="td-right">2,213</td> 
-          <td class="td-right">-1,701</td> 
-          <td>18</td> 
-          <td>中原期货</td> 
-          <td class="td-right">519</td> 
-          <td class="td-right">122</td> 
-          <td>18</td> 
-          <td>华闻期货</td> 
-          <td class="td-right">398</td> 
-          <td class="td-right">-114</td> 
-         </tr> 
-         <tr> 
-          <td>19</td> 
-          <td>国信期货</td> 
-          <td class="td-right">1,805</td> 
-          <td class="td-right">472</td> 
-          <td>19</td> 
-          <td>东吴期货</td> 
-          <td class="td-right">410</td> 
-          <td class="td-right">-7</td> 
-          <td>19</td> 
-          <td>中天期货</td> 
-          <td class="td-right">363</td> 
-          <td class="td-right">0</td> 
-         </tr> 
-         <tr> 
-          <td>20</td> 
-          <td>招金期货</td> 
-          <td class="td-right">1,803</td> 
-          <td class="td-right">-382</td> 
-          <td>20</td> 
-          <td>东方财富</td> 
-          <td class="td-right">409</td> 
-          <td class="td-right">56</td> 
-          <td>20</td> 
-          <td>南华期货</td> 
-          <td class="td-right">363</td> 
-          <td class="td-right">-516</td> 
-         </tr> 
-         <tr> 
-          <td>总计</td> 
-          <td>&nbsp;</td> 
-          <td class="td-right">116,526</td> 
-          <td class="td-right">-28,029</td> 
-          <td>总计</td> 
-          <td>&nbsp;</td> 
-          <td class="td-right">25,831</td> 
-          <td class="td-right">-3,794</td> 
-          <td>总计</td> 
-          <td>&nbsp;</td> 
-          <td class="td-right">27,066</td> 
-          <td class="td-right">-2,978</td> 
-         </tr> 
-        </tbody>
-       </table> 
-      </div> 
-     </div> 
-     <ul class="infoTip"> 
-      <li style="font-size:14px;">说明： </li> 
-      <li style="font-size:14px;">(1) 成交量、持买单量、持卖单量：手 </li> 
-      <!-- <li style="font-size:14px;">(2) 根据相关规则，期货品种信息公布所有合约，期权品种信息公布活跃月份（标的期货合约双边持仓量大于、等于2万手）合约系列 。</li> --> 
-      <!-- modified by wenhongxue 20190605 [双边改单边项目] start --> 
-      <li style="font-size:14px;">(2) 根据相关规则，期货品种信息公布活跃月份（单边持仓量大于、等于1万手）合约，期权品种信息公布活跃月份（标的期货合约单边持仓量大于、等于1万手）合约系列 。</li> 
-      <!-- modified by wenhongxue 20190605 [双边改单边项目] end --> 
-     </ul> 
-    </div> 
-   </div> 
-  </form> 
-  <form action="/publicweb/quotesdata/exportMemberDealPosiQuotesData.html" name="exportForm" id="exportForm" method="post"> 
-   <input type="hidden" id="variety" name="memberDealPosiQuotes.variety" value="a" /> 
-   <input type="hidden" id="trade_type" name="memberDealPosiQuotes.trade_type" value="0" /> 
-   <input type="hidden" id="contract_id" name="contract.contract_id" value="a2111" /> 
-   <input type="hidden" id="variety_id" name="contract.variety_id" value="a" /> 
-   <input type="hidden" name="year" id="year" value="2021" /> 
-   <input type="hidden" name="month" id="month" value="9" /> 
-   <input type="hidden" name="day" id="day" value="19" /> 
-   <input type="hidden" name="exportFlag" id="exportFlag" /> 
-  </form> 
-  <form action="/publicweb/quotesdata/exportMemberDealPosiQuotesBatchData.html" name="batchExportForm" id="batchExportForm" method="post"> 
-   <input type="hidden" id="variety" name="memberDealPosiQuotes.variety" value="a" /> 
-   <input type="hidden" id="trade_type" name="memberDealPosiQuotes.trade_type" value="0" /> 
-   <input type="hidden" id="contract_id" name="contract.contract_id" value="a2111" /> 
-   <input type="hidden" id="variety_id" name="contract.variety_id" value="a" /> 
-   <input type="hidden" name="year" id="year" value="2021" /> 
-   <input type="hidden" name="month" id="month" value="9" /> 
-   <input type="hidden" name="day" id="day" value="19" /> 
-   <input type="hidden" name="batchExportFlag" id="batchExportFlag" /> 
-  </form> 
-  <input id="eprotalCurrentSiteId" value="459264" type="hidden" /> 
-  <input id="eprotalCurrentPageId" value="rcjccpm" type="hidden" /> 
-  <script type="text/javascript">
-    var currYear_='2021';
-    var nowDate = $('#currDate').val();
-    var nyear=$('#year').val();
-    var nmonth=$('#month').val();
-    var nday=$('#day').val();
-    if(nowDate){
-        nyear = nowDate.substr(0,4);
-        nmonth = nowDate.substr(4,6);
-        if(nmonth.substring(0,1)=='0'){
-            nmonth=parseInt(nmonth.substring(1,2))-1+'';
-        }else{
-            nmonth=parseInt(nmonth.substring(0,2))-1+'';
-        }
-        nday = nowDate.substr(6,8);
-    }
-
-    //参数分别是渲染日历的div的id,查询时间的年份,月份,日期,form表单id
-    setCurrYear(currYear_);
-    showCalendar(document.getElementById("cal"),nyear,nmonth,nday,'memberDealPosiQuotesForm','ch');
-</script>  
-  <script type="text/javascript">
-    dataStatistic();
-</script>  
- </body>
-</html>`
+		str = "HTML文本"
 	}
 	doc, err := goquery.NewDocumentFromReader(strings.NewReader(str))
 	if err != nil {
@@ -719,11 +188,19 @@ func DoSearch(body string) (exitProductMaps, exitContractIdMaps, varietyArrMaps
 
 //处理解析Html
 func DoHtml(body, name, contractId string, dateTime time.Time) (err error) {
+	defer func() {
+		if err != nil {
+			fmt.Println("RefreshDataFromDaLian Err:" + err.Error())
+			go utils.SendEmail(utils.APPNAME+"【"+utils.RunMode+"】"+"失败提醒", "RefreshDataFromDaLian ErrMsg:"+err.Error(), utils.EmailSendToUsers)
+		}
+	}()
 	str := body
 	doc, err := goquery.NewDocumentFromReader(strings.NewReader(str))
 	if err != nil {
 		log.Fatal(err)
 	}
+	var isAdd bool
+	addSql := `  INSERT INTO base_from_trade_dalian_index(rank,deal_short_name,deal_name,deal_code,deal_value,buy_short_name,deal_change,buy_name,buy_code,buy_value,buy_change,sold_short_name,sold_name,sold_code,sold_value,sold_change,frequency,classify_name,classify_type,create_time,modify_time,data_time) values  `
 	table := doc.Find("table")
 	var rank, shortName, dealValue, dealChange, buyName, buyValue, buyChange, soldName, soldValue, soldChange string
 	table.Find("tr").Each(func(i int, tr *goquery.Selection) {
@@ -768,37 +245,39 @@ func DoHtml(body, name, contractId string, dateTime time.Time) (err error) {
 					soldChange = strings.Replace(tdText, ",", "", -1)
 				}
 			})
-
 			item.Rank = rank
 			item.DealShortName = shortName
 			item.DealName = shortName + "_成交量_" + contractId
-			item.DealCode = dlIndexCodeGenerator(item.DealName)
+			item.DealCode = DlIndexCodeGenerator(item.DealName, "deal")
 			item.DealValue = dealValue
 			item.BuyShortName = buyName
 			item.DealChange = dealChange
 			item.BuyName = buyName + "_持买单量_" + contractId
-			item.BuyCode = dlIndexCodeGenerator(item.BuyName)
+			item.BuyCode = DlIndexCodeGenerator(item.BuyName, "buy")
 			item.BuyValue = buyValue
 			item.BuyChange = buyChange
 			item.SoldShortName = soldName
-			item.SoldName = soldName + "_持买单量_" + contractId
-			item.SoldCode = dlIndexCodeGenerator(item.SoldName)
+			item.SoldName = soldName + "_持卖单量_" + contractId
+			item.SoldCode = DlIndexCodeGenerator(item.SoldName, "sold")
 			item.SoldValue = soldValue
 			item.SoldChange = soldChange
 			item.Frequency = "日度"
 			item.ClassifyName = name
 			item.ClassifyType = contractId
-			item.CreateTime = time.Now()
-			item.ModifyTime = time.Now()
-			item.DataTime = dateTime
-
-			_, err := models.AddBaseFromTradeDalianIndex(item)
-			if err != nil {
-				fmt.Println(err)
-				return
-			}
+			item.CreateTime = time.Now().Format(utils.FormatDateTime)
+			item.ModifyTime = time.Now().Format(utils.FormatDateTime)
+			item.DataTime = dateTime.Format(utils.FormatDate)
+			addSql += models.GetAddSql(item)
+			isAdd = true
 		}
 	})
+	addSql = strings.TrimRight(addSql, ",")
+	if isAdd {
+		err = models.RefreshEdbDataByDaLian(addSql)
+		if err != nil {
+			return err
+		}
+	}
 	return
 }
 
@@ -814,15 +293,36 @@ type SearchContractId struct {
 
 func SyncRankingFromDalianDo() {
 
-	for i := 179; i >= 0; i-- {
-		//fmt.Println(i)
-		SyncRankingFromDalian2(i)
+	for i := 10; i >= 0; i-- {
+		fmt.Println(i)
+		//SyncRankingFromDalian2(i)
 	}
 }
 
-//郑州商品交易所持单排名
-func SyncRankingFromDalian2(dayNum int) {
+func SyncRankingFromDalianDos() (err error) {
+	fmt.Println(66666)
+	defer func() {
+		if err != nil {
+			fmt.Println("RefreshDataFromWind Err:" + err.Error())
+			go utils.SendEmail(utils.APPNAME+"【"+utils.RunMode+"】"+"失败提醒", "RefreshDataFromWind ErrMsg:"+err.Error(), utils.EmailSendToUsers)
+		}
+	}()
+
+	//SyncRankingFromDalian2(1)
+	return err
+}
+
+//大连交易所持单排名
+func SyncRankingFromDalian(dayNum int) (err error) {
 	fmt.Println("start")
+	defer func() {
+		if err != nil {
+			fmt.Println("RefreshDataFromDalian Err:" + err.Error())
+			go utils.SendEmail(utils.APPNAME+"【"+utils.RunMode+"】"+"失败提醒", "RefreshDataFromDalian ErrMsg:"+err.Error(), utils.EmailSendToUsers)
+		}
+	}()
+	n := utils.GetRandInt(10, 120)
+	time.Sleep(time.Duration(n) * time.Second)
 	searchList := []SearchList{
 		SearchList{VarietyName: "豆一", CarietyCode: "a", List: []SearchContractId{SearchContractId{ContractId: "a2111"}, {ContractId: "a2201"}, {ContractId: "a2203"}}},
 		SearchList{VarietyName: "豆二", CarietyCode: "b", List: []SearchContractId{SearchContractId{ContractId: "b2112"}, {ContractId: "b2201"}}},
@@ -861,7 +361,6 @@ func SyncRankingFromDalian2(dayNum int) {
 		dayStr = strconv.Itoa(day)
 	}
 	monthNum, _ := strconv.Atoi(month)
-	//fmt.Println(currDate)
 	month = strconv.Itoa(monthNum - 1) //获取时月份需要减一
 	for _, v := range searchList {
 		for _, v2 := range v.List {
@@ -882,45 +381,46 @@ func SyncRankingFromDalian2(dayNum int) {
 			fmt.Println(currDate, v.VarietyName, v2.ContractId)
 			if err != nil {
 				utils.FileLog.Info("获取指标失败：" + currDate + v.VarietyName + v2.ContractId)
-				fmt.Println(err)
-				return
+				return err
 			}
 			client := &http.Client{}
 			req, err := http.NewRequest(method, url, payload)
 			if err != nil {
-				fmt.Println(err)
-				return
+				return err
 			}
+
 			req.Header.Add("Cookie", "JSESSIONID=36ACF02A59227A3854F9D5D5E2FB5F2E; WMONID=R5ojcAIIcx-")
 			req.Header.Set("Content-Type", writer.FormDataContentType())
 			res, err := client.Do(req)
 			if err != nil {
-				fmt.Println(err)
-				return
+				return err
 			}
 			defer res.Body.Close()
 			body, err := ioutil.ReadAll(res.Body)
 			if err != nil {
-				fmt.Println(err)
-				return
+				go utils.SendEmail(utils.APPNAME+"【"+utils.RunMode+"】"+"失败提醒", "RefreshDataFromDalian ErrMsg:"+err.Error()+"获取指标失败："+currDate+v.VarietyName+v2.ContractId, utils.EmailSendToUsers)
+				return err
 			}
 			err = DoHtml(string(body), v.VarietyName, v2.ContractId, timeDate)
+			if err != nil {
+				return err
+			}
 		}
 	}
+	return err
 }
 
-//var indexCode string
-//var indexCodeMap = make(map[string]string)
-
-func dlIndexCodeGenerator(indexName string) string {
-	indexCode, _ := indexCodeMap[indexName]
-	if indexCode == "" {
-		indexCode = fmt.Sprintf("DL%s", time.Now().Format(utils.FormatDateTimeUnSpace)+strconv.Itoa(utils.GetRandInt(1, 100)))
-		indexCodeMap[indexName] = indexCode
-		err := models.AddBaseFromTradeMapping(indexName, indexCode, "DL")
+func DlIndexCodeGenerator(indexName, suffix string) string {
+	ineIndexCode, _ := ineIndexCodeMap[indexName]
+	if ineIndexCode == "" {
+		ineIndexCode = fmt.Sprintf("DL%s", time.Now().Format(utils.FormatDateTimeUnSpace)+strconv.Itoa(utils.GetRandInt(1, 100))+suffix)
+		ineIndexCodeMap[indexName] = ineIndexCode
+		err := models.AddBaseFromTradeMapping(indexName, ineIndexCode, "DL")
 		if err != nil {
 			fmt.Println("add Code err:", err)
 		}
 	}
-	return indexCode
+	return ineIndexCode
 }
+
+//rank,deal_short_name,deal_name,deal_code,deal_value,buy_short_name,deal_change,buy_name,buy_code,buy_value,buy_change,sold_short_name,sold_name,sold_code,sold_value,sold_change,frequency,classify_name,classify_type,create_time,modify_time,data_time

services/task.go

@@ -1,7 +1,9 @@
 package services
 
 import (
+	"context"
 	"fmt"
+	"github.com/beego/beego/v2/task"
 )
 
 func Task() {
@@ -10,11 +12,28 @@ func Task() {
 	//SyncRankingFromZhengzhou()
 
 	//SyncRankingFromShangHai()
-	SyncRankingFromIne()
+	//SyncRankingFromIne()
 	//SyncRankingFromCffex()
 
-	SyncRankingFromDalianDo()
+	//time.Sleep(20 * time.Second)
+	//SyncRankingFromDalianDo()
 	//SyncStorageFromEic()
 	//SyncStorageFromEic()
+
+	refreshData := task.NewTask("refreshData", "0 0 3 * * *", RefreshData)
+	task.AddTask("数据爬取", refreshData)
+	task.StartTask()
 	fmt.Println("end crawler")
 }
+
+func RefreshData(cont context.Context) (err error) {
+	//大连
+	SyncRankingFromDalian(1)
+
+	//task2()
+
+	//...
+	//task6()
+
+	return
+}