爬取搜索条件

services/commodity_trade_dalian.go

@@ -15,14 +15,123 @@ import (
 	"time"
 )
 
-//处理搜索条件
-func DoSearch(body string) (exitProductMaps, exitContractIdMaps, varietyArrMaps map[int]string) {
-	var str string
-	if body != "" {
-		str = body
+type SearchList struct {
+	VarietyName string `description:"商品名称"`
+	CarietyCode string `description:"商品名称对应的编码"`
+	List        []SearchContractId
+	ListSearch  []*SearchContractId
+}
+
+type SearchContractId struct {
+	ContractId string `description:"商品类型"`
+}
+
+//同步 N天 之内的数据
+func SyncRankingFromDalianDo() {
+	for i := 200; i >= 0; i-- {
+		SyncRankingFromDalianSearch(i)
+	}
+}
+
+//大连交易所持单排名
+func SyncRankingFromDalianSearch(dayNum int) (err error) {
+	fmt.Println("start")
+	defer func() {
+		if err != nil {
+			fmt.Println("RefreshDataFromDalian Err:" + err.Error())
+			go utils.SendEmail(utils.APPNAME+"【"+utils.RunMode+"】"+"失败提醒", "RefreshDataFromDalian ErrMsg:"+err.Error(), utils.EmailSendToUsers)
+		}
+	}()
+	//定义爬取时间
+	endDate := time.Now().AddDate(0, 0, -dayNum).Format(utils.FormatDateTime)
+	//endDate := time.Now().Format(utils.FormatDateTime)
+	timeDate := utils.StrTimeToTime(endDate)
+	currDate := timeDate.Format(utils.FormatDateUnSpace)
+	year := timeDate.Year()
+	month := timeDate.Format("01")
+	var dayStr string
+	day := timeDate.Day()
+	if day < 10 {
+		dayStr = "0" + strconv.Itoa(day)
 	} else {
-		str = "HTML文本"
+		dayStr = strconv.Itoa(day)
+	}
+	monthNum, _ := strconv.Atoi(month)
+	month = strconv.Itoa(monthNum - 1) //获取时月份需要减一
+	list, err := models.GetBaseFromTradeDalianDataList(timeDate.Format(utils.FormatDate))
+	listDataMap := make(map[string]int)
+	for _, v := range list {
+		listDataMap[v.DealShortName+v.ClassifyType+v.DataTime] = v.BaseFromTradeDalianIndexId
+	}
+	if err != nil {
+		fmt.Println(err)
+		return err
+	}
+	var ContractId string
+	var CarietyCode string
+	var VarietyName string
+	//模拟form表单请求
+	url := "http://www.dce.com.cn/publicweb/quotesdata/memberDealPosiQuotes.html"
+	method := "POST"
+	payload := &bytes.Buffer{}
+	writer := multipart.NewWriter(payload)
+	_ = writer.WriteField("memberDealPosiQuotes.variety", CarietyCode)
+	_ = writer.WriteField("memberDealPosiQuotes.trade_type", "0")
+	_ = writer.WriteField("year", strconv.Itoa(year))
+	_ = writer.WriteField("month", month)
+	_ = writer.WriteField("day", dayStr)
+	_ = writer.WriteField("contract.contract_id", ContractId)
+	_ = writer.WriteField("contract.variety_id", CarietyCode)
+	_ = writer.WriteField("currDate", currDate)
+	err = writer.Close()
+	if err != nil {
+		utils.FileLog.Info("获取指标失败：" + currDate + VarietyName + ContractId)
+		return err
+	}
+	client := &http.Client{}
+	req, err := http.NewRequest(method, url, payload)
+	if err != nil {
+		return err
 	}
+	req.Header.Add("Cookie", "JSESSIONID=36ACF02A59227A3854F9D5D5E2FB5F2E; WMONID=R5ojcAIIcx-")
+	req.Header.Set("Content-Type", writer.FormDataContentType())
+	res, err := client.Do(req)
+	if err != nil {
+		return err
+	}
+	defer res.Body.Close()
+	body, err := ioutil.ReadAll(res.Body)
+	if err != nil {
+		go utils.SendEmail(utils.APPNAME+"【"+utils.RunMode+"】"+"失败提醒", "RefreshDataFromDalian ErrMsg:"+err.Error()+"获取指标失败："+currDate+VarietyName+ContractId, utils.EmailSendToUsers)
+		return err
+	}
+	exitProductMaps, _, varietyArrMaps := DoSearch(string(body))
+	var items []*SearchList
+	for k, v := range exitProductMaps {
+		item := new(SearchList)
+		item.VarietyName = v
+		item.CarietyCode = varietyArrMaps[k]
+		htmlBody, err := GetDalianHtmlBody(dayNum, "", item.CarietyCode, item.VarietyName)
+		if err != nil {
+			return err
+		}
+		listContractId := DoSearchMap(htmlBody)
+		item.ListSearch = listContractId
+		items = append(items, item)
+	}
+	for k, v := range items {
+		for _, v2 := range v.ListSearch {
+			fmt.Println(currDate, v.VarietyName, v2.ContractId, v.CarietyCode, k)
+		}
+	}
+	SyncRankingFromDalian(dayNum, items)
+	return err
+}
+
+//处理搜索条件初始
+func DoSearch(body string) (exitProductMaps, exitContractIdMaps, varietyArrMaps map[int]string) {
+	var str string
+	str = body
 	doc, err := goquery.NewDocumentFromReader(strings.NewReader(str))
 	if err != nil {
 		log.Fatal(err)
@@ -39,22 +148,26 @@ func DoSearch(body string) (exitProductMaps, exitContractIdMaps, varietyArrMaps
 		//解析标签
 		//fmt.Println(i, s.Text())
 		ulTxt := s.Text()
+		//fmt.Println(ulTxt)
 		if ulTxt != "" && (i == 0 || i == 2) {
-			ulTxtArr := strings.Split(ulTxt, " ")
+			ulTxtArr := strings.Split(ulTxt, "\n")
 			for _, v := range ulTxtArr {
-				v = strings.Replace(v, "\n", "", -1)
 				v = strings.Replace(v, " ", "", -1)
-				if v != "" {
+				v = strings.Replace(v, "\n", "", -1)
+				v = strings.Replace(v, "	", "", -1)
+				if v != "" && len(v) > 0 {
 					exitProductMap[pNum] = v
 					pNum++
 				}
 			}
 		}
 		if ulTxt != "" && i == 3 {
+			//fmt.Println(ulTxt)
 			cidTxtArr := strings.Split(ulTxt, " ")
 			for _, v := range cidTxtArr {
 				v = strings.Replace(v, "\n", "", -1)
 				v = strings.Replace(v, " ", "", -1)
+				v = strings.Replace(v, "	", "", -1)
 				if v != "" {
 					exitContractIdMap[cidNum] = v
 					cidNum++
@@ -63,7 +176,6 @@ func DoSearch(body string) (exitProductMaps, exitContractIdMaps, varietyArrMaps
 		}
 	})
 	varietyArr := strings.Split(str, "onclick=\"javascript:setVariety('")
-
 	for _, v := range varietyArr {
 		strnum := strings.Index(v, "');")
 		if strnum > 0 {
@@ -78,12 +190,44 @@ func DoSearch(body string) (exitProductMaps, exitContractIdMaps, varietyArrMaps
 	exitProductMaps = exitProductMap
 	exitContractIdMaps = exitContractIdMap
 	varietyArrMaps = varietyArrMap
-	fmt.Println(exitProductMaps)
-	fmt.Println(exitContractIdMaps)
-	fmt.Println(varietyArrMaps)
 	return
 }
 
+//处理搜索条件
+func DoSearchMap(body string) (items []*SearchContractId) {
+	var str string
+	str = body
+	doc, err := goquery.NewDocumentFromReader(strings.NewReader(str))
+	if err != nil {
+		log.Fatal(err)
+	}
+	exitContractIdMap := make(map[int]string)
+	ul := doc.Find(".selBox ul")
+	var cidNum int
+	ul.Each(func(i int, s *goquery.Selection) {
+		//解析标签
+		ulTxt := s.Text()
+		if ulTxt != "" && i == 3 {
+			cidTxtArr := strings.Split(ulTxt, " ")
+			for _, v := range cidTxtArr {
+				v = strings.Replace(v, "\n", "", -1)
+				v = strings.Replace(v, " ", "", -1)
+				v = strings.Replace(v, "	", "", -1)
+				if v != "" {
+					exitContractIdMap[cidNum] = v
+					cidNum++
+				}
+			}
+		}
+	})
+	for _, v := range exitContractIdMap {
+		item := new(SearchContractId)
+		item.ContractId = v
+		items = append(items, item)
+	}
+	return items
+}
+
 //处理解析Html
 func DoHtml(body, name, contractId string, dateTime time.Time, listDataMap map[string]int) (err error) {
 	defer func() {
@@ -187,26 +331,95 @@ func DoHtml(body, name, contractId string, dateTime time.Time, listDataMap map[s
 	return
 }
 
-type SearchList struct {
-	VarietyName string `description:"商品名称"`
-	CarietyCode string `description:"商品名称对应的编码"`
-	List        []SearchContractId
+func DlIndexCodeGenerator(indexName, suffix string) string {
+	ineIndexCode, _ := ineIndexCodeMap[indexName]
+	if ineIndexCode == "" {
+		ineIndexCode = fmt.Sprintf("DL%s", time.Now().Format(utils.FormatDateTimeUnSpace)+strconv.Itoa(utils.GetRandInt(1, 100))+suffix)
+		ineIndexCodeMap[indexName] = ineIndexCode
+		err := models.AddBaseFromTradeMapping(indexName, ineIndexCode, "DL")
+		if err != nil {
+			fmt.Println("add Code err:", err)
+		}
+	}
+	return ineIndexCode
 }
 
-type SearchContractId struct {
-	ContractId string `description:"商品类型"`
-}
+//大连交易所持单排名
+func GetDalianHtmlBody(dayNum int, contractId, carietyCode, varietyName string) (body string, err error) {
+	defer func() {
+		if err != nil {
+			fmt.Println("GetDalianHtmlBody Err:" + err.Error())
+			go utils.SendEmail(utils.APPNAME+"【"+utils.RunMode+"】"+"失败提醒", "GetDalianHtmlBody ErrMsg:"+err.Error(), utils.EmailSendToUsers)
+		}
+	}()
 
-func SyncRankingFromDalianDo() {
+	//定义爬取时间
+	endDate := time.Now().AddDate(0, 0, -dayNum).Format(utils.FormatDateTime)
+	//endDate := time.Now().Format(utils.FormatDateTime)
+	timeDate := utils.StrTimeToTime(endDate)
+	currDate := timeDate.Format(utils.FormatDateUnSpace)
+	year := timeDate.Year()
+	month := timeDate.Format("01")
+	var dayStr string
+	day := timeDate.Day()
+	if day < 10 {
+		dayStr = "0" + strconv.Itoa(day)
+	} else {
+		dayStr = strconv.Itoa(day)
+	}
+	monthNum, _ := strconv.Atoi(month)
+	month = strconv.Itoa(monthNum - 1) //获取时月份需要减一
 
-	for i := 10; i >= 0; i-- {
-		fmt.Println(i)
-		//SyncRankingFromDalian2(i)
+	list, err := models.GetBaseFromTradeDalianDataList(timeDate.Format(utils.FormatDate))
+	listDataMap := make(map[string]int)
+	for _, v := range list {
+		listDataMap[v.DealShortName+v.ClassifyType+v.DataTime] = v.BaseFromTradeDalianIndexId
+	}
+	if err != nil {
+		fmt.Println(err)
+		return
+	}
+	//模拟form表单请求
+	url := "http://www.dce.com.cn/publicweb/quotesdata/memberDealPosiQuotes.html"
+	method := "POST"
+	payload := &bytes.Buffer{}
+	writer := multipart.NewWriter(payload)
+	_ = writer.WriteField("memberDealPosiQuotes.variety", carietyCode)
+	_ = writer.WriteField("memberDealPosiQuotes.trade_type", "0")
+	_ = writer.WriteField("year", strconv.Itoa(year))
+	_ = writer.WriteField("month", month)
+	_ = writer.WriteField("day", dayStr)
+	_ = writer.WriteField("contract.contract_id", contractId)
+	_ = writer.WriteField("contract.variety_id", carietyCode)
+	_ = writer.WriteField("currDate", currDate)
+	err = writer.Close()
+	if err != nil {
+		utils.FileLog.Info("获取指标失败：" + currDate + varietyName + contractId)
+		return
 	}
+	client := &http.Client{}
+	req, err := http.NewRequest(method, url, payload)
+	if err != nil {
+		return
+	}
+	req.Header.Add("Cookie", "JSESSIONID=36ACF02A59227A3854F9D5D5E2FB5F2E; WMONID=R5ojcAIIcx-")
+	req.Header.Set("Content-Type", writer.FormDataContentType())
+	res, err := client.Do(req)
+	if err != nil {
+		return
+	}
+	defer res.Body.Close()
+	htmlBody, err := ioutil.ReadAll(res.Body)
+	if err != nil {
+		go utils.SendEmail(utils.APPNAME+"【"+utils.RunMode+"】"+"失败提醒", "GetDalianHtmlBody ErrMsg:"+err.Error()+"获取指标失败："+currDate+varietyName+contractId, utils.EmailSendToUsers)
+		return
+	}
+	body = string(htmlBody)
+	return
 }
 
 //大连交易所持单排名
-func SyncRankingFromDalian(dayNum int) (err error) {
+func SyncRankingFromDalian(dayNum int, searchList []*SearchList) (err error) {
 	fmt.Println("start")
 	defer func() {
 		if err != nil {
@@ -214,31 +427,31 @@ func SyncRankingFromDalian(dayNum int) (err error) {
 			go utils.SendEmail(utils.APPNAME+"【"+utils.RunMode+"】"+"失败提醒", "RefreshDataFromDalian ErrMsg:"+err.Error(), utils.EmailSendToUsers)
 		}
 	}()
-	n := utils.GetRandInt(10, 120)
-	time.Sleep(time.Duration(n) * time.Second)
-	searchList := []SearchList{
-		SearchList{VarietyName: "豆一", CarietyCode: "a", List: []SearchContractId{SearchContractId{ContractId: "a2111"}, {ContractId: "a2201"}, {ContractId: "a2203"}}},
-		SearchList{VarietyName: "豆二", CarietyCode: "b", List: []SearchContractId{SearchContractId{ContractId: "b2112"}, {ContractId: "b2201"}}},
-		SearchList{VarietyName: "豆粕", CarietyCode: "m", List: []SearchContractId{SearchContractId{ContractId: "m2111"}, {ContractId: "m2112"}, {ContractId: "m2201"}, {ContractId: "m2203"}, {ContractId: "m2205"}, {ContractId: "m2207"}, {ContractId: "m2208"}, {ContractId: "m2209"}}},
-		SearchList{VarietyName: "豆油", CarietyCode: "y", List: []SearchContractId{SearchContractId{ContractId: "y2112"}, {ContractId: "y2201"}, {ContractId: "y2203"}, {ContractId: "y2205"}, {ContractId: "y2207"}, {ContractId: "y2208"}}},
-		SearchList{VarietyName: "棕榈油", CarietyCode: "p", List: []SearchContractId{SearchContractId{ContractId: "p2112"}, {ContractId: "p2201"}, {ContractId: "p2202"}, {ContractId: "p2203"}, {ContractId: "p2204"}, {ContractId: "p2205"}}},
-		SearchList{VarietyName: "玉米", CarietyCode: "c", List: []SearchContractId{SearchContractId{ContractId: "c2111"}, {ContractId: "c2201"}, {ContractId: "c2203"}, {ContractId: "c2205"}, {ContractId: "c2207"}}},
-		SearchList{VarietyName: "玉米淀粉", CarietyCode: "cs", List: []SearchContractId{SearchContractId{ContractId: "cs2111"}, {ContractId: "cs2201"}, {ContractId: "cs2203"}}},
-		SearchList{VarietyName: "鸡蛋", CarietyCode: "jd", List: []SearchContractId{SearchContractId{ContractId: "jd2201"}, {ContractId: "jd2202"}, {ContractId: "jd2203"}, {ContractId: "jd2204"}, {ContractId: "jd2205"}}},
-		SearchList{VarietyName: "粳米", CarietyCode: "rr", List: []SearchContractId{SearchContractId{ContractId: "rr2112"}}},
-		//SearchList{VarietyName: "纤维板", CarietyCode: "fb", List:[]SearchContractId{}}, //无数据
-		//SearchList{VarietyName: "胶合板", CarietyCode: "bb", List:[]SearchContractId{}}, //无数据
-		SearchList{VarietyName: "生猪", CarietyCode: "lh", List: []SearchContractId{SearchContractId{ContractId: "lh2201"}, {ContractId: "lh2203"}, {ContractId: "lh2205"}}},
-		SearchList{VarietyName: "聚乙烯", CarietyCode: "l", List: []SearchContractId{SearchContractId{ContractId: "l2112"}, {ContractId: "l2201"}, {ContractId: "l2202"}, {ContractId: "l2203"}, {ContractId: "l2204"}, {ContractId: "l2205"}}},
-		SearchList{VarietyName: "聚氯乙烯", CarietyCode: "v", List: []SearchContractId{SearchContractId{ContractId: "v2111"}, {ContractId: "v2112"}, {ContractId: "v2201"}, {ContractId: "v2202"}, {ContractId: "v2203"}, {ContractId: "v2204"}, {ContractId: "v2205"}}},
-		SearchList{VarietyName: "聚丙烯", CarietyCode: "pp", List: []SearchContractId{SearchContractId{ContractId: "pp2112"}, {ContractId: "pp2201"}, {ContractId: "pp2202"}, {ContractId: "pp2203"}, {ContractId: "pp2204"}, {ContractId: "pp2205"}}},
-		SearchList{VarietyName: "苯乙烯", CarietyCode: "eb", List: []SearchContractId{SearchContractId{ContractId: "eb2111"}, {ContractId: "eb2112"}, {ContractId: "eb2201"}, {ContractId: "eb2202"}}},
-		SearchList{VarietyName: "焦炭", CarietyCode: "j", List: []SearchContractId{SearchContractId{ContractId: "j2201"}, {ContractId: "j2205"}}},
-		SearchList{VarietyName: "焦煤", CarietyCode: "jm", List: []SearchContractId{SearchContractId{ContractId: "jm2201"}, {ContractId: "jm2205"}}},
-		SearchList{VarietyName: "铁矿石", CarietyCode: "i", List: []SearchContractId{SearchContractId{ContractId: "i2112"}, {ContractId: "i2201"}, {ContractId: "i2202"}, {ContractId: "i2203"}, {ContractId: "i2204"}, {ContractId: "i2205"}, {ContractId: "i2206"}, {ContractId: "i2207"}, {ContractId: "i2208"}, {ContractId: "i2209"}}},
-		SearchList{VarietyName: "乙二醇", CarietyCode: "eg", List: []SearchContractId{SearchContractId{ContractId: "eg2112"}, {ContractId: "eg2201"}, {ContractId: "eg2202"}, {ContractId: "eg2203"}, {ContractId: "eg2205"}}},
-		SearchList{VarietyName: "液化石油气", CarietyCode: "pg", List: []SearchContractId{SearchContractId{ContractId: "pg2111"}, {ContractId: "pg2112"}, {ContractId: "pg2201"}, {ContractId: "pg2202"}}},
-	}
+	//n := utils.GetRandInt(10, 120)
+	//time.Sleep(time.Duration(n) * time.Second)
+	//searchList := []SearchList{
+	//	SearchList{VarietyName: "豆一", CarietyCode: "a", List: []SearchContractId{SearchContractId{ContractId: "a2111"}, {ContractId: "a2201"}, {ContractId: "a2203"}}},
+	//	SearchList{VarietyName: "豆二", CarietyCode: "b", List: []SearchContractId{SearchContractId{ContractId: "b2112"}, {ContractId: "b2201"}}},
+	//	SearchList{VarietyName: "豆粕", CarietyCode: "m", List: []SearchContractId{SearchContractId{ContractId: "m2111"}, {ContractId: "m2112"}, {ContractId: "m2201"}, {ContractId: "m2203"}, {ContractId: "m2205"}, {ContractId: "m2207"}, {ContractId: "m2208"}, {ContractId: "m2209"}}},
+	//	SearchList{VarietyName: "豆油", CarietyCode: "y", List: []SearchContractId{SearchContractId{ContractId: "y2112"}, {ContractId: "y2201"}, {ContractId: "y2203"}, {ContractId: "y2205"}, {ContractId: "y2207"}, {ContractId: "y2208"}}},
+	//	SearchList{VarietyName: "棕榈油", CarietyCode: "p", List: []SearchContractId{SearchContractId{ContractId: "p2112"}, {ContractId: "p2201"}, {ContractId: "p2202"}, {ContractId: "p2203"}, {ContractId: "p2204"}, {ContractId: "p2205"}}},
+	//	SearchList{VarietyName: "玉米", CarietyCode: "c", List: []SearchContractId{SearchContractId{ContractId: "c2111"}, {ContractId: "c2201"}, {ContractId: "c2203"}, {ContractId: "c2205"}, {ContractId: "c2207"}}},
+	//	SearchList{VarietyName: "玉米淀粉", CarietyCode: "cs", List: []SearchContractId{SearchContractId{ContractId: "cs2111"}, {ContractId: "cs2201"}, {ContractId: "cs2203"}}},
+	//	SearchList{VarietyName: "鸡蛋", CarietyCode: "jd", List: []SearchContractId{SearchContractId{ContractId: "jd2201"}, {ContractId: "jd2202"}, {ContractId: "jd2203"}, {ContractId: "jd2204"}, {ContractId: "jd2205"}}},
+	//	SearchList{VarietyName: "粳米", CarietyCode: "rr", List: []SearchContractId{SearchContractId{ContractId: "rr2112"}}},
+	//	//SearchList{VarietyName: "纤维板", CarietyCode: "fb", List:[]SearchContractId{}}, //无数据
+	//	//SearchList{VarietyName: "胶合板", CarietyCode: "bb", List:[]SearchContractId{}}, //无数据
+	//	SearchList{VarietyName: "生猪", CarietyCode: "lh", List: []SearchContractId{SearchContractId{ContractId: "lh2201"}, {ContractId: "lh2203"}, {ContractId: "lh2205"}}},
+	//	SearchList{VarietyName: "聚乙烯", CarietyCode: "l", List: []SearchContractId{SearchContractId{ContractId: "l2112"}, {ContractId: "l2201"}, {ContractId: "l2202"}, {ContractId: "l2203"}, {ContractId: "l2204"}, {ContractId: "l2205"}}},
+	//	SearchList{VarietyName: "聚氯乙烯", CarietyCode: "v", List: []SearchContractId{SearchContractId{ContractId: "v2111"}, {ContractId: "v2112"}, {ContractId: "v2201"}, {ContractId: "v2202"}, {ContractId: "v2203"}, {ContractId: "v2204"}, {ContractId: "v2205"}}},
+	//	SearchList{VarietyName: "聚丙烯", CarietyCode: "pp", List: []SearchContractId{SearchContractId{ContractId: "pp2112"}, {ContractId: "pp2201"}, {ContractId: "pp2202"}, {ContractId: "pp2203"}, {ContractId: "pp2204"}, {ContractId: "pp2205"}}},
+	//	SearchList{VarietyName: "苯乙烯", CarietyCode: "eb", List: []SearchContractId{SearchContractId{ContractId: "eb2111"}, {ContractId: "eb2112"}, {ContractId: "eb2201"}, {ContractId: "eb2202"}}},
+	//	SearchList{VarietyName: "焦炭", CarietyCode: "j", List: []SearchContractId{SearchContractId{ContractId: "j2201"}, {ContractId: "j2205"}}},
+	//	SearchList{VarietyName: "焦煤", CarietyCode: "jm", List: []SearchContractId{SearchContractId{ContractId: "jm2201"}, {ContractId: "jm2205"}}},
+	//	SearchList{VarietyName: "铁矿石", CarietyCode: "i", List: []SearchContractId{SearchContractId{ContractId: "i2112"}, {ContractId: "i2201"}, {ContractId: "i2202"}, {ContractId: "i2203"}, {ContractId: "i2204"}, {ContractId: "i2205"}, {ContractId: "i2206"}, {ContractId: "i2207"}, {ContractId: "i2208"}, {ContractId: "i2209"}}},
+	//	SearchList{VarietyName: "乙二醇", CarietyCode: "eg", List: []SearchContractId{SearchContractId{ContractId: "eg2112"}, {ContractId: "eg2201"}, {ContractId: "eg2202"}, {ContractId: "eg2203"}, {ContractId: "eg2205"}}},
+	//	SearchList{VarietyName: "液化石油气", CarietyCode: "pg", List: []SearchContractId{SearchContractId{ContractId: "pg2111"}, {ContractId: "pg2112"}, {ContractId: "pg2201"}, {ContractId: "pg2202"}}},
+	//}
 
 	//定义爬取时间
 	endDate := time.Now().AddDate(0, 0, -dayNum).Format(utils.FormatDateTime)
@@ -267,7 +480,7 @@ func SyncRankingFromDalian(dayNum int) (err error) {
 		return err
 	}
 	for _, v := range searchList {
-		for _, v2 := range v.List {
+		for _, v2 := range v.ListSearch {
 			//模拟form表单请求
 			url := "http://www.dce.com.cn/publicweb/quotesdata/memberDealPosiQuotes.html"
 			method := "POST"
@@ -279,7 +492,7 @@ func SyncRankingFromDalian(dayNum int) (err error) {
 			_ = writer.WriteField("month", month)
 			_ = writer.WriteField("day", dayStr)
 			_ = writer.WriteField("contract.contract_id", v2.ContractId)
-			_ = writer.WriteField("variety_id", v.CarietyCode)
+			_ = writer.WriteField("contract.variety_id", v.CarietyCode)
 			_ = writer.WriteField("currDate", currDate)
 			err := writer.Close()
 			fmt.Println(currDate, v.VarietyName, v2.ContractId)
@@ -313,16 +526,3 @@ func SyncRankingFromDalian(dayNum int) (err error) {
 	}
 	return err
 }
-
-func DlIndexCodeGenerator(indexName, suffix string) string {
-	ineIndexCode, _ := ineIndexCodeMap[indexName]
-	if ineIndexCode == "" {
-		ineIndexCode = fmt.Sprintf("DL%s", time.Now().Format(utils.FormatDateTimeUnSpace)+strconv.Itoa(utils.GetRandInt(1, 100))+suffix)
-		ineIndexCodeMap[indexName] = ineIndexCode
-		err := models.AddBaseFromTradeMapping(indexName, ineIndexCode, "DL")
-		if err != nil {
-			fmt.Println("add Code err:", err)
-		}
-	}
-	return ineIndexCode
-}

services/task.go

@@ -19,7 +19,6 @@ func Task() {
 	//SyncRankingFromDalianDo()
 	//SyncStorageFromEic()
 	//SyncStorageFromEic()
-
 	refreshData := task.NewTask("refreshData", "0 0 3 * * *", RefreshData)
 	task.AddTask("数据爬取", refreshData)
 	task.StartTask()
@@ -28,7 +27,7 @@ func Task() {
 
 func RefreshData(cont context.Context) (err error) {
 	//大连
-	SyncRankingFromDalian(1)
+	SyncRankingFromDalianSearch(1)
 	SyncRankingFromZhengzhou()
 	SyncRankingFromShangHai()
 	SyncRankingFromIne()