commodity_trade_dalian.go 20 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529
  1. package services
  2. import (
  3. "bytes"
  4. "fmt"
  5. "github.com/PuerkitoBio/goquery"
  6. "hongze/hongze_data_crawler/models"
  7. "hongze/hongze_data_crawler/utils"
  8. "io/ioutil"
  9. "log"
  10. "mime/multipart"
  11. "net/http"
  12. "strconv"
  13. "strings"
  14. "time"
  15. )
  16. type SearchList struct {
  17. VarietyName string `description:"商品名称"`
  18. CarietyCode string `description:"商品名称对应的编码"`
  19. List []SearchContractId
  20. ListSearch []*SearchContractId
  21. }
  22. type SearchContractId struct {
  23. ContractId string `description:"商品类型"`
  24. }
  25. //同步 N天 之内的数据
  26. func SyncRankingFromDalianDo() {
  27. for i := 200; i >= 0; i-- {
  28. SyncRankingFromDalianSearch(i)
  29. }
  30. }
  31. //大连交易所持单排名
  32. func SyncRankingFromDalianSearch(dayNum int) (err error) {
  33. fmt.Println("start")
  34. defer func() {
  35. if err != nil {
  36. fmt.Println("RefreshDataFromDalian Err:" + err.Error())
  37. go utils.SendEmail(utils.APPNAME+"【"+utils.RunMode+"】"+"失败提醒", "RefreshDataFromDalian ErrMsg:"+err.Error(), utils.EmailSendToUsers)
  38. }
  39. }()
  40. //定义爬取时间
  41. endDate := time.Now().AddDate(0, 0, -dayNum).Format(utils.FormatDateTime)
  42. //endDate := time.Now().Format(utils.FormatDateTime)
  43. timeDate := utils.StrTimeToTime(endDate)
  44. currDate := timeDate.Format(utils.FormatDateUnSpace)
  45. year := timeDate.Year()
  46. month := timeDate.Format("01")
  47. var dayStr string
  48. day := timeDate.Day()
  49. if day < 10 {
  50. dayStr = "0" + strconv.Itoa(day)
  51. } else {
  52. dayStr = strconv.Itoa(day)
  53. }
  54. monthNum, _ := strconv.Atoi(month)
  55. month = strconv.Itoa(monthNum - 1) //获取时月份需要减一
  56. list, err := models.GetBaseFromTradeDalianDataList(timeDate.Format(utils.FormatDate))
  57. if err != nil {
  58. fmt.Println(err)
  59. return err
  60. }
  61. listDataMap := make(map[string]int)
  62. for _, v := range list {
  63. listDataMap[v.DealShortName+v.ClassifyType+v.DataTime] = v.BaseFromTradeDalianIndexId
  64. }
  65. var ContractId string
  66. var CarietyCode string
  67. var VarietyName string
  68. //模拟form表单请求
  69. url := "http://www.dce.com.cn/publicweb/quotesdata/memberDealPosiQuotes.html"
  70. method := "POST"
  71. payload := &bytes.Buffer{}
  72. writer := multipart.NewWriter(payload)
  73. _ = writer.WriteField("memberDealPosiQuotes.variety", CarietyCode)
  74. _ = writer.WriteField("memberDealPosiQuotes.trade_type", "0")
  75. _ = writer.WriteField("year", strconv.Itoa(year))
  76. _ = writer.WriteField("month", month)
  77. _ = writer.WriteField("day", dayStr)
  78. _ = writer.WriteField("contract.contract_id", ContractId)
  79. _ = writer.WriteField("contract.variety_id", CarietyCode)
  80. _ = writer.WriteField("currDate", currDate)
  81. err = writer.Close()
  82. if err != nil {
  83. utils.FileLog.Info("获取指标失败:" + currDate + VarietyName + ContractId)
  84. return err
  85. }
  86. client := &http.Client{}
  87. req, err := http.NewRequest(method, url, payload)
  88. if err != nil {
  89. return err
  90. }
  91. req.Header.Add("Cookie", "JSESSIONID=36ACF02A59227A3854F9D5D5E2FB5F2E; WMONID=R5ojcAIIcx-")
  92. req.Header.Set("Content-Type", writer.FormDataContentType())
  93. res, err := client.Do(req)
  94. if err != nil {
  95. return err
  96. }
  97. defer res.Body.Close()
  98. body, err := ioutil.ReadAll(res.Body)
  99. if err != nil {
  100. go utils.SendEmail(utils.APPNAME+"【"+utils.RunMode+"】"+"失败提醒", "RefreshDataFromDalian ErrMsg:"+err.Error()+"获取指标失败:"+currDate+VarietyName+ContractId, utils.EmailSendToUsers)
  101. return err
  102. }
  103. exitProductMaps, _, varietyArrMaps := DoSearch(string(body))
  104. var items []*SearchList
  105. for k, v := range exitProductMaps {
  106. item := new(SearchList)
  107. item.VarietyName = v
  108. item.CarietyCode = varietyArrMaps[k]
  109. htmlBody, err := GetDalianHtmlBody(dayNum, "", item.CarietyCode, item.VarietyName)
  110. if err != nil {
  111. return err
  112. }
  113. listContractId := DoSearchMap(htmlBody)
  114. item.ListSearch = listContractId
  115. items = append(items, item)
  116. }
  117. for k, v := range items {
  118. for _, v2 := range v.ListSearch {
  119. fmt.Println(currDate, v.VarietyName, v2.ContractId, v.CarietyCode, k)
  120. }
  121. }
  122. SyncRankingFromDalian(dayNum, items)
  123. return err
  124. }
  125. //处理搜索条件初始
  126. func DoSearch(body string) (exitProductMaps, exitContractIdMaps, varietyArrMaps map[int]string) {
  127. var str string
  128. str = body
  129. doc, err := goquery.NewDocumentFromReader(strings.NewReader(str))
  130. if err != nil {
  131. log.Fatal(err)
  132. }
  133. exitProductMap := make(map[int]string)
  134. exitContractIdMap := make(map[int]string)
  135. varietyArrMap := make(map[int]string)
  136. //var productName string
  137. ul := doc.Find(".selBox ul")
  138. var pNum int
  139. var cidNum int
  140. var vNum int
  141. ul.Each(func(i int, s *goquery.Selection) {
  142. //解析标签
  143. //fmt.Println(i, s.Text())
  144. ulTxt := s.Text()
  145. //fmt.Println(ulTxt)
  146. if ulTxt != "" && (i == 0 || i == 2) {
  147. ulTxtArr := strings.Split(ulTxt, "\n")
  148. for _, v := range ulTxtArr {
  149. v = strings.Replace(v, " ", "", -1)
  150. v = strings.Replace(v, "\n", "", -1)
  151. v = strings.Replace(v, " ", "", -1)
  152. if v != "" && len(v) > 0 {
  153. exitProductMap[pNum] = v
  154. pNum++
  155. }
  156. }
  157. }
  158. if ulTxt != "" && i == 3 {
  159. //fmt.Println(ulTxt)
  160. cidTxtArr := strings.Split(ulTxt, " ")
  161. for _, v := range cidTxtArr {
  162. v = strings.Replace(v, "\n", "", -1)
  163. v = strings.Replace(v, " ", "", -1)
  164. v = strings.Replace(v, " ", "", -1)
  165. if v != "" {
  166. exitContractIdMap[cidNum] = v
  167. cidNum++
  168. }
  169. }
  170. }
  171. })
  172. varietyArr := strings.Split(str, "onclick=\"javascript:setVariety('")
  173. for _, v := range varietyArr {
  174. strnum := strings.Index(v, "');")
  175. if strnum > 0 {
  176. varietyStr := v[0:strnum]
  177. if len(varietyStr) < 10 {
  178. //fmt.Println(strnum, varietyStr)
  179. varietyArrMap[vNum] = varietyStr
  180. vNum++
  181. }
  182. }
  183. }
  184. exitProductMaps = exitProductMap
  185. exitContractIdMaps = exitContractIdMap
  186. varietyArrMaps = varietyArrMap
  187. return
  188. }
  189. //处理搜索条件
  190. func DoSearchMap(body string) (items []*SearchContractId) {
  191. var str string
  192. str = body
  193. doc, err := goquery.NewDocumentFromReader(strings.NewReader(str))
  194. if err != nil {
  195. log.Fatal(err)
  196. }
  197. exitContractIdMap := make(map[int]string)
  198. ul := doc.Find(".selBox ul")
  199. var cidNum int
  200. ul.Each(func(i int, s *goquery.Selection) {
  201. //解析标签
  202. ulTxt := s.Text()
  203. if ulTxt != "" && i == 3 {
  204. cidTxtArr := strings.Split(ulTxt, " ")
  205. for _, v := range cidTxtArr {
  206. v = strings.Replace(v, "\n", "", -1)
  207. v = strings.Replace(v, " ", "", -1)
  208. v = strings.Replace(v, " ", "", -1)
  209. if v != "" {
  210. exitContractIdMap[cidNum] = v
  211. cidNum++
  212. }
  213. }
  214. }
  215. })
  216. for _, v := range exitContractIdMap {
  217. item := new(SearchContractId)
  218. item.ContractId = v
  219. items = append(items, item)
  220. }
  221. return items
  222. }
  223. //处理解析Html
  224. func DoHtml(body, name, contractId string, dateTime time.Time, listDataMap map[string]int) (err error) {
  225. defer func() {
  226. if err != nil {
  227. fmt.Println("RefreshDataFromDaLian Err:" + err.Error())
  228. go utils.SendEmail(utils.APPNAME+"【"+utils.RunMode+"】"+"失败提醒", "RefreshDataFromDaLian ErrMsg:"+err.Error(), utils.EmailSendToUsers)
  229. }
  230. }()
  231. str := body
  232. doc, err := goquery.NewDocumentFromReader(strings.NewReader(str))
  233. if err != nil {
  234. log.Fatal(err)
  235. }
  236. var isAdd bool
  237. addSql := ` INSERT INTO base_from_trade_dalian_index(rank,deal_short_name,deal_name,deal_code,deal_value,buy_short_name,deal_change,buy_name,buy_code,buy_value,buy_change,sold_short_name,sold_name,sold_code,sold_value,sold_change,frequency,classify_name,classify_type,create_time,modify_time,data_time) values `
  238. table := doc.Find("table")
  239. var rank, shortName, dealValue, dealChange, buyName, buyValue, buyChange, soldName, soldValue, soldChange string
  240. table.Find("tr").Each(func(i int, tr *goquery.Selection) {
  241. tds := tr.Find("td")
  242. //fmt.Println(tds.Length(), "长度:", i)
  243. if tds.Length() == 0 || tds.Length() == 7 || i == 23 {
  244. tdText := tds.Text()
  245. utils.FileLog.Info(tdText)
  246. } else {
  247. item := new(models.BaseFromTradeDalianIndex)
  248. tds.Each(func(tk int, td *goquery.Selection) {
  249. tdText := td.Text()
  250. if tk == 0 { //名次
  251. rank = tdText
  252. }
  253. if tk == 1 { //会员简称
  254. shortName = tdText
  255. }
  256. if tk == 2 { //成交量
  257. dealValue = strings.Replace(tdText, ",", "", -1)
  258. }
  259. if tk == 3 { //增减
  260. dealChange = strings.Replace(tdText, ",", "", -1)
  261. }
  262. if tk == 5 { //会员简称
  263. buyName = tdText
  264. }
  265. if tk == 6 { //持买单量
  266. buyValue = strings.Replace(tdText, ",", "", -1)
  267. }
  268. if tk == 7 { //增减
  269. buyChange = strings.Replace(tdText, ",", "", -1)
  270. }
  271. if tk == 9 { //会员简称
  272. soldName = tdText
  273. }
  274. if tk == 10 { //持卖单量
  275. soldValue = strings.Replace(tdText, ",", "", -1)
  276. }
  277. if tk == 11 { //增减
  278. soldChange = strings.Replace(tdText, ",", "", -1)
  279. }
  280. })
  281. item.Rank = rank
  282. item.DealShortName = shortName
  283. item.DealName = shortName + "_成交量_" + contractId
  284. item.DealCode = DlIndexCodeGenerator(item.DealName, "deal")
  285. item.DealValue = dealValue
  286. item.BuyShortName = buyName
  287. item.DealChange = dealChange
  288. item.BuyName = buyName + "_持买单量_" + contractId
  289. item.BuyCode = DlIndexCodeGenerator(item.BuyName, "buy")
  290. item.BuyValue = buyValue
  291. item.BuyChange = buyChange
  292. item.SoldShortName = soldName
  293. item.SoldName = soldName + "_持卖单量_" + contractId
  294. item.SoldCode = DlIndexCodeGenerator(item.SoldName, "sold")
  295. item.SoldValue = soldValue
  296. item.SoldChange = soldChange
  297. item.Frequency = "日度"
  298. item.ClassifyName = name
  299. item.ClassifyType = contractId
  300. item.CreateTime = time.Now().Format(utils.FormatDateTime)
  301. item.ModifyTime = time.Now().Format(utils.FormatDateTime)
  302. item.DataTime = dateTime.Format(utils.FormatDate)
  303. if val, ok := listDataMap[item.DealShortName+item.ClassifyType+item.DataTime]; !ok {
  304. addSql += models.GetAddSql(item)
  305. isAdd = true
  306. } else {
  307. //更新
  308. err := models.UpdateBaseFromTradeDalianIndex(item, val)
  309. if err != nil {
  310. fmt.Println("UpdateBaseFromTradeDalianIndex err:", err)
  311. }
  312. }
  313. }
  314. })
  315. addSql = strings.TrimRight(addSql, ",")
  316. if isAdd {
  317. err = models.RefreshEdbDataByDaLian(addSql)
  318. if err != nil {
  319. return err
  320. }
  321. }
  322. return
  323. }
  324. func DlIndexCodeGenerator(indexName, suffix string) string {
  325. ineIndexCode, _ := ineIndexCodeMap[indexName]
  326. if ineIndexCode == "" {
  327. ineIndexCode = fmt.Sprintf("DL%s", time.Now().Format(utils.FormatDateTimeUnSpace)+strconv.Itoa(utils.GetRandInt(1, 100))+suffix)
  328. ineIndexCodeMap[indexName] = ineIndexCode
  329. err := models.AddBaseFromTradeMapping(indexName, ineIndexCode, "DL")
  330. if err != nil {
  331. fmt.Println("add Code err:", err)
  332. }
  333. }
  334. return ineIndexCode
  335. }
  336. //大连交易所持单排名
  337. func GetDalianHtmlBody(dayNum int, contractId, carietyCode, varietyName string) (body string, err error) {
  338. defer func() {
  339. if err != nil {
  340. fmt.Println("GetDalianHtmlBody Err:" + err.Error())
  341. go utils.SendEmail(utils.APPNAME+"【"+utils.RunMode+"】"+"失败提醒", "GetDalianHtmlBody ErrMsg:"+err.Error(), utils.EmailSendToUsers)
  342. }
  343. }()
  344. //定义爬取时间
  345. endDate := time.Now().AddDate(0, 0, -dayNum).Format(utils.FormatDateTime)
  346. //endDate := time.Now().Format(utils.FormatDateTime)
  347. timeDate := utils.StrTimeToTime(endDate)
  348. currDate := timeDate.Format(utils.FormatDateUnSpace)
  349. year := timeDate.Year()
  350. month := timeDate.Format("01")
  351. var dayStr string
  352. day := timeDate.Day()
  353. if day < 10 {
  354. dayStr = "0" + strconv.Itoa(day)
  355. } else {
  356. dayStr = strconv.Itoa(day)
  357. }
  358. monthNum, _ := strconv.Atoi(month)
  359. month = strconv.Itoa(monthNum - 1) //获取时月份需要减一
  360. list, err := models.GetBaseFromTradeDalianDataList(timeDate.Format(utils.FormatDate))
  361. listDataMap := make(map[string]int)
  362. for _, v := range list {
  363. listDataMap[v.DealShortName+v.ClassifyType+v.DataTime] = v.BaseFromTradeDalianIndexId
  364. }
  365. if err != nil {
  366. fmt.Println(err)
  367. return
  368. }
  369. //模拟form表单请求
  370. url := "http://www.dce.com.cn/publicweb/quotesdata/memberDealPosiQuotes.html"
  371. method := "POST"
  372. payload := &bytes.Buffer{}
  373. writer := multipart.NewWriter(payload)
  374. _ = writer.WriteField("memberDealPosiQuotes.variety", carietyCode)
  375. _ = writer.WriteField("memberDealPosiQuotes.trade_type", "0")
  376. _ = writer.WriteField("year", strconv.Itoa(year))
  377. _ = writer.WriteField("month", month)
  378. _ = writer.WriteField("day", dayStr)
  379. _ = writer.WriteField("contract.contract_id", contractId)
  380. _ = writer.WriteField("contract.variety_id", carietyCode)
  381. _ = writer.WriteField("currDate", currDate)
  382. err = writer.Close()
  383. if err != nil {
  384. utils.FileLog.Info("获取指标失败:" + currDate + varietyName + contractId)
  385. return
  386. }
  387. client := &http.Client{}
  388. req, err := http.NewRequest(method, url, payload)
  389. if err != nil {
  390. return
  391. }
  392. req.Header.Add("Cookie", "JSESSIONID=36ACF02A59227A3854F9D5D5E2FB5F2E; WMONID=R5ojcAIIcx-")
  393. req.Header.Set("Content-Type", writer.FormDataContentType())
  394. res, err := client.Do(req)
  395. if err != nil {
  396. return
  397. }
  398. defer res.Body.Close()
  399. htmlBody, err := ioutil.ReadAll(res.Body)
  400. if err != nil {
  401. go utils.SendEmail(utils.APPNAME+"【"+utils.RunMode+"】"+"失败提醒", "GetDalianHtmlBody ErrMsg:"+err.Error()+"获取指标失败:"+currDate+varietyName+contractId, utils.EmailSendToUsers)
  402. return
  403. }
  404. body = string(htmlBody)
  405. return
  406. }
  407. //大连交易所持单排名
  408. func SyncRankingFromDalian(dayNum int, searchList []*SearchList) (err error) {
  409. fmt.Println("start")
  410. defer func() {
  411. if err != nil {
  412. fmt.Println("RefreshDataFromDalian Err:" + err.Error())
  413. go utils.SendEmail(utils.APPNAME+"【"+utils.RunMode+"】"+"失败提醒", "RefreshDataFromDalian ErrMsg:"+err.Error(), utils.EmailSendToUsers)
  414. }
  415. }()
  416. //n := utils.GetRandInt(10, 120)
  417. //time.Sleep(time.Duration(n) * time.Second)
  418. //searchList := []SearchList{
  419. // SearchList{VarietyName: "豆一", CarietyCode: "a", List: []SearchContractId{SearchContractId{ContractId: "a2111"}, {ContractId: "a2201"}, {ContractId: "a2203"}}},
  420. // SearchList{VarietyName: "豆二", CarietyCode: "b", List: []SearchContractId{SearchContractId{ContractId: "b2112"}, {ContractId: "b2201"}}},
  421. // SearchList{VarietyName: "豆粕", CarietyCode: "m", List: []SearchContractId{SearchContractId{ContractId: "m2111"}, {ContractId: "m2112"}, {ContractId: "m2201"}, {ContractId: "m2203"}, {ContractId: "m2205"}, {ContractId: "m2207"}, {ContractId: "m2208"}, {ContractId: "m2209"}}},
  422. // SearchList{VarietyName: "豆油", CarietyCode: "y", List: []SearchContractId{SearchContractId{ContractId: "y2112"}, {ContractId: "y2201"}, {ContractId: "y2203"}, {ContractId: "y2205"}, {ContractId: "y2207"}, {ContractId: "y2208"}}},
  423. // SearchList{VarietyName: "棕榈油", CarietyCode: "p", List: []SearchContractId{SearchContractId{ContractId: "p2112"}, {ContractId: "p2201"}, {ContractId: "p2202"}, {ContractId: "p2203"}, {ContractId: "p2204"}, {ContractId: "p2205"}}},
  424. // SearchList{VarietyName: "玉米", CarietyCode: "c", List: []SearchContractId{SearchContractId{ContractId: "c2111"}, {ContractId: "c2201"}, {ContractId: "c2203"}, {ContractId: "c2205"}, {ContractId: "c2207"}}},
  425. // SearchList{VarietyName: "玉米淀粉", CarietyCode: "cs", List: []SearchContractId{SearchContractId{ContractId: "cs2111"}, {ContractId: "cs2201"}, {ContractId: "cs2203"}}},
  426. // SearchList{VarietyName: "鸡蛋", CarietyCode: "jd", List: []SearchContractId{SearchContractId{ContractId: "jd2201"}, {ContractId: "jd2202"}, {ContractId: "jd2203"}, {ContractId: "jd2204"}, {ContractId: "jd2205"}}},
  427. // SearchList{VarietyName: "粳米", CarietyCode: "rr", List: []SearchContractId{SearchContractId{ContractId: "rr2112"}}},
  428. // //SearchList{VarietyName: "纤维板", CarietyCode: "fb", List:[]SearchContractId{}}, //无数据
  429. // //SearchList{VarietyName: "胶合板", CarietyCode: "bb", List:[]SearchContractId{}}, //无数据
  430. // SearchList{VarietyName: "生猪", CarietyCode: "lh", List: []SearchContractId{SearchContractId{ContractId: "lh2201"}, {ContractId: "lh2203"}, {ContractId: "lh2205"}}},
  431. // SearchList{VarietyName: "聚乙烯", CarietyCode: "l", List: []SearchContractId{SearchContractId{ContractId: "l2112"}, {ContractId: "l2201"}, {ContractId: "l2202"}, {ContractId: "l2203"}, {ContractId: "l2204"}, {ContractId: "l2205"}}},
  432. // SearchList{VarietyName: "聚氯乙烯", CarietyCode: "v", List: []SearchContractId{SearchContractId{ContractId: "v2111"}, {ContractId: "v2112"}, {ContractId: "v2201"}, {ContractId: "v2202"}, {ContractId: "v2203"}, {ContractId: "v2204"}, {ContractId: "v2205"}}},
  433. // SearchList{VarietyName: "聚丙烯", CarietyCode: "pp", List: []SearchContractId{SearchContractId{ContractId: "pp2112"}, {ContractId: "pp2201"}, {ContractId: "pp2202"}, {ContractId: "pp2203"}, {ContractId: "pp2204"}, {ContractId: "pp2205"}}},
  434. // SearchList{VarietyName: "苯乙烯", CarietyCode: "eb", List: []SearchContractId{SearchContractId{ContractId: "eb2111"}, {ContractId: "eb2112"}, {ContractId: "eb2201"}, {ContractId: "eb2202"}}},
  435. // SearchList{VarietyName: "焦炭", CarietyCode: "j", List: []SearchContractId{SearchContractId{ContractId: "j2201"}, {ContractId: "j2205"}}},
  436. // SearchList{VarietyName: "焦煤", CarietyCode: "jm", List: []SearchContractId{SearchContractId{ContractId: "jm2201"}, {ContractId: "jm2205"}}},
  437. // SearchList{VarietyName: "铁矿石", CarietyCode: "i", List: []SearchContractId{SearchContractId{ContractId: "i2112"}, {ContractId: "i2201"}, {ContractId: "i2202"}, {ContractId: "i2203"}, {ContractId: "i2204"}, {ContractId: "i2205"}, {ContractId: "i2206"}, {ContractId: "i2207"}, {ContractId: "i2208"}, {ContractId: "i2209"}}},
  438. // SearchList{VarietyName: "乙二醇", CarietyCode: "eg", List: []SearchContractId{SearchContractId{ContractId: "eg2112"}, {ContractId: "eg2201"}, {ContractId: "eg2202"}, {ContractId: "eg2203"}, {ContractId: "eg2205"}}},
  439. // SearchList{VarietyName: "液化石油气", CarietyCode: "pg", List: []SearchContractId{SearchContractId{ContractId: "pg2111"}, {ContractId: "pg2112"}, {ContractId: "pg2201"}, {ContractId: "pg2202"}}},
  440. //}
  441. //定义爬取时间
  442. endDate := time.Now().AddDate(0, 0, -dayNum).Format(utils.FormatDateTime)
  443. //endDate := time.Now().Format(utils.FormatDateTime)
  444. timeDate := utils.StrTimeToTime(endDate)
  445. currDate := timeDate.Format(utils.FormatDateUnSpace)
  446. year := timeDate.Year()
  447. month := timeDate.Format("01")
  448. var dayStr string
  449. day := timeDate.Day()
  450. if day < 10 {
  451. dayStr = "0" + strconv.Itoa(day)
  452. } else {
  453. dayStr = strconv.Itoa(day)
  454. }
  455. monthNum, _ := strconv.Atoi(month)
  456. month = strconv.Itoa(monthNum - 1) //获取时月份需要减一
  457. list, err := models.GetBaseFromTradeDalianDataList(timeDate.Format(utils.FormatDate))
  458. listDataMap := make(map[string]int)
  459. for _, v := range list {
  460. listDataMap[v.DealShortName+v.ClassifyType+v.DataTime] = v.BaseFromTradeDalianIndexId
  461. }
  462. if err != nil {
  463. fmt.Println(err)
  464. return err
  465. }
  466. for _, v := range searchList {
  467. for _, v2 := range v.ListSearch {
  468. //模拟form表单请求
  469. url := "http://www.dce.com.cn/publicweb/quotesdata/memberDealPosiQuotes.html"
  470. method := "POST"
  471. payload := &bytes.Buffer{}
  472. writer := multipart.NewWriter(payload)
  473. _ = writer.WriteField("memberDealPosiQuotes.variety", v.CarietyCode)
  474. _ = writer.WriteField("memberDealPosiQuotes.trade_type", "0")
  475. _ = writer.WriteField("year", strconv.Itoa(year))
  476. _ = writer.WriteField("month", month)
  477. _ = writer.WriteField("day", dayStr)
  478. _ = writer.WriteField("contract.contract_id", v2.ContractId)
  479. _ = writer.WriteField("contract.variety_id", v.CarietyCode)
  480. _ = writer.WriteField("currDate", currDate)
  481. err := writer.Close()
  482. fmt.Println(currDate, v.VarietyName, v2.ContractId)
  483. if err != nil {
  484. utils.FileLog.Info("获取指标失败:" + currDate + v.VarietyName + v2.ContractId)
  485. return err
  486. }
  487. client := &http.Client{}
  488. req, err := http.NewRequest(method, url, payload)
  489. if err != nil {
  490. return err
  491. }
  492. req.Header.Add("Cookie", "JSESSIONID=36ACF02A59227A3854F9D5D5E2FB5F2E; WMONID=R5ojcAIIcx-")
  493. req.Header.Set("Content-Type", writer.FormDataContentType())
  494. res, err := client.Do(req)
  495. if err != nil {
  496. return err
  497. }
  498. defer res.Body.Close()
  499. body, err := ioutil.ReadAll(res.Body)
  500. if err != nil {
  501. go utils.SendEmail(utils.APPNAME+"【"+utils.RunMode+"】"+"失败提醒", "RefreshDataFromDalian ErrMsg:"+err.Error()+"获取指标失败:"+currDate+v.VarietyName+v2.ContractId, utils.EmailSendToUsers)
  502. return err
  503. }
  504. err = DoHtml(string(body), v.VarietyName, v2.ContractId, timeDate, listDataMap)
  505. if err != nil {
  506. return err
  507. }
  508. }
  509. }
  510. return err
  511. }