trade_guangzhou.go 29 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875
  1. package services
  2. import (
  3. "encoding/json"
  4. "eta/eta_crawler/models"
  5. "eta/eta_crawler/services/alarm_msg"
  6. "eta/eta_crawler/utils"
  7. "fmt"
  8. "github.com/mozillazg/go-pinyin"
  9. "strconv"
  10. "strings"
  11. "time"
  12. )
  13. // SyncRankingFromGuangzhou 广州交易所期货-日行情
  14. func CrawlerFromGuangzhouDayQuotes(date time.Time) {
  15. var err error
  16. defer func() {
  17. if err != nil {
  18. msg := "失败提醒" + "SyncRankingFromGuangzhou ErrMsg:" + err.Error()
  19. fmt.Println("msg:", msg)
  20. utils.FileLog.Info(msg)
  21. go alarm_msg.SendAlarmMsg(msg, 3)
  22. }
  23. }()
  24. now := date //time.Now()
  25. if now.Weekday() == time.Saturday || now.Weekday() == time.Sunday {
  26. return
  27. }
  28. dateStr := now.Format(utils.FormatDateUnSpace)
  29. zzUrl := `http://www.gfex.com.cn/u/interfacesWebTiDayQuotes/loadList`
  30. zzUrl = fmt.Sprintf(zzUrl)
  31. fmt.Println(zzUrl)
  32. // params := ""
  33. params := fmt.Sprintf("trade_date=%s&trade_type=0", dateStr)
  34. headerParams := make(map[string]string)
  35. //headerParams["Cookie"] = "CT6T=312900; SF_cookie_3=68941398"
  36. headerParams["User-Agent"] = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36"
  37. headerParams["Content-Type"] = "application/x-www-form-urlencoded;charset=utf-8"
  38. body, e := utils.HttpPost(zzUrl, params, headerParams)
  39. if e != nil {
  40. err = e
  41. fmt.Println("err:", err)
  42. return
  43. }
  44. fmt.Println("body :" + string(body))
  45. utils.FileLog.Info("body:" + string(body))
  46. if strings.Contains(string(body), "很抱歉! 因系统检测到您的请求可能对网站造成威胁") {
  47. return
  48. }
  49. item := new(models.TradeGuangZhou)
  50. err = json.Unmarshal(body, &item)
  51. if err != nil {
  52. fmt.Println("json.Unmarshal err:" + err.Error())
  53. return
  54. }
  55. gzObj := new(models.BaseFromTradeGuangzhouIndex)
  56. parentId := 1
  57. classifyList, err := gzObj.GetBaseFromTradeGuangzhouClassifyAll(parentId)
  58. if err != nil {
  59. fmt.Println("GetBaseFromTradeGuangzhouClassify Err:" + err.Error())
  60. return
  61. }
  62. classifyMap := make(map[string]*models.BaseFromTradeGuangzhouClassify)
  63. for _, cv := range classifyList {
  64. if cv.ClassifyCode != "" {
  65. classifyMap[cv.ClassifyCode] = cv
  66. }
  67. }
  68. //获取所有指标
  69. indexList, err := gzObj.GetBaseFromTradeGuangzhouIndexAll()
  70. if err != nil {
  71. fmt.Println("GetBaseFromTradeGuangzhouIndexAll Err:" + err.Error())
  72. return
  73. }
  74. indexMap := make(map[string]*models.BaseFromTradeGuangzhouIndex)
  75. for _, iv := range indexList {
  76. indexMap[iv.IndexCode] = iv
  77. }
  78. dataList, err := gzObj.GetBaseFromTradeGuangzhouDataByDate(now.Format(utils.FormatDate))
  79. dataMap := make(map[string]*models.BaseFromTradeGuangzhouData)
  80. for _, dv := range dataList {
  81. dataMap[dv.IndexCode] = dv
  82. }
  83. addDataList := make([]*models.BaseFromTradeGuangzhouData, 0)
  84. updateDataList := make([]*models.BaseFromTradeGuangzhouData, 0)
  85. for _, dv := range item.Data {
  86. if dv.Variety == "总计" {
  87. continue
  88. }
  89. var classifyId int
  90. if classifyItem, ok := classifyMap[dv.VarietyOrder]; !ok {
  91. classifyItem = new(models.BaseFromTradeGuangzhouClassify)
  92. classifyItem.ClassifyCode = dv.VarietyOrder
  93. classifyItem.ClassifyName = dv.Variety
  94. classifyItem.ParentId = 1
  95. classifyItem.ModifyTime = time.Now()
  96. classifyItem.CreateTime = time.Now()
  97. classifyLastId, err := gzObj.AddBaseFromTradeGuangzhouClassify(classifyItem)
  98. if err != nil {
  99. fmt.Println("AddBaseFromTradeGuangzhouClassify err:" + err.Error())
  100. return
  101. }
  102. classifyId = int(classifyLastId)
  103. } else {
  104. classifyId = classifyItem.BaseFromTradeGuangzhouClassifyId
  105. }
  106. if dv.DelivMonth == "" {
  107. continue
  108. }
  109. //指标编码
  110. //开盘价
  111. openIndexCode := dv.VarietyOrder + dv.DelivMonth + "open"
  112. //最高价
  113. highIndexCode := dv.VarietyOrder + dv.DelivMonth + "high"
  114. //最低价
  115. lowIndexCode := dv.VarietyOrder + dv.DelivMonth + "low"
  116. //收盘价
  117. closeIndexCode := dv.VarietyOrder + dv.DelivMonth + "close"
  118. //前结算价
  119. lastClearIndexCode := dv.VarietyOrder + dv.DelivMonth + "lastClear"
  120. //结算价
  121. clearPriceIndexCode := dv.VarietyOrder + dv.DelivMonth + "clearPrice"
  122. //涨跌
  123. diffIndexCode := dv.VarietyOrder + dv.DelivMonth + "diff"
  124. //涨跌1
  125. diff1IndexCode := dv.VarietyOrder + dv.DelivMonth + "diff1"
  126. //成交量
  127. volumeIndexCode := dv.VarietyOrder + dv.DelivMonth + "volumn"
  128. //持仓量
  129. openInterestIndexCode := dv.VarietyOrder + dv.DelivMonth + "openInterest"
  130. //持仓量变化
  131. diffIChangeIndexCode := dv.VarietyOrder + dv.DelivMonth + "diffI"
  132. //成交额
  133. turnoverIndexCode := dv.VarietyOrder + dv.DelivMonth + "turnover"
  134. handleIndexCodeArr := []string{openIndexCode, highIndexCode, lowIndexCode, closeIndexCode, lastClearIndexCode, clearPriceIndexCode, diffIndexCode, diff1IndexCode, volumeIndexCode, openInterestIndexCode, diffIChangeIndexCode, turnoverIndexCode}
  135. prefxCodeArr := []string{"open", "high", "low", "close", "lastClear", "clearPrice", "diff", "diff1", "volumn", "openInterest", "diffI", "turnover"}
  136. handleIndexNameArr := []string{"开盘价", "最高价", "最低价", "收盘价", "前结算价", "结算价", "涨跌", "涨跌1", "成交量", "持仓量", "持仓量变化", "成交额"}
  137. for ck, cv := range handleIndexCodeArr {
  138. var indexId int
  139. handleIndexCode := cv
  140. if indexItem, ok := indexMap[handleIndexCode]; !ok {
  141. indexName := dv.Variety + dv.DelivMonth + handleIndexNameArr[ck]
  142. var unit string
  143. if strings.Contains(indexName, "价") || strings.Contains(indexName, "涨跌") {
  144. unit = "元/吨"
  145. } else if strings.Contains(indexName, "量") {
  146. unit = "手"
  147. } else if strings.Contains(indexName, "成交额") {
  148. unit = "万元"
  149. }
  150. gzObj = &models.BaseFromTradeGuangzhouIndex{
  151. BaseFromTradeGuangzhouClassifyId: classifyId,
  152. IndexCode: handleIndexCode,
  153. IndexName: indexName,
  154. Frequency: "日度",
  155. Unit: unit,
  156. CreateTime: time.Now(),
  157. ModifyTime: time.Now(),
  158. }
  159. lastId, err := gzObj.AddBaseFromTradeGuangzhouIndex()
  160. if err != nil {
  161. fmt.Println("AddBaseFromTradeGuangzhouIndex Err:" + err.Error())
  162. return
  163. }
  164. indexId = int(lastId)
  165. } else {
  166. indexId = indexItem.BaseFromTradeGuangzhouIndexId
  167. }
  168. if dataItem, ok := dataMap[handleIndexCode]; !ok {
  169. dataItem = new(models.BaseFromTradeGuangzhouData)
  170. dataItem.BaseFromTradeGuangzhouIndexId = indexId
  171. dataItem.IndexCode = handleIndexCode
  172. dataItem.DataTime = dateStr
  173. switch prefxCodeArr[ck] {
  174. case "open":
  175. dataItem.Value = float64(dv.Open)
  176. case "high":
  177. dataItem.Value = float64(dv.High)
  178. case "low":
  179. dataItem.Value = float64(dv.Low)
  180. case "close":
  181. dataItem.Value = float64(dv.Close)
  182. case "lastClear":
  183. dataItem.Value = float64(dv.LastClear)
  184. case "clearPrice":
  185. dataItem.Value = float64(dv.ClearPrice)
  186. case "diff":
  187. dataItem.Value = float64(dv.Diff)
  188. case "diff1":
  189. dataItem.Value = float64(dv.Diff1)
  190. case "volumn":
  191. dataItem.Value = float64(dv.Volumn)
  192. case "openInterest":
  193. dataItem.Value = float64(dv.OpenInterest)
  194. case "diffI":
  195. dataItem.Value = float64(dv.DiffI)
  196. case "turnover":
  197. dataItem.Value = dv.Turnover
  198. }
  199. dataItem.CreateTime = time.Now()
  200. dataItem.ModifyTime = time.Now()
  201. addDataList = append(addDataList, dataItem)
  202. } else {
  203. var getValue float64
  204. switch prefxCodeArr[ck] {
  205. case "open":
  206. getValue = float64(dv.Open)
  207. case "high":
  208. getValue = float64(dv.High)
  209. case "low":
  210. getValue = float64(dv.Low)
  211. case "close":
  212. getValue = float64(dv.Close)
  213. case "lastClear":
  214. getValue = float64(dv.LastClear)
  215. case "clearPrice":
  216. getValue = float64(dv.ClearPrice)
  217. case "diff":
  218. getValue = float64(dv.Diff)
  219. case "diff1":
  220. getValue = float64(dv.Diff1)
  221. case "volumn":
  222. getValue = float64(dv.Volumn)
  223. case "openInterest":
  224. getValue = float64(dv.OpenInterest)
  225. case "diffI":
  226. getValue = float64(dv.DiffI)
  227. case "turnover":
  228. getValue = dv.Turnover
  229. }
  230. if dataItem.Value != getValue {
  231. dataItem.BaseFromTradeGuangzhouDataId = dataItem.BaseFromTradeGuangzhouDataId
  232. dataItem.BaseFromTradeGuangzhouIndexId = indexId
  233. dataItem.IndexCode = handleIndexCode
  234. dataItem.DataTime = dateStr
  235. dataItem.Value = getValue
  236. dataItem.ModifyTime = time.Now()
  237. updateDataList = append(updateDataList, dataItem)
  238. }
  239. }
  240. }
  241. }
  242. if len(addDataList) > 0 {
  243. err = gzObj.AddBaseFromTradeGuangzhouData(addDataList)
  244. if err != nil {
  245. fmt.Println("AddBaseFromTradeGuangzhouData Err:" + err.Error())
  246. return
  247. }
  248. }
  249. if len(updateDataList) > 0 {
  250. for _, v := range updateDataList {
  251. updateParams := make(map[string]interface{})
  252. whereParam := make(map[string]interface{})
  253. updateParams["value"] = v.Value
  254. updateParams["modify_time"] = time.Now()
  255. whereParam["base_from_trade_guangzhou_data_id"] = v.BaseFromTradeGuangzhouDataId
  256. err = gzObj.UpdateGuangzhouData(updateParams, whereParam)
  257. if err != nil {
  258. fmt.Println("UpdateGuangzhouData Err:" + err.Error())
  259. return
  260. }
  261. }
  262. }
  263. }
  264. // SyncRankingFromGuangzhou 广州交易所期货-合约
  265. func CrawlerFromGuangzhouContract(date time.Time) {
  266. // 准备: HTTP请求
  267. now := date //time.Now()
  268. if now.Weekday() == time.Saturday || now.Weekday() == time.Sunday {
  269. return
  270. }
  271. dateStr := now.Format(utils.FormatDateUnSpace)
  272. dateFormatStr := now.Format(utils.FormatDate)
  273. gzObj := new(models.BaseFromTradeGuangzhouIndex)
  274. parentId := 2
  275. classifyList, err := gzObj.GetBaseFromTradeGuangzhouClassifyAll(parentId)
  276. if err != nil {
  277. fmt.Println("GetBaseFromTradeGuangzhouClassify Err:" + err.Error())
  278. return
  279. }
  280. contractList, err := gzObj.GetBaseFromTradeGuangzhouContract("")
  281. if err != nil {
  282. fmt.Println("GetBaseFromTradeGuangzhouContract Err:" + err.Error())
  283. return
  284. }
  285. contractMap := make(map[string]*models.BaseFromTradeGuangzhouContract)
  286. for _, v := range contractList {
  287. contractMap[v.Contract+v.TradeDate] = v
  288. }
  289. for _, cv := range classifyList {
  290. if cv.ClassifyCode != "" {
  291. rn := utils.RangeRand(1, 30)
  292. fmt.Println(cv.ClassifyCode+";sleep:", strconv.Itoa(int(rn)))
  293. time.Sleep(time.Duration(rn) * time.Second)
  294. zzUrl := `http://www.gfex.com.cn/u/interfacesWebTiMemberDealPosiQuotes/loadListContract_id`
  295. zzUrl = fmt.Sprintf(zzUrl)
  296. fmt.Println(zzUrl)
  297. params := "variety=" + cv.ClassifyCode + "&trade_date=" + dateStr
  298. headerParams := make(map[string]string)
  299. headerParams["User-Agent"] = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36"
  300. headerParams["Content-Type"] = "application/x-www-form-urlencoded;charset=utf-8"
  301. body, err := utils.HttpPost(zzUrl, params, headerParams)
  302. if err != nil {
  303. fmt.Println("err:", err)
  304. return
  305. }
  306. fmt.Println("body :" + string(body))
  307. utils.FileLog.Info("body:" + string(body))
  308. contractItem := new(models.GuangzhouContract)
  309. err = json.Unmarshal(body, &contractItem)
  310. if err != nil {
  311. fmt.Println("CrawlerFromGuangzhouContract json.Unmarshal err:", err)
  312. return
  313. }
  314. if contractItem.Code == "0" {
  315. for _, dv := range contractItem.Data {
  316. if _, ok := contractMap[dv+dateFormatStr]; !ok {
  317. contractObj := new(models.BaseFromTradeGuangzhouContract)
  318. contractObj.BaseFromTradeGuangzhouClassifyId = cv.BaseFromTradeGuangzhouClassifyId
  319. contractObj.ClassifyCode = cv.ClassifyCode
  320. contractObj.Contract = dv
  321. contractObj.TradeDate = dateFormatStr
  322. err = gzObj.AddBaseFromTradeGuangzhouContract(contractObj)
  323. if err != nil {
  324. fmt.Println("AddBaseFromTradeGuangzhouContract err:", err)
  325. return
  326. }
  327. }
  328. }
  329. } else {
  330. utils.FileLog.Info("CrawlerFromGuangzhouContract Fail:" + string(body))
  331. }
  332. }
  333. }
  334. }
  335. // CrawlerFromGuangzhouDealPosiQuotes 广州交易所期货-日成交持仓排名
  336. func CrawlerFromGuangzhouDealPosiQuotes(date time.Time) {
  337. // 准备: HTTP请求
  338. now := date //time.Now()
  339. if now.Weekday() == time.Saturday || now.Weekday() == time.Sunday {
  340. return
  341. }
  342. dateStr := now.Format(utils.FormatDateUnSpace)
  343. dateStrFormat := now.Format(utils.FormatDate)
  344. gzObj := new(models.BaseFromTradeGuangzhouIndex)
  345. //获取所有指标
  346. indexList, err := gzObj.GetBaseFromTradeGuangzhouIndexAll()
  347. if err != nil {
  348. fmt.Println("GetBaseFromTradeGuangzhouIndexAll Err:" + err.Error())
  349. return
  350. }
  351. indexMap := make(map[string]*models.BaseFromTradeGuangzhouIndex)
  352. for _, iv := range indexList {
  353. indexMap[iv.IndexCode] = iv
  354. }
  355. dataList, err := gzObj.GetBaseFromTradeGuangzhouDataByDate(dateStrFormat)
  356. dataMap := make(map[string]*models.BaseFromTradeGuangzhouData)
  357. for _, dv := range dataList {
  358. dataMap[dv.IndexCode] = dv
  359. }
  360. contractList, err := gzObj.GetBaseFromTradeGuangzhouContract(dateStrFormat)
  361. if err != nil {
  362. fmt.Println("GetBaseFromTradeGuangzhouContract Err:" + err.Error())
  363. return
  364. }
  365. if len(contractList) <= 0 {
  366. fmt.Println("contractList is 0")
  367. return
  368. }
  369. zzUrl := `http://www.gfex.com.cn/u/interfacesWebTiMemberDealPosiQuotes/loadList`
  370. pages := []int{1, 2, 3}
  371. unit := "手"
  372. for _, cv := range contractList {
  373. for _, pv := range pages {
  374. rn := utils.RangeRand(1, 30)
  375. fmt.Println(cv.Contract+";sleep:", strconv.Itoa(int(rn)))
  376. time.Sleep(time.Duration(rn) * time.Second)
  377. params := "trade_date=" + dateStr + "&trade_type=0&variety=" + cv.ClassifyCode + "&contract_id=" + cv.Contract + "&data_type=" + strconv.Itoa(pv)
  378. //params = "trade_date=20231110&trade_type=0&variety=si&contract_id=si2312&data_type=1"
  379. //params = "trade_date=20231110&trade_type=0&variety=si&contract_id=si2402&data_type=1"
  380. fmt.Println("params:", params)
  381. headerParams := make(map[string]string)
  382. headerParams["User-Agent"] = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36"
  383. headerParams["Content-Type"] = "application/x-www-form-urlencoded;charset=utf-8"
  384. body, err := utils.HttpPost(zzUrl, params, headerParams)
  385. if err != nil {
  386. fmt.Println("err:", err)
  387. return
  388. }
  389. fmt.Println("body :" + string(body))
  390. utils.FileLog.Info("body:" + string(body))
  391. dealPosiQuotes := new(models.GuangzhouDealPosiQuotes)
  392. err = json.Unmarshal(body, dealPosiQuotes)
  393. if err != nil {
  394. fmt.Println("CrawlerFromGuangzhouDealPosiQuotes json.Unmarshal Err:" + err.Error())
  395. return
  396. }
  397. if dealPosiQuotes.Code == "0" {
  398. addDataList := make([]*models.BaseFromTradeGuangzhouData, 0)
  399. updateDataList := make([]*models.BaseFromTradeGuangzhouData, 0)
  400. for _, v := range dealPosiQuotes.Data {
  401. if v.Abbr == "总计" {
  402. continue
  403. }
  404. pinyinArg := pinyin.NewArgs()
  405. pinyinArr := pinyin.LazyPinyin(v.Abbr, pinyinArg)
  406. indexCode := strings.Join(pinyinArr, "")
  407. if indexCode == "" {
  408. continue
  409. }
  410. var indexName string
  411. switch pv {
  412. case 1:
  413. indexName = v.Abbr + "成交量"
  414. indexCode = indexCode + "deal"
  415. case 2:
  416. indexName = v.Abbr + "持买单量"
  417. indexCode = indexCode + "buy"
  418. case 3:
  419. indexName = v.Abbr + "持卖单量"
  420. indexCode = indexCode + "sold"
  421. }
  422. var indexId int
  423. if indexItem, ok := indexMap[indexCode]; !ok {
  424. gzObj = &models.BaseFromTradeGuangzhouIndex{
  425. BaseFromTradeGuangzhouClassifyId: cv.BaseFromTradeGuangzhouClassifyId,
  426. IndexCode: indexCode,
  427. IndexName: indexName,
  428. Frequency: "日度",
  429. Unit: unit,
  430. CreateTime: time.Now(),
  431. ModifyTime: time.Now(),
  432. }
  433. lastId, err := gzObj.AddBaseFromTradeGuangzhouIndex()
  434. if err != nil {
  435. fmt.Println("AddBaseFromTradeGuangzhouIndex Err:" + err.Error())
  436. return
  437. }
  438. indexId = int(lastId)
  439. } else {
  440. indexId = indexItem.BaseFromTradeGuangzhouIndexId
  441. }
  442. //判断数据是否存在
  443. if dataItem, ok := dataMap[indexCode]; !ok {
  444. dataItem = new(models.BaseFromTradeGuangzhouData)
  445. dataItem.BaseFromTradeGuangzhouIndexId = indexId
  446. dataItem.IndexCode = indexCode
  447. dataItem.DataTime = dateStr
  448. dataItem.Value = float64(v.TodayQty)
  449. dataItem.QtySub = float64(v.QtySub)
  450. dataItem.CreateTime = time.Now()
  451. dataItem.ModifyTime = time.Now()
  452. addDataList = append(addDataList, dataItem)
  453. } else {
  454. if dataItem.Value != float64(v.TodayQty) {
  455. dataItem.BaseFromTradeGuangzhouDataId = dataItem.BaseFromTradeGuangzhouDataId
  456. dataItem.BaseFromTradeGuangzhouIndexId = indexId
  457. dataItem.IndexCode = indexCode
  458. dataItem.DataTime = dateStr
  459. dataItem.Value = float64(v.TodayQty)
  460. dataItem.QtySub = float64(v.QtySub)
  461. dataItem.ModifyTime = time.Now()
  462. updateDataList = append(updateDataList, dataItem)
  463. }
  464. }
  465. }
  466. fmt.Println("addDataList:", len(addDataList))
  467. if len(addDataList) > 0 {
  468. err = gzObj.AddBaseFromTradeGuangzhouData(addDataList)
  469. if err != nil {
  470. fmt.Println("AddBaseFromTradeGuangzhouData Err:" + err.Error())
  471. return
  472. }
  473. }
  474. if len(updateDataList) > 0 {
  475. for _, v := range updateDataList {
  476. updateParams := make(map[string]interface{})
  477. whereParam := make(map[string]interface{})
  478. updateParams["value"] = v.Value
  479. updateParams["qty_sub"] = v.QtySub
  480. updateParams["modify_time"] = time.Now()
  481. whereParam["base_from_trade_guangzhou_data_id"] = v.BaseFromTradeGuangzhouDataId
  482. err = gzObj.UpdateGuangzhouData(updateParams, whereParam)
  483. if err != nil {
  484. fmt.Println("UpdateGuangzhouData Err:" + err.Error())
  485. return
  486. }
  487. }
  488. }
  489. }
  490. }
  491. }
  492. }
  493. // CrawlerFromGuangzhouWbillWeeklyQuotes 广州交易所期货-仓单日报
  494. func CrawlerFromGuangzhouWbillWeeklyQuotes(date time.Time) {
  495. now := date //time.Now()
  496. if now.Weekday() == time.Saturday || now.Weekday() == time.Sunday {
  497. return
  498. }
  499. dateStr := now.Format(utils.FormatDateUnSpace)
  500. dateStrFormat := now.Format(utils.FormatDate)
  501. gzObj := new(models.BaseFromTradeGuangzhouIndex)
  502. parentId := 3
  503. classifyList, err := gzObj.GetBaseFromTradeGuangzhouClassifyAll(parentId)
  504. if err != nil {
  505. fmt.Println("GetBaseFromTradeGuangzhouClassify Err:" + err.Error())
  506. return
  507. }
  508. classifyMap := make(map[string]*models.BaseFromTradeGuangzhouClassify)
  509. for _, cv := range classifyList {
  510. if cv.ClassifyCode != "" {
  511. classifyMap[cv.ClassifyCode] = cv
  512. }
  513. }
  514. //获取所有指标
  515. indexList, err := gzObj.GetBaseFromTradeGuangzhouIndexAll()
  516. if err != nil {
  517. fmt.Println("GetBaseFromTradeGuangzhouIndexAll Err:" + err.Error())
  518. return
  519. }
  520. indexMap := make(map[string]*models.BaseFromTradeGuangzhouIndex)
  521. for _, iv := range indexList {
  522. indexMap[iv.IndexCode] = iv
  523. }
  524. dataList, err := gzObj.GetBaseFromTradeGuangzhouDataByDate(dateStrFormat)
  525. dataMap := make(map[string]*models.BaseFromTradeGuangzhouData)
  526. for _, dv := range dataList {
  527. dataMap[dv.IndexCode] = dv
  528. }
  529. zzUrl := `http://www.gfex.com.cn/u/interfacesWebTdWbillWeeklyQuotes/loadList`
  530. params := "gen_date=" + dateStr
  531. fmt.Println("params:", params)
  532. headerParams := make(map[string]string)
  533. headerParams["User-Agent"] = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36"
  534. headerParams["Content-Type"] = "application/x-www-form-urlencoded;charset=utf-8"
  535. body, err := utils.HttpPost(zzUrl, params, headerParams)
  536. if err != nil {
  537. fmt.Println("err:", err)
  538. return
  539. }
  540. fmt.Println("body :" + string(body))
  541. utils.FileLog.Info("body:" + string(body))
  542. wbillQty := new(models.GuangzhouWbillQty)
  543. err = json.Unmarshal(body, wbillQty)
  544. if err != nil {
  545. fmt.Println("CrawlerFromGuangzhouWbillWeeklyQuotes json.Unmarshal Err:" + err.Error())
  546. return
  547. }
  548. if wbillQty.Code == "0" {
  549. addDataList := make([]*models.BaseFromTradeGuangzhouData, 0)
  550. updateDataList := make([]*models.BaseFromTradeGuangzhouData, 0)
  551. for _, v := range wbillQty.Data {
  552. if v.WhAbbr == "总计" {
  553. continue
  554. }
  555. pinyinArg := pinyin.NewArgs()
  556. pinyinArr := pinyin.LazyPinyin(v.WhAbbr, pinyinArg)
  557. indexCode := strings.Join(pinyinArr, "")
  558. if indexCode == "" {
  559. continue
  560. }
  561. indexCode += "wbill"
  562. indexName := v.WhAbbr + "仓单量"
  563. var indexId int
  564. if indexItem, ok := indexMap[indexCode]; !ok {
  565. var classifyId int
  566. if classifyItem, ok := classifyMap[v.VarietyOrder]; !ok {
  567. classifyItem = new(models.BaseFromTradeGuangzhouClassify)
  568. classifyItem.ClassifyCode = v.VarietyOrder
  569. classifyItem.ClassifyName = v.Variety
  570. classifyItem.ParentId = 3
  571. classifyItem.ModifyTime = time.Now()
  572. classifyItem.CreateTime = time.Now()
  573. classifyLastId, err := gzObj.AddBaseFromTradeGuangzhouClassify(classifyItem)
  574. if err != nil {
  575. fmt.Println("AddBaseFromTradeGuangzhouClassify err:" + err.Error())
  576. return
  577. }
  578. classifyId = int(classifyLastId)
  579. } else {
  580. classifyId = classifyItem.BaseFromTradeGuangzhouClassifyId
  581. }
  582. gzObj = &models.BaseFromTradeGuangzhouIndex{
  583. BaseFromTradeGuangzhouClassifyId: classifyId,
  584. IndexCode: indexCode,
  585. IndexName: indexName,
  586. Frequency: "日度",
  587. Unit: "手",
  588. CreateTime: time.Now(),
  589. ModifyTime: time.Now(),
  590. }
  591. lastId, err := gzObj.AddBaseFromTradeGuangzhouIndex()
  592. if err != nil {
  593. fmt.Println("AddBaseFromTradeGuangzhouIndex Err:" + err.Error())
  594. return
  595. }
  596. indexId = int(lastId)
  597. } else {
  598. indexId = indexItem.BaseFromTradeGuangzhouIndexId
  599. }
  600. //判断数据是否存在
  601. if dataItem, ok := dataMap[indexCode]; !ok {
  602. dataItem = new(models.BaseFromTradeGuangzhouData)
  603. dataItem.BaseFromTradeGuangzhouIndexId = indexId
  604. dataItem.IndexCode = indexCode
  605. dataItem.DataTime = dateStr
  606. dataItem.Value = float64(v.WbillQty)
  607. dataItem.QtySub = float64(v.RegWbillQty)
  608. dataItem.CreateTime = time.Now()
  609. dataItem.ModifyTime = time.Now()
  610. addDataList = append(addDataList, dataItem)
  611. } else {
  612. if dataItem.Value != float64(v.WbillQty) {
  613. dataItem.BaseFromTradeGuangzhouDataId = dataItem.BaseFromTradeGuangzhouDataId
  614. dataItem.BaseFromTradeGuangzhouIndexId = indexId
  615. dataItem.IndexCode = indexCode
  616. dataItem.DataTime = dateStr
  617. dataItem.Value = float64(v.WbillQty)
  618. dataItem.QtySub = float64(v.RegWbillQty)
  619. dataItem.ModifyTime = time.Now()
  620. updateDataList = append(updateDataList, dataItem)
  621. }
  622. }
  623. }
  624. fmt.Println("addDataList:", len(addDataList))
  625. if len(addDataList) > 0 {
  626. err = gzObj.AddBaseFromTradeGuangzhouData(addDataList)
  627. if err != nil {
  628. fmt.Println("AddBaseFromTradeGuangzhouData Err:" + err.Error())
  629. return
  630. }
  631. }
  632. if len(updateDataList) > 0 {
  633. for _, v := range updateDataList {
  634. updateParams := make(map[string]interface{})
  635. whereParam := make(map[string]interface{})
  636. updateParams["value"] = v.Value
  637. updateParams["qty_sub"] = v.QtySub
  638. updateParams["modify_time"] = time.Now()
  639. whereParam["base_from_trade_guangzhou_data_id"] = v.BaseFromTradeGuangzhouDataId
  640. err = gzObj.UpdateGuangzhouData(updateParams, whereParam)
  641. if err != nil {
  642. fmt.Println("UpdateGuangzhouData Err:" + err.Error())
  643. return
  644. }
  645. }
  646. }
  647. }
  648. }
  649. // CrawlerFromGuangzhouCongregateDeliveryQuotes 广州交易所期货-一次性交割卖方仓单查询
  650. func CrawlerFromGuangzhouCongregateDeliveryQuotes(date time.Time) {
  651. now := date //time.Now()
  652. dateStr := now.Format(utils.FormatMonthDateUnSpace)
  653. _, monthEndDay := utils.GetMonthStartAndEnd(strconv.Itoa(now.Year()), strconv.Itoa(int(now.Month())))
  654. gzObj := new(models.BaseFromTradeGuangzhouIndex)
  655. parentId := 4
  656. classifyList, err := gzObj.GetBaseFromTradeGuangzhouClassifyAll(parentId)
  657. if err != nil {
  658. fmt.Println("GetBaseFromTradeGuangzhouClassify Err:" + err.Error())
  659. return
  660. }
  661. classifyMap := make(map[string]*models.BaseFromTradeGuangzhouClassify)
  662. for _, cv := range classifyList {
  663. if cv.ClassifyCode != "" {
  664. classifyMap[cv.ClassifyCode] = cv
  665. }
  666. }
  667. //获取所有指标
  668. indexList, err := gzObj.GetBaseFromTradeGuangzhouIndexAll()
  669. if err != nil {
  670. fmt.Println("GetBaseFromTradeGuangzhouIndexAll Err:" + err.Error())
  671. return
  672. }
  673. indexMap := make(map[string]*models.BaseFromTradeGuangzhouIndex)
  674. for _, iv := range indexList {
  675. indexMap[iv.IndexCode] = iv
  676. }
  677. dataList, err := gzObj.GetBaseFromTradeGuangzhouDataByDate(monthEndDay)
  678. dataMap := make(map[string]*models.BaseFromTradeGuangzhouData)
  679. for _, dv := range dataList {
  680. dataMap[dv.IndexCode] = dv
  681. }
  682. zzUrl := `http://www.gfex.com.cn/u/interfacesWebTcCongregateDeliveryQuotes/loadList`
  683. params := "contract_month=" + dateStr
  684. fmt.Println("params:", params)
  685. headerParams := make(map[string]string)
  686. headerParams["User-Agent"] = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36"
  687. headerParams["Content-Type"] = "application/x-www-form-urlencoded;charset=utf-8"
  688. body, err := utils.HttpPost(zzUrl, params, headerParams)
  689. if err != nil {
  690. fmt.Println("err:", err)
  691. return
  692. }
  693. fmt.Println("body :" + string(body))
  694. utils.FileLog.Info("body:" + string(body))
  695. deliveryQuotes := new(models.GuangzhouCongregateDeliveryQuotes)
  696. err = json.Unmarshal(body, deliveryQuotes)
  697. if err != nil {
  698. fmt.Println("CrawlerFromGuangzhouCongregateDeliveryQuotes json.Unmarshal Err:" + err.Error())
  699. return
  700. }
  701. if deliveryQuotes.Code == "0" {
  702. addDataList := make([]*models.BaseFromTradeGuangzhouData, 0)
  703. updateDataList := make([]*models.BaseFromTradeGuangzhouData, 0)
  704. for _, v := range deliveryQuotes.Data {
  705. pinyinArg := pinyin.NewArgs()
  706. pinyinArr := pinyin.LazyPinyin(v.WhName, pinyinArg)
  707. indexCode := strings.Join(pinyinArr, "")
  708. if indexCode == "" {
  709. continue
  710. }
  711. indexCode += "wbillseller"
  712. indexName := v.WhName + "一次性交割卖方仓单量"
  713. var indexId int
  714. if indexItem, ok := indexMap[indexCode]; !ok {
  715. var classifyId int
  716. if classifyItem, ok := classifyMap[v.VarietyId]; !ok {
  717. classifyItem = new(models.BaseFromTradeGuangzhouClassify)
  718. classifyItem.ClassifyCode = v.VarietyId
  719. classifyItem.ClassifyName = v.VarietyName
  720. classifyItem.ParentId = 3
  721. classifyItem.ModifyTime = time.Now()
  722. classifyItem.CreateTime = time.Now()
  723. classifyLastId, err := gzObj.AddBaseFromTradeGuangzhouClassify(classifyItem)
  724. if err != nil {
  725. fmt.Println("AddBaseFromTradeGuangzhouClassify err:" + err.Error())
  726. return
  727. }
  728. classifyId = int(classifyLastId)
  729. } else {
  730. classifyId = classifyItem.BaseFromTradeGuangzhouClassifyId
  731. }
  732. gzObj = &models.BaseFromTradeGuangzhouIndex{
  733. BaseFromTradeGuangzhouClassifyId: classifyId,
  734. IndexCode: indexCode,
  735. IndexName: indexName,
  736. Frequency: "月度",
  737. Unit: "手",
  738. CreateTime: time.Now(),
  739. ModifyTime: time.Now(),
  740. }
  741. lastId, err := gzObj.AddBaseFromTradeGuangzhouIndex()
  742. if err != nil {
  743. fmt.Println("AddBaseFromTradeGuangzhouIndex Err:" + err.Error())
  744. return
  745. }
  746. indexId = int(lastId)
  747. } else {
  748. indexId = indexItem.BaseFromTradeGuangzhouIndexId
  749. }
  750. //判断数据是否存在
  751. if dataItem, ok := dataMap[indexCode]; !ok {
  752. dataItem = new(models.BaseFromTradeGuangzhouData)
  753. dataItem.BaseFromTradeGuangzhouIndexId = indexId
  754. dataItem.IndexCode = indexCode
  755. dataItem.DataTime = monthEndDay
  756. dataItem.Value = float64(v.WbillNum)
  757. dataItem.CreateTime = time.Now()
  758. dataItem.ModifyTime = time.Now()
  759. addDataList = append(addDataList, dataItem)
  760. } else {
  761. if dataItem.Value != float64(v.WbillNum) {
  762. dataItem.BaseFromTradeGuangzhouDataId = dataItem.BaseFromTradeGuangzhouDataId
  763. dataItem.BaseFromTradeGuangzhouIndexId = indexId
  764. dataItem.IndexCode = indexCode
  765. dataItem.DataTime = monthEndDay
  766. dataItem.Value = float64(v.WbillNum)
  767. dataItem.ModifyTime = time.Now()
  768. updateDataList = append(updateDataList, dataItem)
  769. }
  770. }
  771. }
  772. fmt.Println("addDataList:", len(addDataList))
  773. if len(addDataList) > 0 {
  774. err = gzObj.AddBaseFromTradeGuangzhouData(addDataList)
  775. if err != nil {
  776. fmt.Println("AddBaseFromTradeGuangzhouData Err:" + err.Error())
  777. return
  778. }
  779. }
  780. if len(updateDataList) > 0 {
  781. for _, v := range updateDataList {
  782. updateParams := make(map[string]interface{})
  783. whereParam := make(map[string]interface{})
  784. updateParams["value"] = v.Value
  785. updateParams["qty_sub"] = v.QtySub
  786. updateParams["modify_time"] = time.Now()
  787. whereParam["base_from_trade_guangzhou_data_id"] = v.BaseFromTradeGuangzhouDataId
  788. err = gzObj.UpdateGuangzhouData(updateParams, whereParam)
  789. if err != nil {
  790. fmt.Println("UpdateGuangzhouData Err:" + err.Error())
  791. return
  792. }
  793. }
  794. }
  795. }
  796. }
  797. //http://www.gfex.com.cn/u/interfacesWebTiMemberDealPosiQuotes/loadList