smm_shanghai.go 14 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560
  1. package services
  2. import (
  3. "context"
  4. "encoding/json"
  5. "eta/eta_crawler/models"
  6. "eta/eta_crawler/services/sci99"
  7. "eta/eta_crawler/utils"
  8. "fmt"
  9. "io"
  10. "net/http"
  11. "strconv"
  12. "strings"
  13. "time"
  14. "github.com/xuri/excelize/v2"
  15. )
  16. type SmmRequest struct {
  17. Name string
  18. IndexCode string
  19. Url string
  20. Frequency string
  21. Unit string
  22. Value string
  23. }
  24. var nickelReqs = []SmmRequest{
  25. // {
  26. // Name: "SMM 1#电解镍",
  27. // IndexCode: "",
  28. // Url: "https://hq.smm.cn/ajax/spot/history/201102250239",
  29. // Frequency: "日度",
  30. // Unit: "元/吨",
  31. // },
  32. {
  33. Name: "1#金川镍",
  34. Url: "https://hq.smm.cn/ajax/spot/history/201102250174",
  35. Frequency: "日度",
  36. Unit: "元/吨",
  37. },
  38. {
  39. Name: "1#进口镍",
  40. Url: "https://hq.smm.cn/ajax/spot/history/201102250423",
  41. Frequency: "日度",
  42. Unit: "元/吨",
  43. },
  44. {
  45. Name: "镍豆",
  46. Url: "https://hq.smm.cn/ajax/spot/history/202008270001",
  47. Frequency: "日度",
  48. Unit: "元/吨",
  49. },
  50. // {
  51. // Name: `印尼内贸红土镍矿1.2%(到厂价)`,
  52. // Url: "https://hq.smm.cn/ajax/spot/history/202311230010",
  53. // Frequency: "周度",
  54. // Unit: "美元/湿吨",
  55. // },
  56. // {
  57. // Name: `印尼内贸红土镍矿1.6%(到厂价)`,
  58. // Url: "https://hq.smm.cn/ajax/spot/history/202311230011",
  59. // Frequency: "周度",
  60. // Unit: "美元/湿吨",
  61. // },
  62. {
  63. Name: `菲律宾红土镍矿0.9%,Al≥7%(CIF)`,
  64. Url: "https://hq.smm.cn/ajax/spot/history/201608170001",
  65. Frequency: "日度",
  66. Unit: "美元/湿吨",
  67. },
  68. // {
  69. // Name: `菲律宾红土镍矿0.9%,Al<7%(CIF) `, // 重名
  70. // Url: "https://hq.smm.cn/ajax/spot/history/202109140006",
  71. // Frequency: "日度",
  72. // Unit: "美元/湿吨",
  73. // },
  74. // {
  75. // Name: `菲律宾红土镍矿1.3%(CIF)`,
  76. // Url: "https://hq.smm.cn/ajax/spot/history/202109140001",
  77. // Frequency: "日度",
  78. // Unit: "美元/湿吨",
  79. // },
  80. // {
  81. // Name: `菲律宾红土镍矿1.4%(CIF)`,
  82. // Url: "https://hq.smm.cn/ajax/spot/history/202109140003",
  83. // Frequency: "日度",
  84. // Unit: "美元/湿吨",
  85. // },
  86. {
  87. Name: `菲律宾红土镍矿1.5%(CIF)`,
  88. Url: "https://hq.smm.cn/ajax/spot/history/201608170002",
  89. Frequency: "日度",
  90. Unit: "美元/湿吨",
  91. },
  92. {
  93. Name: `红土镍矿1.8%(CIF)`,
  94. Url: "https://hq.smm.cn/ajax/spot/history/201608170003",
  95. Frequency: "日度",
  96. Unit: "美元/湿吨",
  97. },
  98. {
  99. Name: `菲律宾红土镍矿0.9%,Al≥7%(FOB)`,
  100. Url: "https://hq.smm.cn/ajax/spot/history/201509220001",
  101. Frequency: "日度",
  102. Unit: "美元/湿吨",
  103. },
  104. // {
  105. // Name: `菲律宾红土镍矿0.9%,Al<7%(FOB)`, //重名
  106. // Url: "https://hq.smm.cn/ajax/spot/history/202109140005",
  107. // Frequency: "日度",
  108. // Unit: "美元/湿吨",
  109. // },
  110. // {
  111. // Name: `菲律宾红土镍矿1.3%(FOB)`,
  112. // Url: "https://hq.smm.cn/ajax/spot/history/202109140002",
  113. // Frequency: "日度",
  114. // Unit: "美元/湿吨",
  115. // },
  116. // {
  117. // Name: `菲律宾红土镍矿1.4%(FOB)`,
  118. // Url: "https://hq.smm.cn/ajax/spot/history/202109140004",
  119. // Frequency: "日度",
  120. // Unit: "美元/湿吨",
  121. // },
  122. {
  123. Name: `菲律宾红土镍矿1.5%(FOB)`,
  124. Url: "https://hq.smm.cn/ajax/spot/history/201509220002",
  125. Frequency: "日度",
  126. Unit: "美元/湿吨",
  127. },
  128. {
  129. Name: `红土镍矿1.8%(FOB)`,
  130. Url: "https://hq.smm.cn/ajax/spot/history/201509220003",
  131. Frequency: "日度",
  132. Unit: "美元/湿吨",
  133. },
  134. {
  135. Name: `8-12%高镍生铁(出厂价)`,
  136. Url: "https://hq.smm.cn/ajax/spot/history/201106150005",
  137. Frequency: "日度",
  138. Unit: "元/镍点",
  139. },
  140. {
  141. Name: `1.5-1.7%镍生铁(出厂价)`,
  142. Url: "https://hq.smm.cn/ajax/spot/history/201106150007",
  143. Frequency: "日度",
  144. Unit: "元/吨",
  145. },
  146. }
  147. var copperReqs = []SmmRequest{
  148. {
  149. Name: "SMM 1#电解铜",
  150. Url: "https://hq.smm.cn/ajax/spot/history/201102250376",
  151. Unit: "元/吨",
  152. Frequency: "日度",
  153. },
  154. {
  155. Name: "SMM 1#电解铜升贴水最大值",
  156. Url: "https://hq.smm.cn/ajax/premium/history/201102250185",
  157. Value: "highs",
  158. Frequency: "日度",
  159. },
  160. {
  161. Name: "SMM 1#电解铜升贴水最小值",
  162. Url: "https://hq.smm.cn/ajax/premium/history/201102250185",
  163. Value: "low",
  164. Frequency: "日度",
  165. },
  166. // {
  167. // Name: "SMM 广东1#电解铜",
  168. // Url: "https://hq.smm.cn/ajax/spot/history/201912300001",
  169. // Unit: "元/吨",
  170. // Frequency: "日度",
  171. // },
  172. {
  173. Name: "SMM 广东1#电解铜升贴水最大值",
  174. Url: "https://hq.smm.cn/ajax/premium/history/201912300002",
  175. Value: "highs",
  176. Frequency: "日度",
  177. },
  178. {
  179. Name: "SMM 广东1#电解铜升贴水最小值",
  180. Url: "https://hq.smm.cn/ajax/premium/history/201912300002",
  181. Value: "low",
  182. Frequency: "日度",
  183. },
  184. {
  185. Name: "进口铜精矿指数(周)",
  186. Url: "https://hq.smm.cn/ajax/spot/history/201910240001",
  187. Unit: "美元/吨",
  188. Frequency: "周度",
  189. },
  190. {
  191. Name: "废铜:广东:1#光亮铜线",
  192. Url: "https://hq.smm.cn/ajax/spot/history/201108090036",
  193. Unit: "元/吨",
  194. Frequency: "日度",
  195. },
  196. }
  197. var zincReqs = []SmmRequest{
  198. {
  199. Name: "SMM 0#锌锭",
  200. Url: "https://hq.smm.cn/ajax/spot/history/201102250173",
  201. Unit: "元/吨",
  202. Frequency: "日度",
  203. },
  204. {
  205. Name: "SMM 1#锌锭",
  206. Url: "https://hq.smm.cn/ajax/spot/history/201102250418",
  207. Unit: "元/吨",
  208. Frequency: "日度",
  209. },
  210. // {
  211. // Name: "SMM 0#锌锭溢价(广东)",
  212. // Url: "https://hq.smm.cn/ajax/spot/history/202010210003",
  213. // Unit: "元/吨",
  214. // Frequency: "日度",
  215. // },
  216. {
  217. Name: "SMM 0#锌锭(广东)",
  218. Url: "https://hq.smm.cn/ajax/spot/history/201102250231",
  219. Unit: "元/吨",
  220. Frequency: "日度",
  221. },
  222. // {
  223. // Name: "SMM 1#锌锭溢价(广东)",
  224. // Url: "https://hq.smm.cn/ajax/spot/history/202010210004",
  225. // Unit: "元/吨",
  226. // Frequency: "日度",
  227. // },
  228. {
  229. Name: "SMM 1#锌锭(广东)",
  230. Url: "https://hq.smm.cn/ajax/spot/history/201102250069",
  231. Unit: "元/吨",
  232. Frequency: "日度",
  233. },
  234. // {
  235. // Name: "SMM 0#锌锭溢价(天津)",
  236. // Url: "https://hq.smm.cn/ajax/spot/history/202010210001",
  237. // Unit: "元/吨",
  238. // Frequency: "日度",
  239. // },
  240. {
  241. Name: "SMM 1#锌锭(天津)",
  242. Url: "https://hq.smm.cn/ajax/spot/history/201102250556",
  243. Unit: "元/吨",
  244. Frequency: "日度",
  245. },
  246. {
  247. Name: "SMM 0#锌锭(天津)",
  248. Url: "https://hq.smm.cn/ajax/spot/history/201102250399",
  249. Unit: "元/吨",
  250. Frequency: "日度",
  251. },
  252. // {
  253. // Name: "SMM 1#锌锭溢价(天津)",
  254. // Url: "https://hq.smm.cn/ajax/spot/history/202010210002",
  255. // Unit: "元/吨",
  256. // Frequency: "日度",
  257. // },
  258. // {
  259. // Name: "SMM 0#锌锭溢价(宁波)",
  260. // Url: "https://hq.smm.cn/ajax/spot/history/202010210005",
  261. // Unit: "元/吨",
  262. // Frequency: "日度",
  263. // },
  264. // {
  265. // Name: "SMM 0#锌锭(宁波)",
  266. // Url: "https://hq.smm.cn/ajax/spot/history/202004070006",
  267. // Unit: "元/吨",
  268. // Frequency: "日度",
  269. // },
  270. // {
  271. // Name: "Zn50国产TC(月)",
  272. // Url: "https://hq.smm.cn/ajax/spot/history/201312030008",
  273. // Unit: "元/金属吨",
  274. // Frequency: "月度",
  275. // },
  276. // {
  277. // Name: "Zn50进口TC(月)",
  278. // Url: "https://hq.smm.cn/ajax/spot/history/201312030009",
  279. // Unit: "美元/千吨",
  280. // Frequency: "月度",
  281. // },
  282. {
  283. Name: "Zn50国产TC(周)",
  284. Url: "https://hq.smm.cn/ajax/spot/history/202004070002",
  285. Unit: "元/金属吨",
  286. Frequency: "周度",
  287. },
  288. {
  289. Name: "Zn50进口TC(周)",
  290. Url: "https://hq.smm.cn/ajax/spot/history/202004070001",
  291. Unit: "美元/千吨",
  292. Frequency: "周度",
  293. },
  294. {
  295. Name: "Zn50内蒙古国产TC(周)",
  296. Url: "https://hq.smm.cn/ajax/spot/history/201504080006",
  297. Unit: "元/金属吨",
  298. Frequency: "周度",
  299. },
  300. {
  301. Name: "Zn50陕西国产TC(周)",
  302. Url: "https://hq.smm.cn/ajax/spot/history/201504080007",
  303. Unit: "元/金属吨",
  304. Frequency: "周度",
  305. },
  306. {
  307. Name: "Zn50甘肃国产TC(周)",
  308. Url: "https://hq.smm.cn/ajax/spot/history/202108050001",
  309. Unit: "元/金属吨",
  310. Frequency: "周度",
  311. },
  312. {
  313. Name: "Zn50四川国产TC(周)",
  314. Url: "https://hq.smm.cn/ajax/spot/history/201504080008",
  315. Unit: "元/金属吨",
  316. Frequency: "周度",
  317. },
  318. {
  319. Name: "Zn50广西国产TC(周)",
  320. Url: "https://hq.smm.cn/ajax/spot/history/201504080009",
  321. Unit: "元/金属吨",
  322. Frequency: "周度",
  323. },
  324. {
  325. Name: "Zn50云南国产TC(周)",
  326. Url: "https://hq.smm.cn/ajax/spot/history/201504080010",
  327. Unit: "元/金属吨",
  328. Frequency: "周度",
  329. },
  330. {
  331. Name: "Zn50湖南国产TC(周)",
  332. Url: "https://hq.smm.cn/ajax/spot/history/201504080011",
  333. Unit: "元/金属吨",
  334. Frequency: "周度",
  335. },
  336. }
  337. type SmmResponse struct {
  338. Code int
  339. Data []*SmmData
  340. Msg string
  341. }
  342. type SmmData struct {
  343. Highs float64 `json:"highs"`
  344. Low float64 `json:"low"`
  345. Average float64 `json:"average"`
  346. VchangeRate float64 `json:"vchange_rate"`
  347. LowShow string `json:"low_show"`
  348. HighShow string `json:"high_show"`
  349. AverageShow string `json:"average_show"`
  350. ProductId string `json:"product_id"`
  351. Vchange float64 `json:"vchange"`
  352. RenewDate string `json:"renew_date"`
  353. ChangeValueShow string `json:"change_value_show"`
  354. ChangeRateShow string `json:"change_rate_show"`
  355. }
  356. type EdbInfoData struct {
  357. ClassifyName string
  358. IndexName string
  359. IndexCode string
  360. Frequency string
  361. Unit string
  362. Value string
  363. LastDate time.Time
  364. OldDate time.Time
  365. Data map[string]float64
  366. SmmData
  367. }
  368. // 如果历史数据入库,就不再执行OldExcel
  369. var isSave bool
  370. func SyncShangHaiSmm(cont context.Context) (err error) {
  371. upMonth := time.Now().AddDate(0, -1, 0).Format(utils.FormatDate)
  372. curDate := time.Now().Format(utils.FormatDate)
  373. reqList := make([]SmmRequest, 0)
  374. reqList = append(reqList, zincReqs...)
  375. reqList = append(reqList, copperReqs...)
  376. reqList = append(reqList, nickelReqs...)
  377. edbInfoList := make([]*EdbInfoData, 0)
  378. for _, v := range reqList {
  379. url := fmt.Sprintf("%s/%s/%s", v.Url, upMonth, curDate)
  380. req, e := http.NewRequest("GET", url, nil)
  381. if e != nil {
  382. fmt.Println(e)
  383. return
  384. }
  385. // 发送请求
  386. client := &http.Client{}
  387. resp, e := client.Do(req)
  388. if e != nil {
  389. fmt.Println(e)
  390. return
  391. }
  392. var respData SmmResponse
  393. body, er := io.ReadAll(resp.Body)
  394. if er != nil {
  395. fmt.Println(er)
  396. return er
  397. }
  398. if err = json.Unmarshal(body, &respData); err != nil {
  399. fmt.Println(err)
  400. return
  401. }
  402. if respData.Code != 0 {
  403. fmt.Printf("上海有色爬虫请求失败, 指标名称:%s, 指标地址:%s", v.Name, v.Url)
  404. continue
  405. }
  406. data := respData.Data
  407. edbInfoList = append(edbInfoList, &EdbInfoData{
  408. IndexName: v.Name,
  409. Frequency: v.Frequency,
  410. Unit: v.Unit,
  411. Value: v.Value,
  412. SmmData: *data[len(data)-1],
  413. })
  414. }
  415. urlStr := `shanghai_smm/refresh/list`
  416. postUrl := utils.EDB_LIB_URL + urlStr
  417. postData, err := json.Marshal(edbInfoList)
  418. if err != nil {
  419. utils.FileLog.Info("Marshal Err:" + err.Error())
  420. return
  421. }
  422. result, err := sci99.HttpPost(postUrl, string(postData), "application/json")
  423. if err != nil {
  424. utils.FileLog.Info("HttpPost Err:" + err.Error())
  425. return
  426. }
  427. resp := new(models.BaseResponse)
  428. err = json.Unmarshal(result, &resp)
  429. if err != nil {
  430. fmt.Println(err)
  431. utils.FileLog.Info("Unmarshal resp Err:" + err.Error())
  432. return
  433. }
  434. if resp.Ret != 200 {
  435. fmt.Println("上海有色爬虫更新失败")
  436. return
  437. }
  438. if resp.Ret == 200 && utils.BusinessCode == utils.BusinessCodeJinRui {
  439. OldExcel()
  440. }
  441. return
  442. }
  443. // 解析读取历史excel数据,将历史数据转化为Smm结构体
  444. func ExcelToSmm(f *excelize.File, s string) (edbInfoList []*EdbInfoData) {
  445. cols, err := f.GetCols(s)
  446. if err != nil {
  447. fmt.Println(err)
  448. return
  449. }
  450. date := cols[0][4:]
  451. dateFormat := make([]string, 0)
  452. for _, v := range date {
  453. if v == "" {
  454. continue
  455. }
  456. var parseDate time.Time
  457. var err error
  458. if strings.Contains(s, "锌") {
  459. parseDate, err = time.Parse(utils.FormatDate, v)
  460. } else {
  461. parseDate, err = time.Parse("01-02-06", v)
  462. }
  463. if err != nil {
  464. fmt.Println(err)
  465. return
  466. }
  467. dateFormat = append(dateFormat, parseDate.Format(utils.FormatDate))
  468. }
  469. for i := 1; i < len(cols); i++ {
  470. tmpEdbInfo := new(EdbInfoData)
  471. tmpEdbInfo.IndexName = cols[i][0]
  472. tmpEdbInfo.IndexCode = cols[i][1]
  473. tmpEdbInfo.Frequency = cols[i][2]
  474. tmpEdbInfo.Unit = cols[i][3]
  475. tmpEdbInfo.Data = make(map[string]float64)
  476. tmpEdbInfo.OldDate = time.Now()
  477. for j := 4; j < len(cols[i]); j++ {
  478. floatVal, err := strconv.ParseFloat(cols[i][j], 64)
  479. if err != nil {
  480. fmt.Println("无效字符", cols[i][j])
  481. continue
  482. }
  483. tmpEdbInfo.Data[dateFormat[j-4]] = floatVal
  484. curTime, _ := time.Parse(utils.FormatDate, dateFormat[j-4])
  485. if curTime.After(tmpEdbInfo.LastDate) {
  486. tmpEdbInfo.LastDate = curTime
  487. }
  488. if curTime.Before(tmpEdbInfo.OldDate) {
  489. tmpEdbInfo.OldDate = curTime
  490. }
  491. }
  492. edbInfoList = append(edbInfoList, tmpEdbInfo)
  493. }
  494. return
  495. }
  496. func OldExcel() {
  497. excelPath := utils.OLD_EXCEL_PATH_JR
  498. if excelPath == "" || isSave {
  499. return
  500. }
  501. f, err := excelize.OpenFile(excelPath)
  502. if err != nil {
  503. fmt.Print(err)
  504. return
  505. }
  506. var edbClassify = []string{`镍`, `铜`, `锌`, `锌(日度)`}
  507. for _, v := range edbClassify {
  508. edbInfoList := ExcelToSmm(f, v)
  509. _, err := json.Marshal(edbInfoList)
  510. if err != nil {
  511. fmt.Println(err)
  512. return
  513. }
  514. fmt.Println("成功:", v)
  515. urlStr := `shanghai_smm/refresh/excel`
  516. postUrl := utils.EDB_LIB_URL + urlStr
  517. postData, err := json.Marshal(edbInfoList)
  518. if err != nil {
  519. utils.FileLog.Info("Marshal Err:" + err.Error())
  520. return
  521. }
  522. result, err := sci99.HttpPost(postUrl, string(postData), "application/json")
  523. if err != nil {
  524. utils.FileLog.Info("HttpPost Err:" + err.Error())
  525. return
  526. }
  527. resp := new(models.BaseResponse)
  528. err = json.Unmarshal(result, &resp)
  529. if err != nil {
  530. fmt.Println(err)
  531. utils.FileLog.Info("Unmarshal resp Err:" + err.Error())
  532. return
  533. }
  534. if resp.Ret != 200 {
  535. fmt.Println("上海有色网excel历史有色数据更新失败")
  536. utils.FileLog.Info("上海有色网excel历史有色数据更新失败")
  537. return
  538. }
  539. time.Sleep(10 * time.Second)
  540. }
  541. isSave = true
  542. }