niaosu_edb.go 46 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463146414651466146714681469147014711472147314741475147614771478147914801481148214831484148514861487148814891490149114921493149414951496149714981499150015011502150315041505150615071508150915101511151215131514151515161517151815191520152115221523152415251526152715281529153015311532153315341535153615371538
  1. package services
  2. import (
  3. "eta/eta_data_analysis/models"
  4. "eta/eta_data_analysis/utils"
  5. "fmt"
  6. "github.com/PuerkitoBio/goquery"
  7. "strings"
  8. "time"
  9. )
  10. var NiaoSuListMap = map[string]string {
  11. "中国尿素分原料产能利用率周数据统计" : "https://list.oilchem.net/188/43702/",
  12. "中国尿素样本生产理论利润周数据统计" : "https://list.oilchem.net/188/43705/",
  13. "中国尿素样本港口库存周数据统计" : "https://list.oilchem.net/188/43704/",
  14. "中国尿素产量周数据分析" : "https://list.oilchem.net/188/43708/",
  15. "中国尿素企业库存周数据分析" : "https://list.oilchem.net/188/43709/",
  16. "中国尿素样本港口库存周数据分析" : "https://list.oilchem.net/188/43709/",
  17. "中国尿素企业预收订单周数据分析" : "https://list.oilchem.net/188/43710/",
  18. "中国复合肥产能利用率周数据统计" : "https://list.oilchem.net/188/43716/",
  19. "中国三聚氰胺产能利用率周数据统计" : "https://list.oilchem.net/188/43717/",
  20. "中国尿素产量分省份月数据统计" : "https://list.oilchem.net/188/43719/",
  21. //"中国尿素进出口量数据总体分析" : "https://list.oilchem.net/188/5315/",
  22. }
  23. func NiaoSuList(num int) (err error) {
  24. for k, v := range NiaoSuListMap {
  25. for i := 1; i < num; i++ {
  26. listUrl := v + fmt.Sprintf("%d.html",i)
  27. fmt.Println("listUrl:",listUrl)
  28. htm, e := FetchPageHtml(listUrl)
  29. if e != nil {
  30. err = e
  31. utils.FileLog.Error(fmt.Sprintf("FetchPageHtml err:%v",err))
  32. fmt.Println("FetchPageHtml err",err)
  33. return
  34. }
  35. err = AnalysisOilchemList(htm, k)
  36. if err != nil {
  37. utils.FileLog.Error(fmt.Sprintf("AnalysisOilchemList err:%v",err))
  38. fmt.Println("AnalysisOilchemList err",err)
  39. return
  40. }
  41. time.Sleep(2*time.Second)
  42. }
  43. }
  44. return
  45. }
  46. // 中国尿素分原料产能利用率周数据统计
  47. func AnalysisOilchemNiaoSu1(htm []byte) (err error) {
  48. if len(htm) == 0 {
  49. utils.FileLog.Info("htm empty")
  50. return
  51. }
  52. doc, e := goquery.NewDocumentFromReader(strings.NewReader(string(htm)))
  53. if e != nil {
  54. err = fmt.Errorf("NewDocumentFromReader err: %v", e)
  55. return
  56. }
  57. titleList := make([]string, 0)
  58. unitList := make([]string, 0)
  59. doc.Find("p").Each(func(i int, selection *goquery.Selection) {
  60. ptext := selection.Text()
  61. if strings.Contains(ptext, "单位:") {
  62. unit := strings.Replace(ptext, "单位:", "", -1)
  63. //fmt.Println("unit:",unit)
  64. unitList = append(unitList, unit)
  65. }
  66. if strings.Contains(ptext, "中国尿素分原料产能利用率周数据统计") {
  67. title := ptext
  68. //fmt.Println("title:",title)
  69. titleList = append(titleList, title)
  70. }
  71. })
  72. area := ""
  73. title := doc.Find("h2").Text()
  74. fmt.Println("title:",title)
  75. createTimeStr := doc.Find("h2").Next().Text()
  76. createTimeStr = strings.TrimLeft(createTimeStr,"发布时间:")
  77. createTimeStrIndex := strings.Index(createTimeStr,"来源:")
  78. createTimeStr = createTimeStr[:createTimeStrIndex]
  79. createTimeStr = strings.TrimSpace(createTimeStr)
  80. createTime, err := time.Parse(utils.HlbFormatDateTimeNoSecond, createTimeStr)
  81. if err != nil {
  82. utils.FileLog.Error(fmt.Sprintf("time.Parse err:%v",err))
  83. return
  84. }
  85. //fmt.Println("createTime:",createTime)
  86. dataTime := createTime.Format(utils.FormatDate)
  87. //fmt.Println("dataTime:",dataTime)
  88. indexList := make([]*models.BaseFromOilchemIndex, 0)
  89. doc.Find("tbody").Each(func(tableIndex int, table *goquery.Selection) {
  90. table.Find("tr").First().Each(func(ii int, table2 *goquery.Selection) {
  91. table.Find("td").Each(func(jj int, table3 *goquery.Selection) {
  92. text3 := table3.Text()
  93. text3 = strings.Replace(text3,"\n","",-1)
  94. text3 = strings.Replace(text3," ","",-1)
  95. if text3 == "上周" || text3 == "环比" || text3 == "地区" {
  96. return
  97. }
  98. //fmt.Println("table3:",text3)
  99. //utils.FileLog.Info(fmt.Sprintf("table3:%s",text3))
  100. //fmt.Println("ii:",ii)
  101. //utils.FileLog.Info(fmt.Sprintf("ii:%d",ii))
  102. //fmt.Println("jj:",jj)
  103. //utils.FileLog.Info(fmt.Sprintf("jj:%d",jj))
  104. //fmt.Println("tableIndex:",tableIndex)
  105. //utils.FileLog.Info(fmt.Sprintf("tableIndex:%d",tableIndex))
  106. if utils.ContainsChinese(text3) && text3 != "本周" {
  107. area = text3
  108. return
  109. }
  110. if area == "" {
  111. return
  112. }
  113. value := text3
  114. value = strings.TrimRight(value, "%")
  115. //valueF, e := strconv.ParseFloat(value, 64)
  116. //if e != nil {
  117. // err = e
  118. // utils.FileLog.Error(fmt.Sprintf("strconv.ParseFloat err:%v",e))
  119. // return
  120. //}
  121. indexName := "中国尿素"+ area
  122. fmt.Println("indexName:",indexName)
  123. fmt.Println("valueF:",value)
  124. unit := ""
  125. if strings.Contains(area,"产量") {
  126. unit = "万吨"
  127. } else {
  128. unit = "%"
  129. }
  130. item := &models.BaseFromOilchemIndex{
  131. IndexName: indexName,
  132. ClassifyId: 2,
  133. Unit: unit,
  134. Frequency: "周度",
  135. Describe: "",
  136. DataTime: dataTime,
  137. Value: value,
  138. Sort: 0,
  139. CreateTime: time.Now(),
  140. ModifyTime: time.Now(),
  141. IndexNameStr: indexName,
  142. MarketName: area,
  143. }
  144. indexList = append(indexList,item)
  145. area = ""
  146. })
  147. })
  148. })
  149. // 写入数据库
  150. err = PostHandleOilchem(indexList)
  151. if err != nil {
  152. utils.FileLog.Error(fmt.Sprintf("PostHandleOilchem err:%v",err))
  153. fmt.Println("PostHandleOilchem err",err)
  154. return
  155. }
  156. return
  157. }
  158. // 中国尿素样本生产理论利润周数据统计
  159. func AnalysisOilchemNiaoSu2(htm []byte) (err error) {
  160. if len(htm) == 0 {
  161. utils.FileLog.Info("htm empty")
  162. return
  163. }
  164. doc, e := goquery.NewDocumentFromReader(strings.NewReader(string(htm)))
  165. if e != nil {
  166. err = fmt.Errorf("NewDocumentFromReader err: %v", e)
  167. return
  168. }
  169. //titleList := make([]string, 0)
  170. unitList := make([]string, 0)
  171. doc.Find("p").Each(func(i int, selection *goquery.Selection) {
  172. ptext := selection.Text()
  173. if strings.Contains(ptext, "单位:") {
  174. unit := strings.Replace(ptext, "单位:", "", -1)
  175. //fmt.Println("unit:",unit)
  176. unitList = append(unitList, unit)
  177. }
  178. //if strings.Contains(ptext, "中国尿素样本生产理论利润周数据统计") {
  179. // title := ptext
  180. // //fmt.Println("title:",title)
  181. // titleList = append(titleList, title)
  182. //}
  183. })
  184. area := ""
  185. title := doc.Find("h2").Text()
  186. fmt.Println("title:",title)
  187. createTimeStr := doc.Find("h2").Next().Text()
  188. createTimeStr = strings.TrimLeft(createTimeStr,"发布时间:")
  189. createTimeStrIndex := strings.Index(createTimeStr,"来源:")
  190. createTimeStr = createTimeStr[:createTimeStrIndex]
  191. createTimeStr = strings.TrimSpace(createTimeStr)
  192. createTime, err := time.Parse(utils.HlbFormatDateTimeNoSecond, createTimeStr)
  193. if err != nil {
  194. utils.FileLog.Error(fmt.Sprintf("time.Parse err:%v",err))
  195. return
  196. }
  197. //fmt.Println("createTime:",createTime)
  198. dataTime := createTime.Format(utils.FormatDate)
  199. //fmt.Println("dataTime:",dataTime)
  200. indexList := make([]*models.BaseFromOilchemIndex, 0)
  201. doc.Find("tbody").Each(func(tableIndex int, table *goquery.Selection) {
  202. table.Find("tr").First().Each(func(ii int, table2 *goquery.Selection) {
  203. table.Find("td").Each(func(jj int, table3 *goquery.Selection) {
  204. text3 := table3.Text()
  205. text3 = strings.Replace(text3,"\n","",-1)
  206. text3 = strings.Replace(text3," ","",-1)
  207. if text3 == "上周" || text3 == "环比" || text3 == "地区" {
  208. return
  209. }
  210. //fmt.Println("table3:",text3)
  211. //utils.FileLog.Info(fmt.Sprintf("table3:%s",text3))
  212. //fmt.Println("ii:",ii)
  213. //utils.FileLog.Info(fmt.Sprintf("ii:%d",ii))
  214. //fmt.Println("jj:",jj)
  215. //utils.FileLog.Info(fmt.Sprintf("jj:%d",jj))
  216. //fmt.Println("tableIndex:",tableIndex)
  217. //utils.FileLog.Info(fmt.Sprintf("tableIndex:%d",tableIndex))
  218. if utils.ContainsChinese(text3) && text3 != "本周" {
  219. area = text3
  220. return
  221. }
  222. if area == "" {
  223. return
  224. }
  225. value := text3
  226. value = strings.TrimRight(value, "%")
  227. //valueF, e := strconv.ParseFloat(value, 64)
  228. //if e != nil {
  229. // err = e
  230. // utils.FileLog.Error(fmt.Sprintf("strconv.ParseFloat err:%v",e))
  231. // return
  232. //}
  233. indexName := "中国尿素样本分工艺理论利润"+"("+ area+")"
  234. fmt.Println("indexName:",indexName)
  235. fmt.Println("valueF:",value)
  236. //unit := ""
  237. //if strings.Contains(area,"产量") {
  238. // unit = "万吨"
  239. //} else {
  240. // unit = "%"
  241. //}
  242. item := &models.BaseFromOilchemIndex{
  243. IndexName: indexName,
  244. ClassifyId: 2,
  245. Unit: "元/吨",
  246. Frequency: "周度",
  247. Describe: "",
  248. DataTime: dataTime,
  249. Value: value,
  250. Sort: 0,
  251. CreateTime: time.Now(),
  252. ModifyTime: time.Now(),
  253. IndexNameStr: indexName,
  254. MarketName: area,
  255. }
  256. indexList = append(indexList,item)
  257. area = ""
  258. })
  259. })
  260. })
  261. // 写入数据库
  262. err = PostHandleOilchem(indexList)
  263. if err != nil {
  264. utils.FileLog.Error(fmt.Sprintf("PostHandleOilchem err:%v",err))
  265. fmt.Println("PostHandleOilchem err",err)
  266. return
  267. }
  268. return
  269. }
  270. // 中国尿素样本港口库存周数据统计
  271. func AnalysisOilchemNiaoSu3(htm []byte) (err error) {
  272. if len(htm) == 0 {
  273. utils.FileLog.Info("htm empty")
  274. return
  275. }
  276. doc, e := goquery.NewDocumentFromReader(strings.NewReader(string(htm)))
  277. if e != nil {
  278. err = fmt.Errorf("NewDocumentFromReader err: %v", e)
  279. return
  280. }
  281. //titleList := make([]string, 0)
  282. unitList := make([]string, 0)
  283. doc.Find("p").Each(func(i int, selection *goquery.Selection) {
  284. ptext := selection.Text()
  285. if strings.Contains(ptext, "单位:") {
  286. unit := strings.Replace(ptext, "单位:", "", -1)
  287. //fmt.Println("unit:",unit)
  288. unitList = append(unitList, unit)
  289. }
  290. //if strings.Contains(ptext, "中国尿素样本生产理论利润周数据统计") {
  291. // title := ptext
  292. // //fmt.Println("title:",title)
  293. // titleList = append(titleList, title)
  294. //}
  295. })
  296. area := ""
  297. title := doc.Find("h2").Text()
  298. fmt.Println("title:",title)
  299. createTimeStr := doc.Find("h2").Next().Text()
  300. createTimeStr = strings.TrimLeft(createTimeStr,"发布时间:")
  301. createTimeStrIndex := strings.Index(createTimeStr,"来源:")
  302. createTimeStr = createTimeStr[:createTimeStrIndex]
  303. createTimeStr = strings.TrimSpace(createTimeStr)
  304. createTime, err := time.Parse(utils.HlbFormatDateTimeNoSecond, createTimeStr)
  305. if err != nil {
  306. utils.FileLog.Error(fmt.Sprintf("time.Parse err:%v",err))
  307. return
  308. }
  309. //fmt.Println("createTime:",createTime)
  310. dataTime := createTime.Format(utils.FormatDate)
  311. //fmt.Println("dataTime:",dataTime)
  312. indexList := make([]*models.BaseFromOilchemIndex, 0)
  313. doc.Find("tbody").Each(func(tableIndex int, table *goquery.Selection) {
  314. table.Find("tr").First().Each(func(ii int, table2 *goquery.Selection) {
  315. table.Find("td").Each(func(jj int, table3 *goquery.Selection) {
  316. text3 := table3.Text()
  317. text3 = strings.Replace(text3,"\n","",-1)
  318. text3 = strings.Replace(text3," ","",-1)
  319. if text3 == "上周" || text3 == "环比" || text3 == "地区" {
  320. return
  321. }
  322. //fmt.Println("table3:",text3)
  323. //utils.FileLog.Info(fmt.Sprintf("table3:%s",text3))
  324. //fmt.Println("ii:",ii)
  325. //utils.FileLog.Info(fmt.Sprintf("ii:%d",ii))
  326. //fmt.Println("jj:",jj)
  327. //utils.FileLog.Info(fmt.Sprintf("jj:%d",jj))
  328. //fmt.Println("tableIndex:",tableIndex)
  329. //utils.FileLog.Info(fmt.Sprintf("tableIndex:%d",tableIndex))
  330. if utils.ContainsChinese(text3) && text3 != "本周" {
  331. area = text3
  332. return
  333. }
  334. if area == "" {
  335. return
  336. }
  337. value := text3
  338. value = strings.TrimRight(value, "%")
  339. //valueF, e := strconv.ParseFloat(value, 64)
  340. //if e != nil {
  341. // err = e
  342. // utils.FileLog.Error(fmt.Sprintf("strconv.ParseFloat err:%v",e))
  343. // return
  344. //}
  345. indexName := "中国尿素样本港口库存"+"("+ area+")"
  346. fmt.Println("indexName:",indexName)
  347. fmt.Println("valueF:",value)
  348. //unit := ""
  349. //if strings.Contains(area,"产量") {
  350. // unit = "万吨"
  351. //} else {
  352. // unit = "%"
  353. //}
  354. item := &models.BaseFromOilchemIndex{
  355. IndexName: indexName,
  356. ClassifyId: 2,
  357. Unit: "万吨",
  358. Frequency: "周度",
  359. Describe: "",
  360. DataTime: dataTime,
  361. Value: value,
  362. Sort: 0,
  363. CreateTime: time.Now(),
  364. ModifyTime: time.Now(),
  365. IndexNameStr: indexName,
  366. MarketName: area,
  367. }
  368. indexList = append(indexList,item)
  369. area = ""
  370. })
  371. })
  372. })
  373. // 写入数据库
  374. err = PostHandleOilchem(indexList)
  375. if err != nil {
  376. utils.FileLog.Error(fmt.Sprintf("PostHandleOilchem err:%v",err))
  377. fmt.Println("PostHandleOilchem err",err)
  378. return
  379. }
  380. return
  381. }
  382. // 中国尿素产量周数据分析
  383. func AnalysisOilchemNiaoSu4(htm []byte) (err error) {
  384. if len(htm) == 0 {
  385. utils.FileLog.Info("htm empty")
  386. return
  387. }
  388. doc, e := goquery.NewDocumentFromReader(strings.NewReader(string(htm)))
  389. if e != nil {
  390. err = fmt.Errorf("NewDocumentFromReader err: %v", e)
  391. return
  392. }
  393. //titleList := make([]string, 0)
  394. unitList := make([]string, 0)
  395. doc.Find("p").Each(func(i int, selection *goquery.Selection) {
  396. ptext := selection.Text()
  397. if strings.Contains(ptext, "单位:") {
  398. unit := strings.Replace(ptext, "单位:", "", -1)
  399. //fmt.Println("unit:",unit)
  400. unitList = append(unitList, unit)
  401. }
  402. //if strings.Contains(ptext, "中国尿素样本生产理论利润周数据统计") {
  403. // title := ptext
  404. // //fmt.Println("title:",title)
  405. // titleList = append(titleList, title)
  406. //}
  407. })
  408. area := ""
  409. title := doc.Find("h2").Text()
  410. fmt.Println("title:",title)
  411. createTimeStr := doc.Find("h2").Next().Text()
  412. createTimeStr = strings.TrimLeft(createTimeStr,"发布时间:")
  413. createTimeStrIndex := strings.Index(createTimeStr,"来源:")
  414. createTimeStr = createTimeStr[:createTimeStrIndex]
  415. createTimeStr = strings.TrimSpace(createTimeStr)
  416. createTime, err := time.Parse(utils.HlbFormatDateTimeNoSecond, createTimeStr)
  417. if err != nil {
  418. utils.FileLog.Error(fmt.Sprintf("time.Parse err:%v",err))
  419. return
  420. }
  421. //fmt.Println("createTime:",createTime)
  422. dataTime := createTime.Format(utils.FormatDate)
  423. //fmt.Println("dataTime:",dataTime)
  424. indexList := make([]*models.BaseFromOilchemIndex, 0)
  425. doc.Find("tbody").Each(func(tableIndex int, table *goquery.Selection) {
  426. table.Find("tr").First().Each(func(ii int, table2 *goquery.Selection) {
  427. table.Find("td").Each(func(jj int, table3 *goquery.Selection) {
  428. text3 := table3.Text()
  429. text3 = strings.Replace(text3,"\n","",-1)
  430. text3 = strings.Replace(text3," ","",-1)
  431. if text3 == "上周" || text3 == "环比" || text3 == "地区" {
  432. return
  433. }
  434. //fmt.Println("table3:",text3)
  435. //utils.FileLog.Info(fmt.Sprintf("table3:%s",text3))
  436. //fmt.Println("ii:",ii)
  437. //utils.FileLog.Info(fmt.Sprintf("ii:%d",ii))
  438. //fmt.Println("jj:",jj)
  439. //utils.FileLog.Info(fmt.Sprintf("jj:%d",jj))
  440. //fmt.Println("tableIndex:",tableIndex)
  441. //utils.FileLog.Info(fmt.Sprintf("tableIndex:%d",tableIndex))
  442. if utils.ContainsChinese(text3) && text3 != "本周" {
  443. area = text3
  444. return
  445. }
  446. if area == "" {
  447. return
  448. }
  449. value := text3
  450. value = strings.TrimRight(value, "%")
  451. //valueF, e := strconv.ParseFloat(value, 64)
  452. //if e != nil {
  453. // err = e
  454. // utils.FileLog.Error(fmt.Sprintf("strconv.ParseFloat err:%v",e))
  455. // return
  456. //}
  457. indexName := "中国尿素产量"+"("+ area+")"
  458. fmt.Println("indexName:",indexName)
  459. fmt.Println("valueF:",value)
  460. //unit := ""
  461. //if strings.Contains(area,"产量") {
  462. // unit = "万吨"
  463. //} else {
  464. // unit = "%"
  465. //}
  466. item := &models.BaseFromOilchemIndex{
  467. IndexName: indexName,
  468. ClassifyId: 2,
  469. Unit: "万吨",
  470. Frequency: "周度",
  471. Describe: "",
  472. DataTime: dataTime,
  473. Value: value,
  474. Sort: 0,
  475. CreateTime: time.Now(),
  476. ModifyTime: time.Now(),
  477. IndexNameStr: indexName,
  478. MarketName: area,
  479. }
  480. indexList = append(indexList,item)
  481. area = ""
  482. })
  483. })
  484. })
  485. // 写入数据库
  486. err = PostHandleOilchem(indexList)
  487. if err != nil {
  488. utils.FileLog.Error(fmt.Sprintf("PostHandleOilchem err:%v",err))
  489. fmt.Println("PostHandleOilchem err",err)
  490. return
  491. }
  492. return
  493. }
  494. // 中国尿素企业库存
  495. func AnalysisOilchemNiaoSu5(htm []byte) (err error) {
  496. if len(htm) == 0 {
  497. utils.FileLog.Info("htm empty")
  498. return
  499. }
  500. doc, e := goquery.NewDocumentFromReader(strings.NewReader(string(htm)))
  501. if e != nil {
  502. err = fmt.Errorf("NewDocumentFromReader err: %v", e)
  503. return
  504. }
  505. //titleList := make([]string, 0)
  506. unitList := make([]string, 0)
  507. doc.Find("p").Each(func(i int, selection *goquery.Selection) {
  508. ptext := selection.Text()
  509. if strings.Contains(ptext, "单位:") {
  510. unit := strings.Replace(ptext, "单位:", "", -1)
  511. //fmt.Println("unit:",unit)
  512. unitList = append(unitList, unit)
  513. }
  514. //if strings.Contains(ptext, "中国尿素样本生产理论利润周数据统计") {
  515. // title := ptext
  516. // //fmt.Println("title:",title)
  517. // titleList = append(titleList, title)
  518. //}
  519. })
  520. area := ""
  521. title := doc.Find("h2").Text()
  522. fmt.Println("title:",title)
  523. createTimeStr := doc.Find("h2").Next().Text()
  524. createTimeStr = strings.TrimLeft(createTimeStr,"发布时间:")
  525. createTimeStrIndex := strings.Index(createTimeStr,"来源:")
  526. createTimeStr = createTimeStr[:createTimeStrIndex]
  527. createTimeStr = strings.TrimSpace(createTimeStr)
  528. createTime, err := time.Parse(utils.HlbFormatDateTimeNoSecond, createTimeStr)
  529. if err != nil {
  530. utils.FileLog.Error(fmt.Sprintf("time.Parse err:%v",err))
  531. return
  532. }
  533. //fmt.Println("createTime:",createTime)
  534. dataTime := createTime.Format(utils.FormatDate)
  535. //fmt.Println("dataTime:",dataTime)
  536. indexList := make([]*models.BaseFromOilchemIndex, 0)
  537. doc.Find("tbody").Each(func(tableIndex int, table *goquery.Selection) {
  538. table.Find("tr").First().Each(func(ii int, table2 *goquery.Selection) {
  539. table.Find("td").Each(func(jj int, table3 *goquery.Selection) {
  540. text3 := table3.Text()
  541. text3 = strings.Replace(text3,"\n","",-1)
  542. text3 = strings.Replace(text3," ","",-1)
  543. if text3 == "上周" || text3 == "环比" || text3 == "地区" {
  544. return
  545. }
  546. //fmt.Println("table3:",text3)
  547. //utils.FileLog.Info(fmt.Sprintf("table3:%s",text3))
  548. //fmt.Println("ii:",ii)
  549. //utils.FileLog.Info(fmt.Sprintf("ii:%d",ii))
  550. //fmt.Println("jj:",jj)
  551. //utils.FileLog.Info(fmt.Sprintf("jj:%d",jj))
  552. //fmt.Println("tableIndex:",tableIndex)
  553. //utils.FileLog.Info(fmt.Sprintf("tableIndex:%d",tableIndex))
  554. if utils.ContainsChinese(text3) && text3 != "本周" {
  555. area = text3
  556. return
  557. }
  558. if area == "" {
  559. return
  560. }
  561. value := text3
  562. value = strings.TrimRight(value, "%")
  563. //valueF, e := strconv.ParseFloat(value, 64)
  564. //if e != nil {
  565. // err = e
  566. // utils.FileLog.Error(fmt.Sprintf("strconv.ParseFloat err:%v",e))
  567. // return
  568. //}
  569. indexName := "中国尿素企业库存"
  570. fmt.Println("indexName:",indexName)
  571. fmt.Println("valueF:",value)
  572. //unit := ""
  573. //if strings.Contains(area,"产量") {
  574. // unit = "万吨"
  575. //} else {
  576. // unit = "%"
  577. //}
  578. item := &models.BaseFromOilchemIndex{
  579. IndexName: indexName,
  580. ClassifyId: 2,
  581. Unit: "万吨",
  582. Frequency: "周度",
  583. Describe: "",
  584. DataTime: dataTime,
  585. Value: value,
  586. Sort: 0,
  587. CreateTime: time.Now(),
  588. ModifyTime: time.Now(),
  589. IndexNameStr: indexName,
  590. MarketName: "",
  591. }
  592. indexList = append(indexList,item)
  593. area = ""
  594. })
  595. })
  596. })
  597. // 写入数据库
  598. err = PostHandleOilchem(indexList)
  599. if err != nil {
  600. utils.FileLog.Error(fmt.Sprintf("PostHandleOilchem err:%v",err))
  601. fmt.Println("PostHandleOilchem err",err)
  602. return
  603. }
  604. return
  605. }
  606. // 中国尿素样本港口库存周数据分析
  607. func AnalysisOilchemNiaoSu6(htm []byte) (err error) {
  608. if len(htm) == 0 {
  609. utils.FileLog.Info("htm empty")
  610. return
  611. }
  612. doc, e := goquery.NewDocumentFromReader(strings.NewReader(string(htm)))
  613. if e != nil {
  614. err = fmt.Errorf("NewDocumentFromReader err: %v", e)
  615. return
  616. }
  617. //titleList := make([]string, 0)
  618. unitList := make([]string, 0)
  619. doc.Find("p").Each(func(i int, selection *goquery.Selection) {
  620. ptext := selection.Text()
  621. if strings.Contains(ptext, "单位:") {
  622. unit := strings.Replace(ptext, "单位:", "", -1)
  623. //fmt.Println("unit:",unit)
  624. unitList = append(unitList, unit)
  625. }
  626. //if strings.Contains(ptext, "中国尿素样本生产理论利润周数据统计") {
  627. // title := ptext
  628. // //fmt.Println("title:",title)
  629. // titleList = append(titleList, title)
  630. //}
  631. })
  632. area := ""
  633. title := doc.Find("h2").Text()
  634. fmt.Println("title:",title)
  635. createTimeStr := doc.Find("h2").Next().Text()
  636. createTimeStr = strings.TrimLeft(createTimeStr,"发布时间:")
  637. createTimeStrIndex := strings.Index(createTimeStr,"来源:")
  638. createTimeStr = createTimeStr[:createTimeStrIndex]
  639. createTimeStr = strings.TrimSpace(createTimeStr)
  640. createTime, err := time.Parse(utils.HlbFormatDateTimeNoSecond, createTimeStr)
  641. if err != nil {
  642. utils.FileLog.Error(fmt.Sprintf("time.Parse err:%v",err))
  643. return
  644. }
  645. //fmt.Println("createTime:",createTime)
  646. dataTime := createTime.Format(utils.FormatDate)
  647. //fmt.Println("dataTime:",dataTime)
  648. indexList := make([]*models.BaseFromOilchemIndex, 0)
  649. doc.Find("tbody").Each(func(tableIndex int, table *goquery.Selection) {
  650. table.Find("tr").First().Each(func(ii int, table2 *goquery.Selection) {
  651. table.Find("td").Each(func(jj int, table3 *goquery.Selection) {
  652. text3 := table3.Text()
  653. text3 = strings.Replace(text3,"\n","",-1)
  654. text3 = strings.Replace(text3," ","",-1)
  655. if text3 == "上周" || text3 == "环比" || text3 == "地区" {
  656. return
  657. }
  658. //fmt.Println("table3:",text3)
  659. //utils.FileLog.Info(fmt.Sprintf("table3:%s",text3))
  660. //fmt.Println("ii:",ii)
  661. //utils.FileLog.Info(fmt.Sprintf("ii:%d",ii))
  662. //fmt.Println("jj:",jj)
  663. //utils.FileLog.Info(fmt.Sprintf("jj:%d",jj))
  664. //fmt.Println("tableIndex:",tableIndex)
  665. //utils.FileLog.Info(fmt.Sprintf("tableIndex:%d",tableIndex))
  666. if utils.ContainsChinese(text3) && text3 != "本周" {
  667. area = text3
  668. return
  669. }
  670. if area == "" {
  671. return
  672. }
  673. value := text3
  674. value = strings.TrimRight(value, "%")
  675. //valueF, e := strconv.ParseFloat(value, 64)
  676. //if e != nil {
  677. // err = e
  678. // utils.FileLog.Error(fmt.Sprintf("strconv.ParseFloat err:%v",e))
  679. // return
  680. //}
  681. indexName := "中国尿素样本分港口库存" + "(" + area + ")"
  682. fmt.Println("indexName:",indexName)
  683. fmt.Println("valueF:",value)
  684. //unit := ""
  685. //if strings.Contains(area,"产量") {
  686. // unit = "万吨"
  687. //} else {
  688. // unit = "%"
  689. //}
  690. item := &models.BaseFromOilchemIndex{
  691. IndexName: indexName,
  692. ClassifyId: 2,
  693. Unit: "万吨",
  694. Frequency: "周度",
  695. Describe: "",
  696. DataTime: dataTime,
  697. Value: value,
  698. Sort: 0,
  699. CreateTime: time.Now(),
  700. ModifyTime: time.Now(),
  701. IndexNameStr: indexName,
  702. MarketName: area,
  703. }
  704. indexList = append(indexList,item)
  705. area = ""
  706. })
  707. })
  708. })
  709. // 写入数据库
  710. err = PostHandleOilchem(indexList)
  711. if err != nil {
  712. utils.FileLog.Error(fmt.Sprintf("PostHandleOilchem err:%v",err))
  713. fmt.Println("PostHandleOilchem err",err)
  714. return
  715. }
  716. return
  717. }
  718. // 中国尿素企业预收订单周数据分析
  719. func AnalysisOilchemNiaoSu7(htm []byte) (err error) {
  720. if len(htm) == 0 {
  721. utils.FileLog.Info("htm empty")
  722. return
  723. }
  724. doc, e := goquery.NewDocumentFromReader(strings.NewReader(string(htm)))
  725. if e != nil {
  726. err = fmt.Errorf("NewDocumentFromReader err: %v", e)
  727. return
  728. }
  729. //titleList := make([]string, 0)
  730. unitList := make([]string, 0)
  731. doc.Find("p").Each(func(i int, selection *goquery.Selection) {
  732. ptext := selection.Text()
  733. if strings.Contains(ptext, "单位:") {
  734. unit := strings.Replace(ptext, "单位:", "", -1)
  735. //fmt.Println("unit:",unit)
  736. unitList = append(unitList, unit)
  737. }
  738. //if strings.Contains(ptext, "中国尿素样本生产理论利润周数据统计") {
  739. // title := ptext
  740. // //fmt.Println("title:",title)
  741. // titleList = append(titleList, title)
  742. //}
  743. })
  744. area := ""
  745. title := doc.Find("h2").Text()
  746. fmt.Println("title:",title)
  747. createTimeStr := doc.Find("h2").Next().Text()
  748. createTimeStr = strings.TrimLeft(createTimeStr,"发布时间:")
  749. createTimeStrIndex := strings.Index(createTimeStr,"来源:")
  750. createTimeStr = createTimeStr[:createTimeStrIndex]
  751. createTimeStr = strings.TrimSpace(createTimeStr)
  752. createTime, err := time.Parse(utils.HlbFormatDateTimeNoSecond, createTimeStr)
  753. if err != nil {
  754. utils.FileLog.Error(fmt.Sprintf("time.Parse err:%v",err))
  755. return
  756. }
  757. //fmt.Println("createTime:",createTime)
  758. dataTime := createTime.Format(utils.FormatDate)
  759. //fmt.Println("dataTime:",dataTime)
  760. indexList := make([]*models.BaseFromOilchemIndex, 0)
  761. doc.Find("tbody").Each(func(tableIndex int, table *goquery.Selection) {
  762. table.Find("tr").First().Each(func(ii int, table2 *goquery.Selection) {
  763. table.Find("td").Each(func(jj int, table3 *goquery.Selection) {
  764. text3 := table3.Text()
  765. text3 = strings.Replace(text3,"\n","",-1)
  766. text3 = strings.Replace(text3," ","",-1)
  767. if text3 == "上周" || text3 == "环比" || text3 == "地区" {
  768. return
  769. }
  770. //fmt.Println("table3:",text3)
  771. //utils.FileLog.Info(fmt.Sprintf("table3:%s",text3))
  772. //fmt.Println("ii:",ii)
  773. //utils.FileLog.Info(fmt.Sprintf("ii:%d",ii))
  774. //fmt.Println("jj:",jj)
  775. //utils.FileLog.Info(fmt.Sprintf("jj:%d",jj))
  776. //fmt.Println("tableIndex:",tableIndex)
  777. //utils.FileLog.Info(fmt.Sprintf("tableIndex:%d",tableIndex))
  778. if utils.ContainsChinese(text3) && text3 != "本周" {
  779. area = text3
  780. return
  781. }
  782. if area == "" {
  783. return
  784. }
  785. value := text3
  786. value = strings.TrimRight(value, "%")
  787. //valueF, e := strconv.ParseFloat(value, 64)
  788. //if e != nil {
  789. // err = e
  790. // utils.FileLog.Error(fmt.Sprintf("strconv.ParseFloat err:%v",e))
  791. // return
  792. //}
  793. indexName := "中国尿素企业预收订单"
  794. fmt.Println("indexName:",indexName)
  795. fmt.Println("valueF:",value)
  796. //unit := ""
  797. //if strings.Contains(area,"产量") {
  798. // unit = "万吨"
  799. //} else {
  800. // unit = "%"
  801. //}
  802. item := &models.BaseFromOilchemIndex{
  803. IndexName: indexName,
  804. ClassifyId: 2,
  805. Unit: "日",
  806. Frequency: "周度",
  807. Describe: "",
  808. DataTime: dataTime,
  809. Value: value,
  810. Sort: 0,
  811. CreateTime: time.Now(),
  812. ModifyTime: time.Now(),
  813. IndexNameStr: indexName,
  814. MarketName: area,
  815. }
  816. indexList = append(indexList,item)
  817. area = ""
  818. })
  819. })
  820. })
  821. // 写入数据库
  822. err = PostHandleOilchem(indexList)
  823. if err != nil {
  824. utils.FileLog.Error(fmt.Sprintf("PostHandleOilchem err:%v",err))
  825. fmt.Println("PostHandleOilchem err",err)
  826. return
  827. }
  828. return
  829. }
  830. // 中国尿素样本理论利润周数据分析
  831. func AnalysisOilchemNiaoSu8(htm []byte) (err error) {
  832. if len(htm) == 0 {
  833. utils.FileLog.Info("htm empty")
  834. return
  835. }
  836. doc, e := goquery.NewDocumentFromReader(strings.NewReader(string(htm)))
  837. if e != nil {
  838. err = fmt.Errorf("NewDocumentFromReader err: %v", e)
  839. return
  840. }
  841. //titleList := make([]string, 0)
  842. unitList := make([]string, 0)
  843. doc.Find("p").Each(func(i int, selection *goquery.Selection) {
  844. ptext := selection.Text()
  845. if strings.Contains(ptext, "单位:") {
  846. unit := strings.Replace(ptext, "单位:", "", -1)
  847. //fmt.Println("unit:",unit)
  848. unitList = append(unitList, unit)
  849. }
  850. //if strings.Contains(ptext, "中国尿素样本生产理论利润周数据统计") {
  851. // title := ptext
  852. // //fmt.Println("title:",title)
  853. // titleList = append(titleList, title)
  854. //}
  855. })
  856. area := ""
  857. title := doc.Find("h2").Text()
  858. fmt.Println("title:",title)
  859. createTimeStr := doc.Find("h2").Next().Text()
  860. createTimeStr = strings.TrimLeft(createTimeStr,"发布时间:")
  861. createTimeStrIndex := strings.Index(createTimeStr,"来源:")
  862. createTimeStr = createTimeStr[:createTimeStrIndex]
  863. createTimeStr = strings.TrimSpace(createTimeStr)
  864. createTime, err := time.Parse(utils.HlbFormatDateTimeNoSecond, createTimeStr)
  865. if err != nil {
  866. utils.FileLog.Error(fmt.Sprintf("time.Parse err:%v",err))
  867. return
  868. }
  869. //fmt.Println("createTime:",createTime)
  870. dataTime := createTime.Format(utils.FormatDate)
  871. //fmt.Println("dataTime:",dataTime)
  872. indexList := make([]*models.BaseFromOilchemIndex, 0)
  873. doc.Find("tbody").Each(func(tableIndex int, table *goquery.Selection) {
  874. table.Find("tr").First().Each(func(ii int, table2 *goquery.Selection) {
  875. table.Find("td").Each(func(jj int, table3 *goquery.Selection) {
  876. text3 := table3.Text()
  877. text3 = strings.Replace(text3,"\n","",-1)
  878. text3 = strings.Replace(text3," ","",-1)
  879. if text3 == "上周" || text3 == "环比" || text3 == "地区" {
  880. return
  881. }
  882. //fmt.Println("table3:",text3)
  883. //utils.FileLog.Info(fmt.Sprintf("table3:%s",text3))
  884. //fmt.Println("ii:",ii)
  885. //utils.FileLog.Info(fmt.Sprintf("ii:%d",ii))
  886. //fmt.Println("jj:",jj)
  887. //utils.FileLog.Info(fmt.Sprintf("jj:%d",jj))
  888. //fmt.Println("tableIndex:",tableIndex)
  889. //utils.FileLog.Info(fmt.Sprintf("tableIndex:%d",tableIndex))
  890. if utils.ContainsChinese(text3) && text3 != "本周" {
  891. area = text3
  892. return
  893. }
  894. if area == "" {
  895. return
  896. }
  897. value := text3
  898. value = strings.TrimRight(value, "%")
  899. //valueF, e := strconv.ParseFloat(value, 64)
  900. //if e != nil {
  901. // err = e
  902. // utils.FileLog.Error(fmt.Sprintf("strconv.ParseFloat err:%v",e))
  903. // return
  904. //}
  905. indexName := "中国尿素企业预收订单"
  906. fmt.Println("indexName:",indexName)
  907. fmt.Println("valueF:",value)
  908. //unit := ""
  909. //if strings.Contains(area,"产量") {
  910. // unit = "万吨"
  911. //} else {
  912. // unit = "%"
  913. //}
  914. item := &models.BaseFromOilchemIndex{
  915. IndexName: indexName,
  916. ClassifyId: 2,
  917. Unit: "日",
  918. Frequency: "周度",
  919. Describe: "",
  920. DataTime: dataTime,
  921. Value: value,
  922. Sort: 0,
  923. CreateTime: time.Now(),
  924. ModifyTime: time.Now(),
  925. IndexNameStr: indexName,
  926. MarketName: area,
  927. }
  928. indexList = append(indexList,item)
  929. area = ""
  930. })
  931. })
  932. })
  933. // 写入数据库
  934. err = PostHandleOilchem(indexList)
  935. if err != nil {
  936. utils.FileLog.Error(fmt.Sprintf("PostHandleOilchem err:%v",err))
  937. fmt.Println("PostHandleOilchem err",err)
  938. return
  939. }
  940. return
  941. }
  942. // 中国复合肥产能利用率周数据统计
  943. func AnalysisOilchemNiaoSu9(htm []byte) (err error) {
  944. if len(htm) == 0 {
  945. utils.FileLog.Info("htm empty")
  946. return
  947. }
  948. doc, e := goquery.NewDocumentFromReader(strings.NewReader(string(htm)))
  949. if e != nil {
  950. err = fmt.Errorf("NewDocumentFromReader err: %v", e)
  951. return
  952. }
  953. //titleList := make([]string, 0)
  954. unitList := make([]string, 0)
  955. doc.Find("p").Each(func(i int, selection *goquery.Selection) {
  956. ptext := selection.Text()
  957. if strings.Contains(ptext, "单位:") {
  958. unit := strings.Replace(ptext, "单位:", "", -1)
  959. //fmt.Println("unit:",unit)
  960. unitList = append(unitList, unit)
  961. }
  962. //if strings.Contains(ptext, "中国尿素样本生产理论利润周数据统计") {
  963. // title := ptext
  964. // //fmt.Println("title:",title)
  965. // titleList = append(titleList, title)
  966. //}
  967. })
  968. area := ""
  969. title := doc.Find("h2").Text()
  970. fmt.Println("title:",title)
  971. createTimeStr := doc.Find("h2").Next().Text()
  972. createTimeStr = strings.TrimLeft(createTimeStr,"发布时间:")
  973. createTimeStrIndex := strings.Index(createTimeStr,"来源:")
  974. createTimeStr = createTimeStr[:createTimeStrIndex]
  975. createTimeStr = strings.TrimSpace(createTimeStr)
  976. createTime, err := time.Parse(utils.HlbFormatDateTimeNoSecond, createTimeStr)
  977. if err != nil {
  978. utils.FileLog.Error(fmt.Sprintf("time.Parse err:%v",err))
  979. return
  980. }
  981. //fmt.Println("createTime:",createTime)
  982. dataTime := createTime.Format(utils.FormatDate)
  983. //fmt.Println("dataTime:",dataTime)
  984. indexList := make([]*models.BaseFromOilchemIndex, 0)
  985. doc.Find("tbody").Each(func(tableIndex int, table *goquery.Selection) {
  986. table.Find("tr").First().Each(func(ii int, table2 *goquery.Selection) {
  987. table.Find("td").Each(func(jj int, table3 *goquery.Selection) {
  988. text3 := table3.Text()
  989. text3 = strings.Replace(text3,"\n","",-1)
  990. text3 = strings.Replace(text3," ","",-1)
  991. if text3 == "上周" || text3 == "环比" || text3 == "地区" {
  992. return
  993. }
  994. //fmt.Println("table3:",text3)
  995. //utils.FileLog.Info(fmt.Sprintf("table3:%s",text3))
  996. //fmt.Println("ii:",ii)
  997. //utils.FileLog.Info(fmt.Sprintf("ii:%d",ii))
  998. //fmt.Println("jj:",jj)
  999. //utils.FileLog.Info(fmt.Sprintf("jj:%d",jj))
  1000. //fmt.Println("tableIndex:",tableIndex)
  1001. //utils.FileLog.Info(fmt.Sprintf("tableIndex:%d",tableIndex))
  1002. if utils.ContainsChinese(text3) && text3 != "本周" {
  1003. area = text3
  1004. return
  1005. }
  1006. if area == "" {
  1007. return
  1008. }
  1009. value := text3
  1010. value = strings.TrimRight(value, "%")
  1011. //valueF, e := strconv.ParseFloat(value, 64)
  1012. //if e != nil {
  1013. // err = e
  1014. // utils.FileLog.Error(fmt.Sprintf("strconv.ParseFloat err:%v",e))
  1015. // return
  1016. //}
  1017. indexName := "中国复合肥产能利用率"
  1018. fmt.Println("indexName:",indexName)
  1019. fmt.Println("valueF:",value)
  1020. //unit := ""
  1021. //if strings.Contains(area,"产量") {
  1022. // unit = "万吨"
  1023. //} else {
  1024. // unit = "%"
  1025. //}
  1026. item := &models.BaseFromOilchemIndex{
  1027. IndexName: indexName,
  1028. ClassifyId: 2,
  1029. Unit: "%",
  1030. Frequency: "周度",
  1031. Describe: "",
  1032. DataTime: dataTime,
  1033. Value: value,
  1034. Sort: 0,
  1035. CreateTime: time.Now(),
  1036. ModifyTime: time.Now(),
  1037. IndexNameStr: indexName,
  1038. MarketName: area,
  1039. }
  1040. indexList = append(indexList,item)
  1041. area = ""
  1042. })
  1043. })
  1044. })
  1045. // 写入数据库
  1046. err = PostHandleOilchem(indexList)
  1047. if err != nil {
  1048. utils.FileLog.Error(fmt.Sprintf("PostHandleOilchem err:%v",err))
  1049. fmt.Println("PostHandleOilchem err",err)
  1050. return
  1051. }
  1052. return
  1053. }
  1054. // 中国三聚氰胺产能利用率周数据统计
  1055. func AnalysisOilchemNiaoSu10(htm []byte) (err error) {
  1056. if len(htm) == 0 {
  1057. utils.FileLog.Info("htm empty")
  1058. return
  1059. }
  1060. doc, e := goquery.NewDocumentFromReader(strings.NewReader(string(htm)))
  1061. if e != nil {
  1062. err = fmt.Errorf("NewDocumentFromReader err: %v", e)
  1063. return
  1064. }
  1065. //titleList := make([]string, 0)
  1066. unitList := make([]string, 0)
  1067. doc.Find("p").Each(func(i int, selection *goquery.Selection) {
  1068. ptext := selection.Text()
  1069. if strings.Contains(ptext, "单位:") {
  1070. unit := strings.Replace(ptext, "单位:", "", -1)
  1071. //fmt.Println("unit:",unit)
  1072. unitList = append(unitList, unit)
  1073. }
  1074. //if strings.Contains(ptext, "中国尿素样本生产理论利润周数据统计") {
  1075. // title := ptext
  1076. // //fmt.Println("title:",title)
  1077. // titleList = append(titleList, title)
  1078. //}
  1079. })
  1080. area := ""
  1081. title := doc.Find("h2").Text()
  1082. fmt.Println("title:",title)
  1083. createTimeStr := doc.Find("h2").Next().Text()
  1084. createTimeStr = strings.TrimLeft(createTimeStr,"发布时间:")
  1085. createTimeStrIndex := strings.Index(createTimeStr,"来源:")
  1086. createTimeStr = createTimeStr[:createTimeStrIndex]
  1087. createTimeStr = strings.TrimSpace(createTimeStr)
  1088. createTime, err := time.Parse(utils.HlbFormatDateTimeNoSecond, createTimeStr)
  1089. if err != nil {
  1090. utils.FileLog.Error(fmt.Sprintf("time.Parse err:%v",err))
  1091. return
  1092. }
  1093. //fmt.Println("createTime:",createTime)
  1094. dataTime := createTime.Format(utils.FormatDate)
  1095. //fmt.Println("dataTime:",dataTime)
  1096. indexList := make([]*models.BaseFromOilchemIndex, 0)
  1097. doc.Find("tbody").Each(func(tableIndex int, table *goquery.Selection) {
  1098. table.Find("tr").First().Each(func(ii int, table2 *goquery.Selection) {
  1099. table.Find("td").Each(func(jj int, table3 *goquery.Selection) {
  1100. text3 := table3.Text()
  1101. text3 = strings.Replace(text3,"\n","",-1)
  1102. text3 = strings.Replace(text3," ","",-1)
  1103. if text3 == "上周" || text3 == "环比" || text3 == "地区" {
  1104. return
  1105. }
  1106. //fmt.Println("table3:",text3)
  1107. //utils.FileLog.Info(fmt.Sprintf("table3:%s",text3))
  1108. //fmt.Println("ii:",ii)
  1109. //utils.FileLog.Info(fmt.Sprintf("ii:%d",ii))
  1110. //fmt.Println("jj:",jj)
  1111. //utils.FileLog.Info(fmt.Sprintf("jj:%d",jj))
  1112. //fmt.Println("tableIndex:",tableIndex)
  1113. //utils.FileLog.Info(fmt.Sprintf("tableIndex:%d",tableIndex))
  1114. if utils.ContainsChinese(text3) && text3 != "本周" {
  1115. area = text3
  1116. return
  1117. }
  1118. if area == "" {
  1119. return
  1120. }
  1121. value := text3
  1122. value = strings.TrimRight(value, "%")
  1123. //valueF, e := strconv.ParseFloat(value, 64)
  1124. //if e != nil {
  1125. // err = e
  1126. // utils.FileLog.Error(fmt.Sprintf("strconv.ParseFloat err:%v",e))
  1127. // return
  1128. //}
  1129. indexName := "中国三聚氰胺产能利用率"
  1130. fmt.Println("indexName:",indexName)
  1131. fmt.Println("valueF:",value)
  1132. //unit := ""
  1133. //if strings.Contains(area,"产量") {
  1134. // unit = "万吨"
  1135. //} else {
  1136. // unit = "%"
  1137. //}
  1138. item := &models.BaseFromOilchemIndex{
  1139. IndexName: indexName,
  1140. ClassifyId: 2,
  1141. Unit: "%",
  1142. Frequency: "周度",
  1143. Describe: "",
  1144. DataTime: dataTime,
  1145. Value: value,
  1146. Sort: 0,
  1147. CreateTime: time.Now(),
  1148. ModifyTime: time.Now(),
  1149. IndexNameStr: indexName,
  1150. MarketName: area,
  1151. }
  1152. indexList = append(indexList,item)
  1153. area = ""
  1154. })
  1155. })
  1156. })
  1157. // 写入数据库
  1158. err = PostHandleOilchem(indexList)
  1159. if err != nil {
  1160. utils.FileLog.Error(fmt.Sprintf("PostHandleOilchem err:%v",err))
  1161. fmt.Println("PostHandleOilchem err",err)
  1162. return
  1163. }
  1164. return
  1165. }
  1166. // 中国尿素产量分省份月数据统计
  1167. func AnalysisOilchemNiaoSu11(htm []byte) (err error) {
  1168. if len(htm) == 0 {
  1169. utils.FileLog.Info("htm empty")
  1170. return
  1171. }
  1172. doc, e := goquery.NewDocumentFromReader(strings.NewReader(string(htm)))
  1173. if e != nil {
  1174. err = fmt.Errorf("NewDocumentFromReader err: %v", e)
  1175. return
  1176. }
  1177. //titleList := make([]string, 0)
  1178. unitList := make([]string, 0)
  1179. doc.Find("p").Each(func(i int, selection *goquery.Selection) {
  1180. ptext := selection.Text()
  1181. if strings.Contains(ptext, "单位:") {
  1182. unit := strings.Replace(ptext, "单位:", "", -1)
  1183. //fmt.Println("unit:",unit)
  1184. unitList = append(unitList, unit)
  1185. }
  1186. //if strings.Contains(ptext, "中国尿素样本生产理论利润周数据统计") {
  1187. // title := ptext
  1188. // //fmt.Println("title:",title)
  1189. // titleList = append(titleList, title)
  1190. //}
  1191. })
  1192. area := ""
  1193. title := doc.Find("h2").Text()
  1194. fmt.Println("title:",title)
  1195. createTimeStr := doc.Find("h2").Next().Text()
  1196. createTimeStr = strings.TrimLeft(createTimeStr,"发布时间:")
  1197. createTimeStrIndex := strings.Index(createTimeStr,"来源:")
  1198. createTimeStr = createTimeStr[:createTimeStrIndex]
  1199. createTimeStr = strings.TrimSpace(createTimeStr)
  1200. createTime, err := time.Parse(utils.HlbFormatDateTimeNoSecond, createTimeStr)
  1201. if err != nil {
  1202. utils.FileLog.Error(fmt.Sprintf("time.Parse err:%v",err))
  1203. return
  1204. }
  1205. //fmt.Println("createTime:",createTime)
  1206. dataTime := createTime.Format(utils.FormatDate)
  1207. //fmt.Println("dataTime:",dataTime)
  1208. indexList := make([]*models.BaseFromOilchemIndex, 0)
  1209. doc.Find("tbody").Each(func(tableIndex int, table *goquery.Selection) {
  1210. table.Find("tr").First().Each(func(ii int, table2 *goquery.Selection) {
  1211. table.Find("td").Each(func(jj int, table3 *goquery.Selection) {
  1212. text3 := table3.Text()
  1213. text3 = strings.Replace(text3,"\n","",-1)
  1214. text3 = strings.Replace(text3," ","",-1)
  1215. if text3 == "上周" || text3 == "环比" || text3 == "地区" {
  1216. return
  1217. }
  1218. //fmt.Println("table3:",text3)
  1219. //utils.FileLog.Info(fmt.Sprintf("table3:%s",text3))
  1220. //fmt.Println("ii:",ii)
  1221. //utils.FileLog.Info(fmt.Sprintf("ii:%d",ii))
  1222. //fmt.Println("jj:",jj)
  1223. //utils.FileLog.Info(fmt.Sprintf("jj:%d",jj))
  1224. //fmt.Println("tableIndex:",tableIndex)
  1225. //utils.FileLog.Info(fmt.Sprintf("tableIndex:%d",tableIndex))
  1226. if utils.ContainsChinese(text3) && text3 != "本周" {
  1227. area = text3
  1228. return
  1229. }
  1230. if area == "" {
  1231. return
  1232. }
  1233. value := text3
  1234. value = strings.TrimRight(value, "%")
  1235. //valueF, e := strconv.ParseFloat(value, 64)
  1236. //if e != nil {
  1237. // err = e
  1238. // utils.FileLog.Error(fmt.Sprintf("strconv.ParseFloat err:%v",e))
  1239. // return
  1240. //}
  1241. indexName := "中国尿素产量" + "(" + area + ")"
  1242. fmt.Println("indexName:",indexName)
  1243. fmt.Println("valueF:",value)
  1244. //unit := ""
  1245. //if strings.Contains(area,"产量") {
  1246. // unit = "万吨"
  1247. //} else {
  1248. // unit = "%"
  1249. //}
  1250. item := &models.BaseFromOilchemIndex{
  1251. IndexName: indexName,
  1252. ClassifyId: 2,
  1253. Unit: "吨",
  1254. Frequency: "周度",
  1255. Describe: "",
  1256. DataTime: dataTime,
  1257. Value: value,
  1258. Sort: 0,
  1259. CreateTime: time.Now(),
  1260. ModifyTime: time.Now(),
  1261. IndexNameStr: indexName,
  1262. MarketName: area,
  1263. }
  1264. indexList = append(indexList,item)
  1265. area = ""
  1266. })
  1267. })
  1268. })
  1269. // 写入数据库
  1270. err = PostHandleOilchem(indexList)
  1271. if err != nil {
  1272. utils.FileLog.Error(fmt.Sprintf("PostHandleOilchem err:%v",err))
  1273. fmt.Println("PostHandleOilchem err",err)
  1274. return
  1275. }
  1276. return
  1277. }
  1278. // 中国尿素进出口量数据总体分析 todo
  1279. func AnalysisOilchemNiaoSu12(htm []byte) (err error) {
  1280. if len(htm) == 0 {
  1281. utils.FileLog.Info("htm empty")
  1282. return
  1283. }
  1284. doc, e := goquery.NewDocumentFromReader(strings.NewReader(string(htm)))
  1285. if e != nil {
  1286. err = fmt.Errorf("NewDocumentFromReader err: %v", e)
  1287. return
  1288. }
  1289. //titleList := make([]string, 0)
  1290. unitList := make([]string, 0)
  1291. doc.Find("p").Each(func(i int, selection *goquery.Selection) {
  1292. ptext := selection.Text()
  1293. if strings.Contains(ptext, "单位:") {
  1294. unit := strings.Replace(ptext, "单位:", "", -1)
  1295. //fmt.Println("unit:",unit)
  1296. unitList = append(unitList, unit)
  1297. }
  1298. //if strings.Contains(ptext, "中国尿素样本生产理论利润周数据统计") {
  1299. // title := ptext
  1300. // //fmt.Println("title:",title)
  1301. // titleList = append(titleList, title)
  1302. //}
  1303. })
  1304. area := ""
  1305. title := doc.Find("h2").Text()
  1306. fmt.Println("title:",title)
  1307. createTimeStr := doc.Find("h2").Next().Text()
  1308. createTimeStr = strings.TrimLeft(createTimeStr,"发布时间:")
  1309. createTimeStrIndex := strings.Index(createTimeStr,"来源:")
  1310. createTimeStr = createTimeStr[:createTimeStrIndex]
  1311. createTimeStr = strings.TrimSpace(createTimeStr)
  1312. createTime, err := time.Parse(utils.HlbFormatDateTimeNoSecond, createTimeStr)
  1313. if err != nil {
  1314. utils.FileLog.Error(fmt.Sprintf("time.Parse err:%v",err))
  1315. return
  1316. }
  1317. //fmt.Println("createTime:",createTime)
  1318. dataTime := createTime.Format(utils.FormatDate)
  1319. //fmt.Println("dataTime:",dataTime)
  1320. indexTitle := "中国尿素"
  1321. indexList := make([]*models.BaseFromOilchemIndex, 0)
  1322. doc.Find("tbody").Each(func(tableIndex int, table *goquery.Selection) {
  1323. table.Find("tr").Each(func(ii int, table2 *goquery.Selection) {
  1324. table2.Find("td").Each(func(jj int, table3 *goquery.Selection) {
  1325. text3 := table3.Text()
  1326. text3 = strings.Replace(text3,"\n","",-1)
  1327. text3 = strings.Replace(text3," ","",-1)
  1328. if text3 == "上周" || text3 == "环比" || text3 == "地区" {
  1329. return
  1330. }
  1331. //fmt.Println("table3:",text3)
  1332. if jj == 1 {
  1333. indexTitle = "中国尿素进口量"
  1334. } else if jj == 4 {
  1335. indexTitle = "中国尿素出口量"
  1336. }
  1337. //utils.FileLog.Info(fmt.Sprintf("table3:%s",text3))
  1338. //fmt.Println("ii:",ii)
  1339. //utils.FileLog.Info(fmt.Sprintf("ii:%d",ii))
  1340. //fmt.Println("jj:",jj)
  1341. //utils.FileLog.Info(fmt.Sprintf("jj:%d",jj))
  1342. //fmt.Println("tableIndex:",tableIndex)
  1343. //utils.FileLog.Info(fmt.Sprintf("tableIndex:%d",tableIndex))
  1344. if utils.ContainsChinese(text3) && text3 != "本周" {
  1345. area = text3
  1346. return
  1347. }
  1348. if area == "" {
  1349. return
  1350. }
  1351. fmt.Println("area:",area)
  1352. if jj != 1 && jj != 4 {
  1353. return
  1354. }
  1355. value := text3
  1356. value = strings.TrimRight(value, "%")
  1357. //valueF, e := strconv.ParseFloat(value, 64)
  1358. //if e != nil {
  1359. // err = e
  1360. // utils.FileLog.Error(fmt.Sprintf("strconv.ParseFloat err:%v",e))
  1361. // return
  1362. //}
  1363. indexName := indexTitle + "(" + area + ")"
  1364. fmt.Println("indexName:",indexName)
  1365. fmt.Println("valueF:",value)
  1366. //unit := ""
  1367. //if strings.Contains(area,"产量") {
  1368. // unit = "万吨"
  1369. //} else {
  1370. // unit = "%"
  1371. //}
  1372. item := &models.BaseFromOilchemIndex{
  1373. IndexName: indexName,
  1374. ClassifyId: 2,
  1375. Unit: "吨",
  1376. Frequency: "周度",
  1377. Describe: "",
  1378. DataTime: dataTime,
  1379. Value: value,
  1380. Sort: 0,
  1381. CreateTime: time.Now(),
  1382. ModifyTime: time.Now(),
  1383. IndexNameStr: indexName,
  1384. MarketName: area,
  1385. }
  1386. indexList = append(indexList,item)
  1387. area = ""
  1388. })
  1389. })
  1390. })
  1391. // 写入数据库
  1392. err = PostHandleOilchem(indexList)
  1393. if err != nil {
  1394. utils.FileLog.Error(fmt.Sprintf("PostHandleOilchem err:%v",err))
  1395. fmt.Println("PostHandleOilchem err",err)
  1396. return
  1397. }
  1398. return
  1399. }