eta_data_analysis/services/base_from_oilchem/fuhefei_edb.go

package services

import (
	"eta/eta_data_analysis/models"
	"eta/eta_data_analysis/utils"
	"fmt"
	"github.com/PuerkitoBio/goquery"
	"strings"
	"time"
)

var FuHeFeiListMap = map[string]string {
	"中国磷铵产能利用率周数据统计" : "https://list.oilchem.net/192/6522/",
	"中国钾肥样本港口库存量周数据统计" : "https://list.oilchem.net/192/50857/",
	"中国复合肥进出口月数据分析" : "https://list.oilchem.net/192/5379/",
}

func FuHeFeiList(num int) (err error) {
	for k, v := range FuHeFeiListMap {
		for i := 1; i < num; i++ {
			listUrl := v + fmt.Sprintf("%d.html",i)
			fmt.Println("listUrl:",listUrl)
			htm, e := FetchPageHtml(listUrl)
			if e != nil {
				err = e
				utils.FileLog.Error(fmt.Sprintf("FetchPageHtml err:%v",err))
				fmt.Println("FetchPageHtml err",err)
				return
			}
			err = AnalysisOilchemList(htm, k)
			if err != nil {
				utils.FileLog.Error(fmt.Sprintf("AnalysisOilchemList err:%v",err))
				fmt.Println("AnalysisOilchemList err",err)
				return
			}
			time.Sleep(2*time.Second)
		}
	}
	return
}


// 中国磷铵产能利用率周数据统计
func AnalysisOilchemFuHeFei1(htm []byte) (err error) {
	if len(htm) == 0 {
		utils.FileLog.Info("htm empty")
		return
	}
	doc, e := goquery.NewDocumentFromReader(strings.NewReader(string(htm)))
	if e != nil {
		err = fmt.Errorf("NewDocumentFromReader err: %v", e)
		return
	}
	titleList := make([]string, 0)
	unitList := make([]string, 0)
	doc.Find("p").Each(func(i int, selection *goquery.Selection) {
		ptext := selection.Text()
		if strings.Contains(ptext, "单位：") {
			unit := strings.Replace(ptext, "单位：", "", -1)
			//fmt.Println("unit:",unit)
			unitList = append(unitList, unit)
		}
		if strings.Contains(ptext, "中国磷铵装置产能利用率") {
			title := ptext
			//fmt.Println("title:",title)
			titleList = append(titleList, title)
		}
	})
	area := ""
	title := doc.Find("h2").Text()
	fmt.Println("title:",title)
	createTimeStr := doc.Find("h2").Next().Text()
	createTimeStr = strings.TrimLeft(createTimeStr,"发布时间：")
	createTimeStrIndex := strings.Index(createTimeStr,"来源：")
	createTimeStr = createTimeStr[:createTimeStrIndex]
	createTimeStr = strings.TrimSpace(createTimeStr)
	createTime, err := time.Parse(utils.HlbFormatDateTimeNoSecond, createTimeStr)
	if err != nil {
		utils.FileLog.Error(fmt.Sprintf("time.Parse err:%v",err))
		return
	}
	//fmt.Println("createTime:",createTime)
	dataTime := createTime.Format(utils.FormatDate)
	//fmt.Println("dataTime:",dataTime)

	indexList := make([]*models.BaseFromOilchemIndex, 0)
	doc.Find("tbody").Each(func(tableIndex int, table *goquery.Selection) {
		table.Find("tr").First().Each(func(ii int, table2 *goquery.Selection) {
			table.Find("td").Each(func(jj int, table3 *goquery.Selection) {
				text3 := table3.Text()
				text3 = strings.Replace(text3,"\n","",-1)
				text3 = strings.Replace(text3," ","",-1)
				if text3 == "上周" || text3 == "环比" || text3 == "地区" {
					return
				}
				//fmt.Println("table3:",text3)
				//utils.FileLog.Info(fmt.Sprintf("table3:%s",text3))
				//fmt.Println("ii:",ii)
				//utils.FileLog.Info(fmt.Sprintf("ii:%d",ii))
				//fmt.Println("jj:",jj)
				//utils.FileLog.Info(fmt.Sprintf("jj:%d",jj))
				//fmt.Println("tableIndex:",tableIndex)
				//utils.FileLog.Info(fmt.Sprintf("tableIndex:%d",tableIndex))

				if utils.ContainsChinese(text3) && text3 != "本周" {
					area = text3
					return
				}
				if area == "" {
					return
				}
				value := text3
				value = strings.TrimRight(value, "%")
				//valueF, e := strconv.ParseFloat(value, 64)
				//if e != nil {
				//	err = e
				//	utils.FileLog.Error(fmt.Sprintf("strconv.ParseFloat err:%v",e))
				//	return
				//}


				indexName := "中国磷铵产能利用率"+ "（" +area+"）"
				fmt.Println("indexName:",indexName)
				fmt.Println("valueF:",value)
				unit := ""
				if strings.Contains(area,"产量") {
					unit = "万吨"
				} else {
					unit = "%"
				}

				item := &models.BaseFromOilchemIndex{
					IndexName:              indexName,
					ClassifyId:             3,
					Unit:                   unit,
					Frequency:              "周度",
					Describe:               "",
					DataTime:               dataTime,
					Value:                  value,
					Sort:                   0,
					CreateTime:             time.Now(),
					ModifyTime:             time.Now(),
					IndexNameStr:           "中国磷铵产能利用率",
					MarketName:             area,
				}
				indexList = append(indexList,item)
				area = ""
			})
		})
	})

	// 写入数据库
	err = PostHandleOilchem(indexList)
	if err != nil {
		utils.FileLog.Error(fmt.Sprintf("PostHandleOilchem err:%v",err))
		fmt.Println("PostHandleOilchem err",err)
		return
	}




	return
}

// 中国钾肥样本港口库存量周数据统计
func AnalysisOilchemFuHeFei2(htm []byte) (err error) {
	if len(htm) == 0 {
		utils.FileLog.Info("htm empty")
		return
	}
	doc, e := goquery.NewDocumentFromReader(strings.NewReader(string(htm)))
	if e != nil {
		err = fmt.Errorf("NewDocumentFromReader err: %v", e)
		return
	}
	//titleList := make([]string, 0)
	unitList := make([]string, 0)
	doc.Find("p").Each(func(i int, selection *goquery.Selection) {
		ptext := selection.Text()
		if strings.Contains(ptext, "单位：") {
			unit := strings.Replace(ptext, "单位：", "", -1)
			//fmt.Println("unit:",unit)
			unitList = append(unitList, unit)
		}
		//if strings.Contains(ptext, "中国磷铵装置产能利用率") {
		//	title := ptext
		//	//fmt.Println("title:",title)
		//	titleList = append(titleList, title)
		//}
	})
	area := ""
	title := doc.Find("h2").Text()
	fmt.Println("title:",title)
	createTimeStr := doc.Find("h2").Next().Text()
	createTimeStr = strings.TrimLeft(createTimeStr,"发布时间：")
	createTimeStrIndex := strings.Index(createTimeStr,"来源：")
	createTimeStr = createTimeStr[:createTimeStrIndex]
	createTimeStr = strings.TrimSpace(createTimeStr)
	createTime, err := time.Parse(utils.HlbFormatDateTimeNoSecond, createTimeStr)
	if err != nil {
		utils.FileLog.Error(fmt.Sprintf("time.Parse err:%v",err))
		return
	}
	//fmt.Println("createTime:",createTime)
	dataTime := createTime.Format(utils.FormatDate)
	//fmt.Println("dataTime:",dataTime)

	indexList := make([]*models.BaseFromOilchemIndex, 0)
	doc.Find("tbody").Each(func(tableIndex int, table *goquery.Selection) {
		table.Find("tr").First().Each(func(ii int, table2 *goquery.Selection) {
			table.Find("td").Each(func(jj int, table3 *goquery.Selection) {
				text3 := table3.Text()
				text3 = strings.Replace(text3,"\n","",-1)
				text3 = strings.Replace(text3," ","",-1)
				if text3 == "上周" || text3 == "环比" || text3 == "地区" {
					return
				}
				//fmt.Println("table3:",text3)
				//utils.FileLog.Info(fmt.Sprintf("table3:%s",text3))
				//fmt.Println("ii:",ii)
				//utils.FileLog.Info(fmt.Sprintf("ii:%d",ii))
				//fmt.Println("jj:",jj)
				//utils.FileLog.Info(fmt.Sprintf("jj:%d",jj))
				//fmt.Println("tableIndex:",tableIndex)
				//utils.FileLog.Info(fmt.Sprintf("tableIndex:%d",tableIndex))

				if utils.ContainsChinese(text3) && text3 != "本周" {
					area = text3
					return
				}
				if area == "" {
					return
				}
				value := text3
				value = strings.TrimRight(value, "%")
				//valueF, e := strconv.ParseFloat(value, 64)
				//if e != nil {
				//	err = e
				//	utils.FileLog.Error(fmt.Sprintf("strconv.ParseFloat err:%v",e))
				//	return
				//}


				indexName := "中国进口氯化钾港口库存"+ "（" +area+"）"
				fmt.Println("indexName:",indexName)
				fmt.Println("valueF:",value)
				//unit := ""
				//if strings.Contains(area,"产量") {
				//	unit = "万吨"
				//} else {
				//	unit = "%"
				//}

				item := &models.BaseFromOilchemIndex{
					IndexName:              indexName,
					ClassifyId:             3,
					Unit:                   "万吨",
					Frequency:              "周度",
					Describe:               "",
					DataTime:               dataTime,
					Value:                  value,
					Sort:                   0,
					CreateTime:             time.Now(),
					ModifyTime:             time.Now(),
					IndexNameStr:           "中国进口氯化钾港口库存",
					MarketName:             area,
				}
				indexList = append(indexList,item)
				area = ""
			})
		})
	})

	// 写入数据库
	err = PostHandleOilchem(indexList)
	if err != nil {
		utils.FileLog.Error(fmt.Sprintf("PostHandleOilchem err:%v",err))
		fmt.Println("PostHandleOilchem err",err)
		return
	}




	return
}

// 中国复合肥进出口月数据分析
func AnalysisOilchemFuHeFei3(htm []byte) (err error) {
	if len(htm) == 0 {
		utils.FileLog.Info("htm empty")
		return
	}
	doc, e := goquery.NewDocumentFromReader(strings.NewReader(string(htm)))
	if e != nil {
		err = fmt.Errorf("NewDocumentFromReader err: %v", e)
		return
	}
	titleList := make([]string, 0)
	unitList := make([]string, 0)
	doc.Find("p[style*='text-align: center;']").Each(func(i int, selection *goquery.Selection) {
		ptext := selection.Text()
		if strings.Contains(ptext, "单位：") {
			unit := strings.Replace(ptext, "单位：", "", -1)
			//fmt.Println("unit:",unit)
			unitList = append(unitList, unit)
		}
		if strings.Contains(ptext, "统计") {
			title := ptext
			//fmt.Println("title:",title)
			titleList = append(titleList, title)
		}
	})
	area := ""
	title := doc.Find("h2").Text()
	fmt.Println("title:",title)
	createTimeStr := doc.Find("h2").Next().Text()
	createTimeStr = strings.TrimLeft(createTimeStr,"发布时间：")
	createTimeStrIndex := strings.Index(createTimeStr,"来源：")
	createTimeStr = createTimeStr[:createTimeStrIndex]
	createTimeStr = strings.TrimSpace(createTimeStr)
	createTime, err := time.Parse(utils.HlbFormatDateTimeNoSecond, createTimeStr)
	if err != nil {
		utils.FileLog.Error(fmt.Sprintf("time.Parse err:%v",err))
		return
	}
	//fmt.Println("createTime:",createTime)
	dataTime := createTime.Format(utils.FormatDate)
	//fmt.Println("dataTime:",dataTime)

	indexList := make([]*models.BaseFromOilchemIndex, 0)
	doc.Find("tbody").Each(func(tableIndex int, table *goquery.Selection) {
		table.Find("tr").First().Each(func(ii int, table2 *goquery.Selection) {
			table.Find("td").Each(func(jj int, table3 *goquery.Selection) {
				text3 := table3.Text()
				text3 = strings.Replace(text3,"\n","",-1)
				text3 = strings.Replace(text3," ","",-1)
				if text3 == "上周" || text3 == "环比" || text3 == "地区" {
					return
				}
				//fmt.Println("table3:",text3)
				//utils.FileLog.Info(fmt.Sprintf("table3:%s",text3))
				//fmt.Println("ii:",ii)
				//utils.FileLog.Info(fmt.Sprintf("ii:%d",ii))
				//fmt.Println("jj:",jj)
				//utils.FileLog.Info(fmt.Sprintf("jj:%d",jj))
				//fmt.Println("tableIndex:",tableIndex)
				//utils.FileLog.Info(fmt.Sprintf("tableIndex:%d",tableIndex))

				if utils.ContainsChinese(text3) && text3 != "本周" {
					area = text3
					return
				}
				if area == "" {
					return
				}
				value := text3
				value = strings.TrimRight(value, "%")
				//valueF, e := strconv.ParseFloat(value, 64)
				//if e != nil {
				//	err = e
				//	utils.FileLog.Error(fmt.Sprintf("strconv.ParseFloat err:%v",e))
				//	return
				//}

				title := titleList[tableIndex]
				//fmt.Println("tbody title:",title)
				if strings.Contains(title,"中国氮磷钾复合肥按产销国进口量月数据统计") {
					title = "中国氮磷钾复合肥产销国进口量"
				} else if strings.Contains(title,"中国氮磷钾复合肥按贸易方式进口量月数据统计") {
					title = "中国氮磷钾复合肥贸易方式进口量"
				} else if strings.Contains(title,"中国氮磷钾三元肥产销国出口量月数据统计") {
					title = "中国氮磷钾复合肥产销国出口量"
				} else if strings.Contains(title,"中国氮磷钾三元肥贸易方式出口量月数据统计") {
					title = "中国氮磷钾复合肥贸易方式出口量"
				} else {
					return
				}

				indexName := title + "（" +area+"）"
				fmt.Println("indexName:",indexName)
				fmt.Println("valueF:",value)
				//unit := ""
				//if strings.Contains(area,"产量") {
				//	unit = "万吨"
				//} else {
				//	unit = "%"
				//}

				item := &models.BaseFromOilchemIndex{
					IndexName:              indexName,
					ClassifyId:             3,
					Unit:                   "吨",
					Frequency:              "月度",
					Describe:               "",
					DataTime:               dataTime,
					Value:                  value,
					Sort:                   0,
					CreateTime:             time.Now(),
					ModifyTime:             time.Now(),
					IndexNameStr:           title,
					MarketName:             area,
				}
				indexList = append(indexList,item)
				area = ""
			})
		})
	})

	//写入数据库
	err = PostHandleOilchem(indexList)
	if err != nil {
		utils.FileLog.Error(fmt.Sprintf("PostHandleOilchem err:%v",err))
		fmt.Println("PostHandleOilchem err",err)
		return
	}




	return
}