package services import ( "eta/eta_data_analysis/models" "eta/eta_data_analysis/utils" "fmt" "github.com/PuerkitoBio/goquery" "strings" "time" ) var NiaoSuListMap = map[string]string { "中国尿素分原料产能利用率周数据统计" : "https://list.oilchem.net/188/43702/", "中国尿素样本生产理论利润周数据统计" : "https://list.oilchem.net/188/43705/", "中国尿素样本港口库存周数据统计" : "https://list.oilchem.net/188/43704/", "中国尿素产量周数据分析" : "https://list.oilchem.net/188/43708/", "中国尿素企业库存周数据分析" : "https://list.oilchem.net/188/43709/", "中国尿素样本港口库存周数据分析" : "https://list.oilchem.net/188/43709/", "中国尿素企业预收订单周数据分析" : "https://list.oilchem.net/188/43710/", "中国复合肥产能利用率周数据统计" : "https://list.oilchem.net/188/43716/", "中国三聚氰胺产能利用率周数据统计" : "https://list.oilchem.net/188/43717/", "中国尿素产量分省份月数据统计" : "https://list.oilchem.net/188/43719/", //"中国尿素进出口量数据总体分析" : "https://list.oilchem.net/188/5315/", } func NiaoSuList(num int) (err error) { for k, v := range NiaoSuListMap { for i := 1; i < num; i++ { listUrl := v + fmt.Sprintf("%d.html",i) fmt.Println("listUrl:",listUrl) htm, e := FetchPageHtml(listUrl) if e != nil { err = e utils.FileLog.Error(fmt.Sprintf("FetchPageHtml err:%v",err)) fmt.Println("FetchPageHtml err",err) return } err = AnalysisOilchemList(htm, k) if err != nil { utils.FileLog.Error(fmt.Sprintf("AnalysisOilchemList err:%v",err)) fmt.Println("AnalysisOilchemList err",err) return } time.Sleep(2*time.Second) } } return } // 中国尿素分原料产能利用率周数据统计 func AnalysisOilchemNiaoSu1(htm []byte) (err error) { if len(htm) == 0 { utils.FileLog.Info("htm empty") return } doc, e := goquery.NewDocumentFromReader(strings.NewReader(string(htm))) if e != nil { err = fmt.Errorf("NewDocumentFromReader err: %v", e) return } titleList := make([]string, 0) unitList := make([]string, 0) doc.Find("p").Each(func(i int, selection *goquery.Selection) { ptext := selection.Text() if strings.Contains(ptext, "单位:") { unit := strings.Replace(ptext, "单位:", "", -1) //fmt.Println("unit:",unit) unitList = append(unitList, unit) } if strings.Contains(ptext, "中国尿素分原料产能利用率周数据统计") { title := ptext //fmt.Println("title:",title) titleList = append(titleList, title) } }) area := "" title := doc.Find("h2").Text() fmt.Println("title:",title) createTimeStr := doc.Find("h2").Next().Text() createTimeStr = strings.TrimLeft(createTimeStr,"发布时间:") createTimeStrIndex := strings.Index(createTimeStr,"来源:") createTimeStr = createTimeStr[:createTimeStrIndex] createTimeStr = strings.TrimSpace(createTimeStr) createTime, err := time.Parse(utils.HlbFormatDateTimeNoSecond, createTimeStr) if err != nil { utils.FileLog.Error(fmt.Sprintf("time.Parse err:%v",err)) return } //fmt.Println("createTime:",createTime) dataTime := createTime.Format(utils.FormatDate) //fmt.Println("dataTime:",dataTime) indexList := make([]*models.BaseFromOilchemIndex, 0) doc.Find("tbody").Each(func(tableIndex int, table *goquery.Selection) { table.Find("tr").First().Each(func(ii int, table2 *goquery.Selection) { table.Find("td").Each(func(jj int, table3 *goquery.Selection) { text3 := table3.Text() text3 = strings.Replace(text3,"\n","",-1) text3 = strings.Replace(text3," ","",-1) if text3 == "上周" || text3 == "环比" || text3 == "地区" { return } //fmt.Println("table3:",text3) //utils.FileLog.Info(fmt.Sprintf("table3:%s",text3)) //fmt.Println("ii:",ii) //utils.FileLog.Info(fmt.Sprintf("ii:%d",ii)) //fmt.Println("jj:",jj) //utils.FileLog.Info(fmt.Sprintf("jj:%d",jj)) //fmt.Println("tableIndex:",tableIndex) //utils.FileLog.Info(fmt.Sprintf("tableIndex:%d",tableIndex)) if utils.ContainsChinese(text3) && text3 != "本周" { area = text3 return } if area == "" { return } value := text3 value = strings.TrimRight(value, "%") //valueF, e := strconv.ParseFloat(value, 64) //if e != nil { // err = e // utils.FileLog.Error(fmt.Sprintf("strconv.ParseFloat err:%v",e)) // return //} indexName := "中国尿素"+ area fmt.Println("indexName:",indexName) fmt.Println("valueF:",value) unit := "" if strings.Contains(area,"产量") { unit = "万吨" } else { unit = "%" } item := &models.BaseFromOilchemIndex{ IndexName: indexName, ClassifyId: 2, Unit: unit, Frequency: "周度", Describe: "", DataTime: dataTime, Value: value, Sort: 0, CreateTime: time.Now(), ModifyTime: time.Now(), IndexNameStr: indexName, MarketName: area, } indexList = append(indexList,item) area = "" }) }) }) // 写入数据库 err = PostHandleOilchem(indexList) if err != nil { utils.FileLog.Error(fmt.Sprintf("PostHandleOilchem err:%v",err)) fmt.Println("PostHandleOilchem err",err) return } return } // 中国尿素样本生产理论利润周数据统计 func AnalysisOilchemNiaoSu2(htm []byte) (err error) { if len(htm) == 0 { utils.FileLog.Info("htm empty") return } doc, e := goquery.NewDocumentFromReader(strings.NewReader(string(htm))) if e != nil { err = fmt.Errorf("NewDocumentFromReader err: %v", e) return } //titleList := make([]string, 0) unitList := make([]string, 0) doc.Find("p").Each(func(i int, selection *goquery.Selection) { ptext := selection.Text() if strings.Contains(ptext, "单位:") { unit := strings.Replace(ptext, "单位:", "", -1) //fmt.Println("unit:",unit) unitList = append(unitList, unit) } //if strings.Contains(ptext, "中国尿素样本生产理论利润周数据统计") { // title := ptext // //fmt.Println("title:",title) // titleList = append(titleList, title) //} }) area := "" title := doc.Find("h2").Text() fmt.Println("title:",title) createTimeStr := doc.Find("h2").Next().Text() createTimeStr = strings.TrimLeft(createTimeStr,"发布时间:") createTimeStrIndex := strings.Index(createTimeStr,"来源:") createTimeStr = createTimeStr[:createTimeStrIndex] createTimeStr = strings.TrimSpace(createTimeStr) createTime, err := time.Parse(utils.HlbFormatDateTimeNoSecond, createTimeStr) if err != nil { utils.FileLog.Error(fmt.Sprintf("time.Parse err:%v",err)) return } //fmt.Println("createTime:",createTime) dataTime := createTime.Format(utils.FormatDate) //fmt.Println("dataTime:",dataTime) indexList := make([]*models.BaseFromOilchemIndex, 0) doc.Find("tbody").Each(func(tableIndex int, table *goquery.Selection) { table.Find("tr").First().Each(func(ii int, table2 *goquery.Selection) { table.Find("td").Each(func(jj int, table3 *goquery.Selection) { text3 := table3.Text() text3 = strings.Replace(text3,"\n","",-1) text3 = strings.Replace(text3," ","",-1) if text3 == "上周" || text3 == "环比" || text3 == "地区" { return } //fmt.Println("table3:",text3) //utils.FileLog.Info(fmt.Sprintf("table3:%s",text3)) //fmt.Println("ii:",ii) //utils.FileLog.Info(fmt.Sprintf("ii:%d",ii)) //fmt.Println("jj:",jj) //utils.FileLog.Info(fmt.Sprintf("jj:%d",jj)) //fmt.Println("tableIndex:",tableIndex) //utils.FileLog.Info(fmt.Sprintf("tableIndex:%d",tableIndex)) if utils.ContainsChinese(text3) && text3 != "本周" { area = text3 return } if area == "" { return } value := text3 value = strings.TrimRight(value, "%") //valueF, e := strconv.ParseFloat(value, 64) //if e != nil { // err = e // utils.FileLog.Error(fmt.Sprintf("strconv.ParseFloat err:%v",e)) // return //} indexName := "中国尿素样本分工艺理论利润"+"("+ area+")" fmt.Println("indexName:",indexName) fmt.Println("valueF:",value) //unit := "" //if strings.Contains(area,"产量") { // unit = "万吨" //} else { // unit = "%" //} item := &models.BaseFromOilchemIndex{ IndexName: indexName, ClassifyId: 2, Unit: "元/吨", Frequency: "周度", Describe: "", DataTime: dataTime, Value: value, Sort: 0, CreateTime: time.Now(), ModifyTime: time.Now(), IndexNameStr: indexName, MarketName: area, } indexList = append(indexList,item) area = "" }) }) }) // 写入数据库 err = PostHandleOilchem(indexList) if err != nil { utils.FileLog.Error(fmt.Sprintf("PostHandleOilchem err:%v",err)) fmt.Println("PostHandleOilchem err",err) return } return } // 中国尿素样本港口库存周数据统计 func AnalysisOilchemNiaoSu3(htm []byte) (err error) { if len(htm) == 0 { utils.FileLog.Info("htm empty") return } doc, e := goquery.NewDocumentFromReader(strings.NewReader(string(htm))) if e != nil { err = fmt.Errorf("NewDocumentFromReader err: %v", e) return } //titleList := make([]string, 0) unitList := make([]string, 0) doc.Find("p").Each(func(i int, selection *goquery.Selection) { ptext := selection.Text() if strings.Contains(ptext, "单位:") { unit := strings.Replace(ptext, "单位:", "", -1) //fmt.Println("unit:",unit) unitList = append(unitList, unit) } //if strings.Contains(ptext, "中国尿素样本生产理论利润周数据统计") { // title := ptext // //fmt.Println("title:",title) // titleList = append(titleList, title) //} }) area := "" title := doc.Find("h2").Text() fmt.Println("title:",title) createTimeStr := doc.Find("h2").Next().Text() createTimeStr = strings.TrimLeft(createTimeStr,"发布时间:") createTimeStrIndex := strings.Index(createTimeStr,"来源:") createTimeStr = createTimeStr[:createTimeStrIndex] createTimeStr = strings.TrimSpace(createTimeStr) createTime, err := time.Parse(utils.HlbFormatDateTimeNoSecond, createTimeStr) if err != nil { utils.FileLog.Error(fmt.Sprintf("time.Parse err:%v",err)) return } //fmt.Println("createTime:",createTime) dataTime := createTime.Format(utils.FormatDate) //fmt.Println("dataTime:",dataTime) indexList := make([]*models.BaseFromOilchemIndex, 0) doc.Find("tbody").Each(func(tableIndex int, table *goquery.Selection) { table.Find("tr").First().Each(func(ii int, table2 *goquery.Selection) { table.Find("td").Each(func(jj int, table3 *goquery.Selection) { text3 := table3.Text() text3 = strings.Replace(text3,"\n","",-1) text3 = strings.Replace(text3," ","",-1) if text3 == "上周" || text3 == "环比" || text3 == "地区" { return } //fmt.Println("table3:",text3) //utils.FileLog.Info(fmt.Sprintf("table3:%s",text3)) //fmt.Println("ii:",ii) //utils.FileLog.Info(fmt.Sprintf("ii:%d",ii)) //fmt.Println("jj:",jj) //utils.FileLog.Info(fmt.Sprintf("jj:%d",jj)) //fmt.Println("tableIndex:",tableIndex) //utils.FileLog.Info(fmt.Sprintf("tableIndex:%d",tableIndex)) if utils.ContainsChinese(text3) && text3 != "本周" { area = text3 return } if area == "" { return } value := text3 value = strings.TrimRight(value, "%") //valueF, e := strconv.ParseFloat(value, 64) //if e != nil { // err = e // utils.FileLog.Error(fmt.Sprintf("strconv.ParseFloat err:%v",e)) // return //} indexName := "中国尿素样本港口库存"+"("+ area+")" fmt.Println("indexName:",indexName) fmt.Println("valueF:",value) //unit := "" //if strings.Contains(area,"产量") { // unit = "万吨" //} else { // unit = "%" //} item := &models.BaseFromOilchemIndex{ IndexName: indexName, ClassifyId: 2, Unit: "万吨", Frequency: "周度", Describe: "", DataTime: dataTime, Value: value, Sort: 0, CreateTime: time.Now(), ModifyTime: time.Now(), IndexNameStr: indexName, MarketName: area, } indexList = append(indexList,item) area = "" }) }) }) // 写入数据库 err = PostHandleOilchem(indexList) if err != nil { utils.FileLog.Error(fmt.Sprintf("PostHandleOilchem err:%v",err)) fmt.Println("PostHandleOilchem err",err) return } return } // 中国尿素产量周数据分析 func AnalysisOilchemNiaoSu4(htm []byte) (err error) { if len(htm) == 0 { utils.FileLog.Info("htm empty") return } doc, e := goquery.NewDocumentFromReader(strings.NewReader(string(htm))) if e != nil { err = fmt.Errorf("NewDocumentFromReader err: %v", e) return } //titleList := make([]string, 0) unitList := make([]string, 0) doc.Find("p").Each(func(i int, selection *goquery.Selection) { ptext := selection.Text() if strings.Contains(ptext, "单位:") { unit := strings.Replace(ptext, "单位:", "", -1) //fmt.Println("unit:",unit) unitList = append(unitList, unit) } //if strings.Contains(ptext, "中国尿素样本生产理论利润周数据统计") { // title := ptext // //fmt.Println("title:",title) // titleList = append(titleList, title) //} }) area := "" title := doc.Find("h2").Text() fmt.Println("title:",title) createTimeStr := doc.Find("h2").Next().Text() createTimeStr = strings.TrimLeft(createTimeStr,"发布时间:") createTimeStrIndex := strings.Index(createTimeStr,"来源:") createTimeStr = createTimeStr[:createTimeStrIndex] createTimeStr = strings.TrimSpace(createTimeStr) createTime, err := time.Parse(utils.HlbFormatDateTimeNoSecond, createTimeStr) if err != nil { utils.FileLog.Error(fmt.Sprintf("time.Parse err:%v",err)) return } //fmt.Println("createTime:",createTime) dataTime := createTime.Format(utils.FormatDate) //fmt.Println("dataTime:",dataTime) indexList := make([]*models.BaseFromOilchemIndex, 0) doc.Find("tbody").Each(func(tableIndex int, table *goquery.Selection) { table.Find("tr").First().Each(func(ii int, table2 *goquery.Selection) { table.Find("td").Each(func(jj int, table3 *goquery.Selection) { text3 := table3.Text() text3 = strings.Replace(text3,"\n","",-1) text3 = strings.Replace(text3," ","",-1) if text3 == "上周" || text3 == "环比" || text3 == "地区" { return } //fmt.Println("table3:",text3) //utils.FileLog.Info(fmt.Sprintf("table3:%s",text3)) //fmt.Println("ii:",ii) //utils.FileLog.Info(fmt.Sprintf("ii:%d",ii)) //fmt.Println("jj:",jj) //utils.FileLog.Info(fmt.Sprintf("jj:%d",jj)) //fmt.Println("tableIndex:",tableIndex) //utils.FileLog.Info(fmt.Sprintf("tableIndex:%d",tableIndex)) if utils.ContainsChinese(text3) && text3 != "本周" { area = text3 return } if area == "" { return } value := text3 value = strings.TrimRight(value, "%") //valueF, e := strconv.ParseFloat(value, 64) //if e != nil { // err = e // utils.FileLog.Error(fmt.Sprintf("strconv.ParseFloat err:%v",e)) // return //} indexName := "中国尿素产量"+"("+ area+")" fmt.Println("indexName:",indexName) fmt.Println("valueF:",value) //unit := "" //if strings.Contains(area,"产量") { // unit = "万吨" //} else { // unit = "%" //} item := &models.BaseFromOilchemIndex{ IndexName: indexName, ClassifyId: 2, Unit: "万吨", Frequency: "周度", Describe: "", DataTime: dataTime, Value: value, Sort: 0, CreateTime: time.Now(), ModifyTime: time.Now(), IndexNameStr: indexName, MarketName: area, } indexList = append(indexList,item) area = "" }) }) }) // 写入数据库 err = PostHandleOilchem(indexList) if err != nil { utils.FileLog.Error(fmt.Sprintf("PostHandleOilchem err:%v",err)) fmt.Println("PostHandleOilchem err",err) return } return } // 中国尿素企业库存 func AnalysisOilchemNiaoSu5(htm []byte) (err error) { if len(htm) == 0 { utils.FileLog.Info("htm empty") return } doc, e := goquery.NewDocumentFromReader(strings.NewReader(string(htm))) if e != nil { err = fmt.Errorf("NewDocumentFromReader err: %v", e) return } //titleList := make([]string, 0) unitList := make([]string, 0) doc.Find("p").Each(func(i int, selection *goquery.Selection) { ptext := selection.Text() if strings.Contains(ptext, "单位:") { unit := strings.Replace(ptext, "单位:", "", -1) //fmt.Println("unit:",unit) unitList = append(unitList, unit) } //if strings.Contains(ptext, "中国尿素样本生产理论利润周数据统计") { // title := ptext // //fmt.Println("title:",title) // titleList = append(titleList, title) //} }) area := "" title := doc.Find("h2").Text() fmt.Println("title:",title) createTimeStr := doc.Find("h2").Next().Text() createTimeStr = strings.TrimLeft(createTimeStr,"发布时间:") createTimeStrIndex := strings.Index(createTimeStr,"来源:") createTimeStr = createTimeStr[:createTimeStrIndex] createTimeStr = strings.TrimSpace(createTimeStr) createTime, err := time.Parse(utils.HlbFormatDateTimeNoSecond, createTimeStr) if err != nil { utils.FileLog.Error(fmt.Sprintf("time.Parse err:%v",err)) return } //fmt.Println("createTime:",createTime) dataTime := createTime.Format(utils.FormatDate) //fmt.Println("dataTime:",dataTime) indexList := make([]*models.BaseFromOilchemIndex, 0) doc.Find("tbody").Each(func(tableIndex int, table *goquery.Selection) { table.Find("tr").First().Each(func(ii int, table2 *goquery.Selection) { table.Find("td").Each(func(jj int, table3 *goquery.Selection) { text3 := table3.Text() text3 = strings.Replace(text3,"\n","",-1) text3 = strings.Replace(text3," ","",-1) if text3 == "上周" || text3 == "环比" || text3 == "地区" { return } //fmt.Println("table3:",text3) //utils.FileLog.Info(fmt.Sprintf("table3:%s",text3)) //fmt.Println("ii:",ii) //utils.FileLog.Info(fmt.Sprintf("ii:%d",ii)) //fmt.Println("jj:",jj) //utils.FileLog.Info(fmt.Sprintf("jj:%d",jj)) //fmt.Println("tableIndex:",tableIndex) //utils.FileLog.Info(fmt.Sprintf("tableIndex:%d",tableIndex)) if utils.ContainsChinese(text3) && text3 != "本周" { area = text3 return } if area == "" { return } value := text3 value = strings.TrimRight(value, "%") //valueF, e := strconv.ParseFloat(value, 64) //if e != nil { // err = e // utils.FileLog.Error(fmt.Sprintf("strconv.ParseFloat err:%v",e)) // return //} indexName := "中国尿素企业库存" fmt.Println("indexName:",indexName) fmt.Println("valueF:",value) //unit := "" //if strings.Contains(area,"产量") { // unit = "万吨" //} else { // unit = "%" //} item := &models.BaseFromOilchemIndex{ IndexName: indexName, ClassifyId: 2, Unit: "万吨", Frequency: "周度", Describe: "", DataTime: dataTime, Value: value, Sort: 0, CreateTime: time.Now(), ModifyTime: time.Now(), IndexNameStr: indexName, MarketName: "", } indexList = append(indexList,item) area = "" }) }) }) // 写入数据库 err = PostHandleOilchem(indexList) if err != nil { utils.FileLog.Error(fmt.Sprintf("PostHandleOilchem err:%v",err)) fmt.Println("PostHandleOilchem err",err) return } return } // 中国尿素样本港口库存周数据分析 func AnalysisOilchemNiaoSu6(htm []byte) (err error) { if len(htm) == 0 { utils.FileLog.Info("htm empty") return } doc, e := goquery.NewDocumentFromReader(strings.NewReader(string(htm))) if e != nil { err = fmt.Errorf("NewDocumentFromReader err: %v", e) return } //titleList := make([]string, 0) unitList := make([]string, 0) doc.Find("p").Each(func(i int, selection *goquery.Selection) { ptext := selection.Text() if strings.Contains(ptext, "单位:") { unit := strings.Replace(ptext, "单位:", "", -1) //fmt.Println("unit:",unit) unitList = append(unitList, unit) } //if strings.Contains(ptext, "中国尿素样本生产理论利润周数据统计") { // title := ptext // //fmt.Println("title:",title) // titleList = append(titleList, title) //} }) area := "" title := doc.Find("h2").Text() fmt.Println("title:",title) createTimeStr := doc.Find("h2").Next().Text() createTimeStr = strings.TrimLeft(createTimeStr,"发布时间:") createTimeStrIndex := strings.Index(createTimeStr,"来源:") createTimeStr = createTimeStr[:createTimeStrIndex] createTimeStr = strings.TrimSpace(createTimeStr) createTime, err := time.Parse(utils.HlbFormatDateTimeNoSecond, createTimeStr) if err != nil { utils.FileLog.Error(fmt.Sprintf("time.Parse err:%v",err)) return } //fmt.Println("createTime:",createTime) dataTime := createTime.Format(utils.FormatDate) //fmt.Println("dataTime:",dataTime) indexList := make([]*models.BaseFromOilchemIndex, 0) doc.Find("tbody").Each(func(tableIndex int, table *goquery.Selection) { table.Find("tr").First().Each(func(ii int, table2 *goquery.Selection) { table.Find("td").Each(func(jj int, table3 *goquery.Selection) { text3 := table3.Text() text3 = strings.Replace(text3,"\n","",-1) text3 = strings.Replace(text3," ","",-1) if text3 == "上周" || text3 == "环比" || text3 == "地区" { return } //fmt.Println("table3:",text3) //utils.FileLog.Info(fmt.Sprintf("table3:%s",text3)) //fmt.Println("ii:",ii) //utils.FileLog.Info(fmt.Sprintf("ii:%d",ii)) //fmt.Println("jj:",jj) //utils.FileLog.Info(fmt.Sprintf("jj:%d",jj)) //fmt.Println("tableIndex:",tableIndex) //utils.FileLog.Info(fmt.Sprintf("tableIndex:%d",tableIndex)) if utils.ContainsChinese(text3) && text3 != "本周" { area = text3 return } if area == "" { return } value := text3 value = strings.TrimRight(value, "%") //valueF, e := strconv.ParseFloat(value, 64) //if e != nil { // err = e // utils.FileLog.Error(fmt.Sprintf("strconv.ParseFloat err:%v",e)) // return //} indexName := "中国尿素样本分港口库存" + "(" + area + ")" fmt.Println("indexName:",indexName) fmt.Println("valueF:",value) //unit := "" //if strings.Contains(area,"产量") { // unit = "万吨" //} else { // unit = "%" //} item := &models.BaseFromOilchemIndex{ IndexName: indexName, ClassifyId: 2, Unit: "万吨", Frequency: "周度", Describe: "", DataTime: dataTime, Value: value, Sort: 0, CreateTime: time.Now(), ModifyTime: time.Now(), IndexNameStr: indexName, MarketName: area, } indexList = append(indexList,item) area = "" }) }) }) // 写入数据库 err = PostHandleOilchem(indexList) if err != nil { utils.FileLog.Error(fmt.Sprintf("PostHandleOilchem err:%v",err)) fmt.Println("PostHandleOilchem err",err) return } return } // 中国尿素企业预收订单周数据分析 func AnalysisOilchemNiaoSu7(htm []byte) (err error) { if len(htm) == 0 { utils.FileLog.Info("htm empty") return } doc, e := goquery.NewDocumentFromReader(strings.NewReader(string(htm))) if e != nil { err = fmt.Errorf("NewDocumentFromReader err: %v", e) return } //titleList := make([]string, 0) unitList := make([]string, 0) doc.Find("p").Each(func(i int, selection *goquery.Selection) { ptext := selection.Text() if strings.Contains(ptext, "单位:") { unit := strings.Replace(ptext, "单位:", "", -1) //fmt.Println("unit:",unit) unitList = append(unitList, unit) } //if strings.Contains(ptext, "中国尿素样本生产理论利润周数据统计") { // title := ptext // //fmt.Println("title:",title) // titleList = append(titleList, title) //} }) area := "" title := doc.Find("h2").Text() fmt.Println("title:",title) createTimeStr := doc.Find("h2").Next().Text() createTimeStr = strings.TrimLeft(createTimeStr,"发布时间:") createTimeStrIndex := strings.Index(createTimeStr,"来源:") createTimeStr = createTimeStr[:createTimeStrIndex] createTimeStr = strings.TrimSpace(createTimeStr) createTime, err := time.Parse(utils.HlbFormatDateTimeNoSecond, createTimeStr) if err != nil { utils.FileLog.Error(fmt.Sprintf("time.Parse err:%v",err)) return } //fmt.Println("createTime:",createTime) dataTime := createTime.Format(utils.FormatDate) //fmt.Println("dataTime:",dataTime) indexList := make([]*models.BaseFromOilchemIndex, 0) doc.Find("tbody").Each(func(tableIndex int, table *goquery.Selection) { table.Find("tr").First().Each(func(ii int, table2 *goquery.Selection) { table.Find("td").Each(func(jj int, table3 *goquery.Selection) { text3 := table3.Text() text3 = strings.Replace(text3,"\n","",-1) text3 = strings.Replace(text3," ","",-1) if text3 == "上周" || text3 == "环比" || text3 == "地区" { return } //fmt.Println("table3:",text3) //utils.FileLog.Info(fmt.Sprintf("table3:%s",text3)) //fmt.Println("ii:",ii) //utils.FileLog.Info(fmt.Sprintf("ii:%d",ii)) //fmt.Println("jj:",jj) //utils.FileLog.Info(fmt.Sprintf("jj:%d",jj)) //fmt.Println("tableIndex:",tableIndex) //utils.FileLog.Info(fmt.Sprintf("tableIndex:%d",tableIndex)) if utils.ContainsChinese(text3) && text3 != "本周" { area = text3 return } if area == "" { return } value := text3 value = strings.TrimRight(value, "%") //valueF, e := strconv.ParseFloat(value, 64) //if e != nil { // err = e // utils.FileLog.Error(fmt.Sprintf("strconv.ParseFloat err:%v",e)) // return //} indexName := "中国尿素企业预收订单" fmt.Println("indexName:",indexName) fmt.Println("valueF:",value) //unit := "" //if strings.Contains(area,"产量") { // unit = "万吨" //} else { // unit = "%" //} item := &models.BaseFromOilchemIndex{ IndexName: indexName, ClassifyId: 2, Unit: "日", Frequency: "周度", Describe: "", DataTime: dataTime, Value: value, Sort: 0, CreateTime: time.Now(), ModifyTime: time.Now(), IndexNameStr: indexName, MarketName: area, } indexList = append(indexList,item) area = "" }) }) }) // 写入数据库 err = PostHandleOilchem(indexList) if err != nil { utils.FileLog.Error(fmt.Sprintf("PostHandleOilchem err:%v",err)) fmt.Println("PostHandleOilchem err",err) return } return } // 中国尿素样本理论利润周数据分析 func AnalysisOilchemNiaoSu8(htm []byte) (err error) { if len(htm) == 0 { utils.FileLog.Info("htm empty") return } doc, e := goquery.NewDocumentFromReader(strings.NewReader(string(htm))) if e != nil { err = fmt.Errorf("NewDocumentFromReader err: %v", e) return } //titleList := make([]string, 0) unitList := make([]string, 0) doc.Find("p").Each(func(i int, selection *goquery.Selection) { ptext := selection.Text() if strings.Contains(ptext, "单位:") { unit := strings.Replace(ptext, "单位:", "", -1) //fmt.Println("unit:",unit) unitList = append(unitList, unit) } //if strings.Contains(ptext, "中国尿素样本生产理论利润周数据统计") { // title := ptext // //fmt.Println("title:",title) // titleList = append(titleList, title) //} }) area := "" title := doc.Find("h2").Text() fmt.Println("title:",title) createTimeStr := doc.Find("h2").Next().Text() createTimeStr = strings.TrimLeft(createTimeStr,"发布时间:") createTimeStrIndex := strings.Index(createTimeStr,"来源:") createTimeStr = createTimeStr[:createTimeStrIndex] createTimeStr = strings.TrimSpace(createTimeStr) createTime, err := time.Parse(utils.HlbFormatDateTimeNoSecond, createTimeStr) if err != nil { utils.FileLog.Error(fmt.Sprintf("time.Parse err:%v",err)) return } //fmt.Println("createTime:",createTime) dataTime := createTime.Format(utils.FormatDate) //fmt.Println("dataTime:",dataTime) indexList := make([]*models.BaseFromOilchemIndex, 0) doc.Find("tbody").Each(func(tableIndex int, table *goquery.Selection) { table.Find("tr").First().Each(func(ii int, table2 *goquery.Selection) { table.Find("td").Each(func(jj int, table3 *goquery.Selection) { text3 := table3.Text() text3 = strings.Replace(text3,"\n","",-1) text3 = strings.Replace(text3," ","",-1) if text3 == "上周" || text3 == "环比" || text3 == "地区" { return } //fmt.Println("table3:",text3) //utils.FileLog.Info(fmt.Sprintf("table3:%s",text3)) //fmt.Println("ii:",ii) //utils.FileLog.Info(fmt.Sprintf("ii:%d",ii)) //fmt.Println("jj:",jj) //utils.FileLog.Info(fmt.Sprintf("jj:%d",jj)) //fmt.Println("tableIndex:",tableIndex) //utils.FileLog.Info(fmt.Sprintf("tableIndex:%d",tableIndex)) if utils.ContainsChinese(text3) && text3 != "本周" { area = text3 return } if area == "" { return } value := text3 value = strings.TrimRight(value, "%") //valueF, e := strconv.ParseFloat(value, 64) //if e != nil { // err = e // utils.FileLog.Error(fmt.Sprintf("strconv.ParseFloat err:%v",e)) // return //} indexName := "中国尿素企业预收订单" fmt.Println("indexName:",indexName) fmt.Println("valueF:",value) //unit := "" //if strings.Contains(area,"产量") { // unit = "万吨" //} else { // unit = "%" //} item := &models.BaseFromOilchemIndex{ IndexName: indexName, ClassifyId: 2, Unit: "日", Frequency: "周度", Describe: "", DataTime: dataTime, Value: value, Sort: 0, CreateTime: time.Now(), ModifyTime: time.Now(), IndexNameStr: indexName, MarketName: area, } indexList = append(indexList,item) area = "" }) }) }) // 写入数据库 err = PostHandleOilchem(indexList) if err != nil { utils.FileLog.Error(fmt.Sprintf("PostHandleOilchem err:%v",err)) fmt.Println("PostHandleOilchem err",err) return } return } // 中国复合肥产能利用率周数据统计 func AnalysisOilchemNiaoSu9(htm []byte) (err error) { if len(htm) == 0 { utils.FileLog.Info("htm empty") return } doc, e := goquery.NewDocumentFromReader(strings.NewReader(string(htm))) if e != nil { err = fmt.Errorf("NewDocumentFromReader err: %v", e) return } //titleList := make([]string, 0) unitList := make([]string, 0) doc.Find("p").Each(func(i int, selection *goquery.Selection) { ptext := selection.Text() if strings.Contains(ptext, "单位:") { unit := strings.Replace(ptext, "单位:", "", -1) //fmt.Println("unit:",unit) unitList = append(unitList, unit) } //if strings.Contains(ptext, "中国尿素样本生产理论利润周数据统计") { // title := ptext // //fmt.Println("title:",title) // titleList = append(titleList, title) //} }) area := "" title := doc.Find("h2").Text() fmt.Println("title:",title) createTimeStr := doc.Find("h2").Next().Text() createTimeStr = strings.TrimLeft(createTimeStr,"发布时间:") createTimeStrIndex := strings.Index(createTimeStr,"来源:") createTimeStr = createTimeStr[:createTimeStrIndex] createTimeStr = strings.TrimSpace(createTimeStr) createTime, err := time.Parse(utils.HlbFormatDateTimeNoSecond, createTimeStr) if err != nil { utils.FileLog.Error(fmt.Sprintf("time.Parse err:%v",err)) return } //fmt.Println("createTime:",createTime) dataTime := createTime.Format(utils.FormatDate) //fmt.Println("dataTime:",dataTime) indexList := make([]*models.BaseFromOilchemIndex, 0) doc.Find("tbody").Each(func(tableIndex int, table *goquery.Selection) { table.Find("tr").First().Each(func(ii int, table2 *goquery.Selection) { table.Find("td").Each(func(jj int, table3 *goquery.Selection) { text3 := table3.Text() text3 = strings.Replace(text3,"\n","",-1) text3 = strings.Replace(text3," ","",-1) if text3 == "上周" || text3 == "环比" || text3 == "地区" { return } //fmt.Println("table3:",text3) //utils.FileLog.Info(fmt.Sprintf("table3:%s",text3)) //fmt.Println("ii:",ii) //utils.FileLog.Info(fmt.Sprintf("ii:%d",ii)) //fmt.Println("jj:",jj) //utils.FileLog.Info(fmt.Sprintf("jj:%d",jj)) //fmt.Println("tableIndex:",tableIndex) //utils.FileLog.Info(fmt.Sprintf("tableIndex:%d",tableIndex)) if utils.ContainsChinese(text3) && text3 != "本周" { area = text3 return } if area == "" { return } value := text3 value = strings.TrimRight(value, "%") //valueF, e := strconv.ParseFloat(value, 64) //if e != nil { // err = e // utils.FileLog.Error(fmt.Sprintf("strconv.ParseFloat err:%v",e)) // return //} indexName := "中国复合肥产能利用率" fmt.Println("indexName:",indexName) fmt.Println("valueF:",value) //unit := "" //if strings.Contains(area,"产量") { // unit = "万吨" //} else { // unit = "%" //} item := &models.BaseFromOilchemIndex{ IndexName: indexName, ClassifyId: 2, Unit: "%", Frequency: "周度", Describe: "", DataTime: dataTime, Value: value, Sort: 0, CreateTime: time.Now(), ModifyTime: time.Now(), IndexNameStr: indexName, MarketName: area, } indexList = append(indexList,item) area = "" }) }) }) // 写入数据库 err = PostHandleOilchem(indexList) if err != nil { utils.FileLog.Error(fmt.Sprintf("PostHandleOilchem err:%v",err)) fmt.Println("PostHandleOilchem err",err) return } return } // 中国三聚氰胺产能利用率周数据统计 func AnalysisOilchemNiaoSu10(htm []byte) (err error) { if len(htm) == 0 { utils.FileLog.Info("htm empty") return } doc, e := goquery.NewDocumentFromReader(strings.NewReader(string(htm))) if e != nil { err = fmt.Errorf("NewDocumentFromReader err: %v", e) return } //titleList := make([]string, 0) unitList := make([]string, 0) doc.Find("p").Each(func(i int, selection *goquery.Selection) { ptext := selection.Text() if strings.Contains(ptext, "单位:") { unit := strings.Replace(ptext, "单位:", "", -1) //fmt.Println("unit:",unit) unitList = append(unitList, unit) } //if strings.Contains(ptext, "中国尿素样本生产理论利润周数据统计") { // title := ptext // //fmt.Println("title:",title) // titleList = append(titleList, title) //} }) area := "" title := doc.Find("h2").Text() fmt.Println("title:",title) createTimeStr := doc.Find("h2").Next().Text() createTimeStr = strings.TrimLeft(createTimeStr,"发布时间:") createTimeStrIndex := strings.Index(createTimeStr,"来源:") createTimeStr = createTimeStr[:createTimeStrIndex] createTimeStr = strings.TrimSpace(createTimeStr) createTime, err := time.Parse(utils.HlbFormatDateTimeNoSecond, createTimeStr) if err != nil { utils.FileLog.Error(fmt.Sprintf("time.Parse err:%v",err)) return } //fmt.Println("createTime:",createTime) dataTime := createTime.Format(utils.FormatDate) //fmt.Println("dataTime:",dataTime) indexList := make([]*models.BaseFromOilchemIndex, 0) doc.Find("tbody").Each(func(tableIndex int, table *goquery.Selection) { table.Find("tr").First().Each(func(ii int, table2 *goquery.Selection) { table.Find("td").Each(func(jj int, table3 *goquery.Selection) { text3 := table3.Text() text3 = strings.Replace(text3,"\n","",-1) text3 = strings.Replace(text3," ","",-1) if text3 == "上周" || text3 == "环比" || text3 == "地区" { return } //fmt.Println("table3:",text3) //utils.FileLog.Info(fmt.Sprintf("table3:%s",text3)) //fmt.Println("ii:",ii) //utils.FileLog.Info(fmt.Sprintf("ii:%d",ii)) //fmt.Println("jj:",jj) //utils.FileLog.Info(fmt.Sprintf("jj:%d",jj)) //fmt.Println("tableIndex:",tableIndex) //utils.FileLog.Info(fmt.Sprintf("tableIndex:%d",tableIndex)) if utils.ContainsChinese(text3) && text3 != "本周" { area = text3 return } if area == "" { return } value := text3 value = strings.TrimRight(value, "%") //valueF, e := strconv.ParseFloat(value, 64) //if e != nil { // err = e // utils.FileLog.Error(fmt.Sprintf("strconv.ParseFloat err:%v",e)) // return //} indexName := "中国三聚氰胺产能利用率" fmt.Println("indexName:",indexName) fmt.Println("valueF:",value) //unit := "" //if strings.Contains(area,"产量") { // unit = "万吨" //} else { // unit = "%" //} item := &models.BaseFromOilchemIndex{ IndexName: indexName, ClassifyId: 2, Unit: "%", Frequency: "周度", Describe: "", DataTime: dataTime, Value: value, Sort: 0, CreateTime: time.Now(), ModifyTime: time.Now(), IndexNameStr: indexName, MarketName: area, } indexList = append(indexList,item) area = "" }) }) }) // 写入数据库 err = PostHandleOilchem(indexList) if err != nil { utils.FileLog.Error(fmt.Sprintf("PostHandleOilchem err:%v",err)) fmt.Println("PostHandleOilchem err",err) return } return } // 中国尿素产量分省份月数据统计 func AnalysisOilchemNiaoSu11(htm []byte) (err error) { if len(htm) == 0 { utils.FileLog.Info("htm empty") return } doc, e := goquery.NewDocumentFromReader(strings.NewReader(string(htm))) if e != nil { err = fmt.Errorf("NewDocumentFromReader err: %v", e) return } //titleList := make([]string, 0) unitList := make([]string, 0) doc.Find("p").Each(func(i int, selection *goquery.Selection) { ptext := selection.Text() if strings.Contains(ptext, "单位:") { unit := strings.Replace(ptext, "单位:", "", -1) //fmt.Println("unit:",unit) unitList = append(unitList, unit) } //if strings.Contains(ptext, "中国尿素样本生产理论利润周数据统计") { // title := ptext // //fmt.Println("title:",title) // titleList = append(titleList, title) //} }) area := "" title := doc.Find("h2").Text() fmt.Println("title:",title) createTimeStr := doc.Find("h2").Next().Text() createTimeStr = strings.TrimLeft(createTimeStr,"发布时间:") createTimeStrIndex := strings.Index(createTimeStr,"来源:") createTimeStr = createTimeStr[:createTimeStrIndex] createTimeStr = strings.TrimSpace(createTimeStr) createTime, err := time.Parse(utils.HlbFormatDateTimeNoSecond, createTimeStr) if err != nil { utils.FileLog.Error(fmt.Sprintf("time.Parse err:%v",err)) return } //fmt.Println("createTime:",createTime) dataTime := createTime.Format(utils.FormatDate) //fmt.Println("dataTime:",dataTime) indexList := make([]*models.BaseFromOilchemIndex, 0) doc.Find("tbody").Each(func(tableIndex int, table *goquery.Selection) { table.Find("tr").First().Each(func(ii int, table2 *goquery.Selection) { table.Find("td").Each(func(jj int, table3 *goquery.Selection) { text3 := table3.Text() text3 = strings.Replace(text3,"\n","",-1) text3 = strings.Replace(text3," ","",-1) if text3 == "上周" || text3 == "环比" || text3 == "地区" { return } //fmt.Println("table3:",text3) //utils.FileLog.Info(fmt.Sprintf("table3:%s",text3)) //fmt.Println("ii:",ii) //utils.FileLog.Info(fmt.Sprintf("ii:%d",ii)) //fmt.Println("jj:",jj) //utils.FileLog.Info(fmt.Sprintf("jj:%d",jj)) //fmt.Println("tableIndex:",tableIndex) //utils.FileLog.Info(fmt.Sprintf("tableIndex:%d",tableIndex)) if utils.ContainsChinese(text3) && text3 != "本周" { area = text3 return } if area == "" { return } value := text3 value = strings.TrimRight(value, "%") //valueF, e := strconv.ParseFloat(value, 64) //if e != nil { // err = e // utils.FileLog.Error(fmt.Sprintf("strconv.ParseFloat err:%v",e)) // return //} indexName := "中国尿素产量" + "(" + area + ")" fmt.Println("indexName:",indexName) fmt.Println("valueF:",value) //unit := "" //if strings.Contains(area,"产量") { // unit = "万吨" //} else { // unit = "%" //} item := &models.BaseFromOilchemIndex{ IndexName: indexName, ClassifyId: 2, Unit: "吨", Frequency: "周度", Describe: "", DataTime: dataTime, Value: value, Sort: 0, CreateTime: time.Now(), ModifyTime: time.Now(), IndexNameStr: indexName, MarketName: area, } indexList = append(indexList,item) area = "" }) }) }) // 写入数据库 err = PostHandleOilchem(indexList) if err != nil { utils.FileLog.Error(fmt.Sprintf("PostHandleOilchem err:%v",err)) fmt.Println("PostHandleOilchem err",err) return } return } // 中国尿素进出口量数据总体分析 todo func AnalysisOilchemNiaoSu12(htm []byte) (err error) { if len(htm) == 0 { utils.FileLog.Info("htm empty") return } doc, e := goquery.NewDocumentFromReader(strings.NewReader(string(htm))) if e != nil { err = fmt.Errorf("NewDocumentFromReader err: %v", e) return } //titleList := make([]string, 0) unitList := make([]string, 0) doc.Find("p").Each(func(i int, selection *goquery.Selection) { ptext := selection.Text() if strings.Contains(ptext, "单位:") { unit := strings.Replace(ptext, "单位:", "", -1) //fmt.Println("unit:",unit) unitList = append(unitList, unit) } //if strings.Contains(ptext, "中国尿素样本生产理论利润周数据统计") { // title := ptext // //fmt.Println("title:",title) // titleList = append(titleList, title) //} }) area := "" title := doc.Find("h2").Text() fmt.Println("title:",title) createTimeStr := doc.Find("h2").Next().Text() createTimeStr = strings.TrimLeft(createTimeStr,"发布时间:") createTimeStrIndex := strings.Index(createTimeStr,"来源:") createTimeStr = createTimeStr[:createTimeStrIndex] createTimeStr = strings.TrimSpace(createTimeStr) createTime, err := time.Parse(utils.HlbFormatDateTimeNoSecond, createTimeStr) if err != nil { utils.FileLog.Error(fmt.Sprintf("time.Parse err:%v",err)) return } //fmt.Println("createTime:",createTime) dataTime := createTime.Format(utils.FormatDate) //fmt.Println("dataTime:",dataTime) indexTitle := "中国尿素" indexList := make([]*models.BaseFromOilchemIndex, 0) doc.Find("tbody").Each(func(tableIndex int, table *goquery.Selection) { table.Find("tr").Each(func(ii int, table2 *goquery.Selection) { table2.Find("td").Each(func(jj int, table3 *goquery.Selection) { text3 := table3.Text() text3 = strings.Replace(text3,"\n","",-1) text3 = strings.Replace(text3," ","",-1) if text3 == "上周" || text3 == "环比" || text3 == "地区" { return } //fmt.Println("table3:",text3) if jj == 1 { indexTitle = "中国尿素进口量" } else if jj == 4 { indexTitle = "中国尿素出口量" } //utils.FileLog.Info(fmt.Sprintf("table3:%s",text3)) //fmt.Println("ii:",ii) //utils.FileLog.Info(fmt.Sprintf("ii:%d",ii)) //fmt.Println("jj:",jj) //utils.FileLog.Info(fmt.Sprintf("jj:%d",jj)) //fmt.Println("tableIndex:",tableIndex) //utils.FileLog.Info(fmt.Sprintf("tableIndex:%d",tableIndex)) if utils.ContainsChinese(text3) && text3 != "本周" { area = text3 return } if area == "" { return } fmt.Println("area:",area) if jj != 1 && jj != 4 { return } value := text3 value = strings.TrimRight(value, "%") //valueF, e := strconv.ParseFloat(value, 64) //if e != nil { // err = e // utils.FileLog.Error(fmt.Sprintf("strconv.ParseFloat err:%v",e)) // return //} indexName := indexTitle + "(" + area + ")" fmt.Println("indexName:",indexName) fmt.Println("valueF:",value) //unit := "" //if strings.Contains(area,"产量") { // unit = "万吨" //} else { // unit = "%" //} item := &models.BaseFromOilchemIndex{ IndexName: indexName, ClassifyId: 2, Unit: "吨", Frequency: "周度", Describe: "", DataTime: dataTime, Value: value, Sort: 0, CreateTime: time.Now(), ModifyTime: time.Now(), IndexNameStr: indexName, MarketName: area, } indexList = append(indexList,item) area = "" }) }) }) // 写入数据库 err = PostHandleOilchem(indexList) if err != nil { utils.FileLog.Error(fmt.Sprintf("PostHandleOilchem err:%v",err)) fmt.Println("PostHandleOilchem err",err) return } return }