Преглед на файлове

Merge branch 'bzq/coal_mail_listen' of eta_server/eta_crawler into master

baoziqiang преди 2 месеца
родител
ревизия
2c1177b1f1
променени са 13 файла, в които са добавени 1416 реда и са изтрити 10 реда
  1. 6 1
      go.mod
  2. 13 0
      go.sum
  3. 40 7
      models/base_from_coalmine.go
  4. 60 0
      services/base.go
  5. 622 0
      services/commodity_coal.go
  6. 88 0
      services/commodity_coal_watch.go
  7. 66 0
      services/email/mail.go
  8. 10 0
      services/task.go
  9. 9 2
      utils/common.go
  10. 30 0
      utils/config.go
  11. 12 0
      utils/constants.go
  12. 94 0
      utils/mail/charset_reader.go
  13. 366 0
      utils/mail/imap.go

+ 6 - 1
go.mod

@@ -8,12 +8,17 @@ require (
 	github.com/beego/beego/v2 v2.1.0
 	github.com/chromedp/cdproto v0.0.0-20240312231614-1e5096e63154
 	github.com/chromedp/chromedp v0.9.5
+	github.com/emersion/go-imap v1.2.1
+	github.com/emersion/go-message v0.18.1
 	github.com/go-sql-driver/mysql v1.8.0
+	github.com/h2non/filetype v1.1.3
 	github.com/mozillazg/go-pinyin v0.20.0
+	github.com/patrickmn/go-cache v2.1.0+incompatible
 	github.com/rdlucklib/rdluck_tools v1.0.3
 	github.com/shopspring/decimal v1.3.1
 	github.com/tealeg/xlsx v1.0.5
 	github.com/xuri/excelize/v2 v2.8.1
+	golang.org/x/text v0.14.0
 	gopkg.in/gomail.v2 v2.0.0-20160411212932-81ebce5c23df
 )
 
@@ -23,6 +28,7 @@ require (
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/cespare/xxhash/v2 v2.2.0 // indirect
 	github.com/chromedp/sysutil v1.0.0 // indirect
+	github.com/emersion/go-sasl v0.0.0-20200509203442-7bfe0ed36a21 // indirect
 	github.com/gobwas/httphead v0.1.0 // indirect
 	github.com/gobwas/pool v0.2.1 // indirect
 	github.com/gobwas/ws v1.3.2 // indirect
@@ -46,7 +52,6 @@ require (
 	golang.org/x/crypto v0.19.0 // indirect
 	golang.org/x/net v0.21.0 // indirect
 	golang.org/x/sys v0.17.0 // indirect
-	golang.org/x/text v0.14.0 // indirect
 	google.golang.org/protobuf v1.30.0 // indirect
 	gopkg.in/alexcesaro/quotedprintable.v3 v3.0.0-20150716171945-2caba252f4dc // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect

+ 13 - 0
go.sum

@@ -48,6 +48,14 @@ github.com/elastic/go-elasticsearch/v6 v6.8.5/go.mod h1:UwaDJsD3rWLM5rKNFzv9hgox
 github.com/elazarl/go-bindata-assetfs v1.0.0/go.mod h1:v+YaWX3bdea5J/mo8dSETolEo7R71Vk1u8bnjau5yw4=
 github.com/elazarl/go-bindata-assetfs v1.0.1 h1:m0kkaHRKEu7tUIUFVwhGGGYClXvyl4RE03qmvRTNfbw=
 github.com/elazarl/go-bindata-assetfs v1.0.1/go.mod h1:v+YaWX3bdea5J/mo8dSETolEo7R71Vk1u8bnjau5yw4=
+github.com/emersion/go-imap v1.2.1 h1:+s9ZjMEjOB8NzZMVTM3cCenz2JrQIGGo5j1df19WjTA=
+github.com/emersion/go-imap v1.2.1/go.mod h1:Qlx1FSx2FTxjnjWpIlVNEuX+ylerZQNFE5NsmKFSejY=
+github.com/emersion/go-message v0.15.0/go.mod h1:wQUEfE+38+7EW8p8aZ96ptg6bAb1iwdgej19uXASlE4=
+github.com/emersion/go-message v0.18.1 h1:tfTxIoXFSFRwWaZsgnqS1DSZuGpYGzSmCZD8SK3QA2E=
+github.com/emersion/go-message v0.18.1/go.mod h1:XpJyL70LwRvq2a8rVbHXikPgKj8+aI0kGdHlg16ibYA=
+github.com/emersion/go-sasl v0.0.0-20200509203442-7bfe0ed36a21 h1:OJyUGMJTzHTd1XQp98QTaHernxMYzRaOasRir9hUlFQ=
+github.com/emersion/go-sasl v0.0.0-20200509203442-7bfe0ed36a21/go.mod h1:iL2twTeMvZnrg54ZoPDNfJaJaqy0xIQFuBdrLsmspwQ=
+github.com/emersion/go-textwrapper v0.0.0-20200911093747-65d896831594/go.mod h1:aqO8z8wPrjkscevZJFVE1wXJrLpC5LtJG7fqLOsPb2U=
 github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
 github.com/garyburd/redigo v1.6.3/go.mod h1:rTb6epsqigu3kYKBnaF028A7Tf/Aw5s0cqA47doKKqw=
 github.com/glendc/gopher-json v0.0.0-20170414221815-dc4743023d0c/go.mod h1:Gja1A+xZ9BoviGJNA2E9vFkPjjsl+CoJxSXiQM1UXtw=
@@ -91,6 +99,8 @@ github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/
 github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=
 github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
 github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
+github.com/h2non/filetype v1.1.3 h1:FKkx9QbD7HR/zjK1Ia5XiBsq9zdLi5Kf3zGyFTAFkGg=
+github.com/h2non/filetype v1.1.3/go.mod h1:319b3zT68BvV+WRj7cwy856M2ehB3HqNOt6sy1HndBY=
 github.com/hashicorp/golang-lru v0.5.4 h1:YDjusn29QI/Das2iO9M0BHnIbxPeyuCHsjMW+lJfyTc=
 github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4=
 github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
@@ -138,6 +148,8 @@ github.com/onsi/ginkgo v1.12.0/go.mod h1:oUhWkIvk5aDxtKvDDuw8gItl8pKl42LzjC9KZE0
 github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY=
 github.com/orisano/pixelmatch v0.0.0-20220722002657-fb0b55479cde h1:x0TT0RDC7UhAVbbWWBzr41ElhJx5tXPWkIHA2HWPRuw=
 github.com/orisano/pixelmatch v0.0.0-20220722002657-fb0b55479cde/go.mod h1:nZgzbfBr3hhjoZnS66nKrHmduYNpc34ny7RK4z5/HM0=
+github.com/patrickmn/go-cache v2.1.0+incompatible h1:HRMgzkcYKYpi3C8ajMPV8OFXaaRUnok+kx1WdO15EQc=
+github.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ=
 github.com/pelletier/go-toml v1.0.1/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
 github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
 github.com/peterh/liner v1.0.1-0.20171122030339-3681c2a91233/go.mod h1:xIteQHvHuaLYG9IFj6mSxM0fCKrs34IrEQUhOYuGPHc=
@@ -261,6 +273,7 @@ golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
 golang.org/x/term v0.7.0/go.mod h1:P32HKFT3hSsZrRxla30E9HqToFYAQPCMs/zFMBUFqPY=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
 golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
 golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=

+ 40 - 7
models/base_from_coalmine.go

@@ -1,10 +1,43 @@
 package models
 
 import (
-	"github.com/beego/beego/v2/client/orm"
 	"time"
+
+	"github.com/beego/beego/v2/client/orm"
 )
 
+type CoalSheetData struct {
+	Name     string
+	Rows     []Row
+	Cols     []*Col
+	MaxRow   int
+	MaxCol   int
+	Hidden   bool
+	Selected bool
+}
+
+type Row struct {
+	Cells        []Cell
+	Hidden       bool
+	Height       float64
+	OutlineLevel uint8
+	isCustom     bool
+}
+
+type Col struct {
+	Min          int
+	Max          int
+	Hidden       bool
+	Width        float64
+	Collapsed    bool
+	OutlineLevel uint8
+	numFmt       string
+}
+
+type Cell struct {
+	Value string
+}
+
 type BaseFromCoalmineMapping struct {
 	BaseFromCoalmineMappingId int       `orm:"column(base_from_coalmine_mapping_id);pk"`
 	IndexName                 string    `description:"持买单量指标名称"`
@@ -44,14 +77,14 @@ type BaseFromCoalmineCompanyIndex struct {
 	ModifyTime                     time.Time `description:"修改时间"`
 }
 
-//添加指标
+// 添加指标
 func AddBaseFromCoalmineMapping(item *BaseFromCoalmineMapping) (lastId int64, err error) {
 	o := orm.NewOrmUsingDB("data")
 	lastId, err = o.Insert(item)
 	return
 }
 
-//查询指标
+// 查询指标
 func GetBaseFromCoalmineMapping() (items []*BaseFromCoalmineMapping, err error) {
 	o := orm.NewOrmUsingDB("data")
 	sql := `SELECT * FROM base_from_coalmine_mapping`
@@ -59,7 +92,7 @@ func GetBaseFromCoalmineMapping() (items []*BaseFromCoalmineMapping, err error)
 	return
 }
 
-//查询数据
+// 查询数据
 func GetBaseFromCoalmineIndex() (items []*BaseFromCoalmineJsmIndex, err error) {
 	o := orm.NewOrmUsingDB("data")
 	sql := `SELECT * FROM base_from_coalmine_jsm_index`
@@ -74,21 +107,21 @@ func UpdateBaseFromCoalmineIndex(item *BaseFromCoalmineJsmIndex) (err error) {
 	return
 }
 
-//添加数据
+// 添加数据
 func AddBaseFromCoalmineIndex(item *BaseFromCoalmineJsmIndex) (lastId int64, err error) {
 	o := orm.NewOrmUsingDB("data")
 	lastId, err = o.Insert(item)
 	return
 }
 
-//添加公司指标
+// 添加公司指标
 func AddBaseFromCoalmineCompanyIndex(item *BaseFromCoalmineCompanyIndex) (lastId int64, err error) {
 	o := orm.NewOrmUsingDB("data")
 	lastId, err = o.Insert(item)
 	return
 }
 
-//查询公司指标
+// 查询公司指标
 func GetBaseFromCoalmineCompanyIndex() (items []*BaseFromCoalmineCompanyIndex, err error) {
 	o := orm.NewOrmUsingDB("data")
 	sql := `SELECT * FROM base_from_coalmine_company_index`

+ 60 - 0
services/base.go

@@ -0,0 +1,60 @@
+package services
+
+import (
+	"encoding/json"
+	"eta/eta_crawler/utils"
+	"fmt"
+	"io/ioutil"
+	"net/http"
+	"strings"
+)
+
+// PostEdbLib 调用指标接口
+func PostEdbLib(param map[string]interface{}, method string) (result []byte, err error) {
+	postUrl := utils.EDB_LIB_URL + method
+	postData, err := json.Marshal(param)
+	if err != nil {
+		return
+	}
+	result, err = HttpPost(postUrl, string(postData), "application/json")
+	if err != nil {
+		return
+	}
+	return
+}
+
+func HttpPost(url, postData string, params ...string) ([]byte, error) {
+	fmt.Println("HttpPost Url:" + url)
+	body := ioutil.NopCloser(strings.NewReader(postData))
+	client := &http.Client{}
+	req, err := http.NewRequest("POST", url, body)
+	if err != nil {
+		return nil, err
+	}
+	contentType := "application/x-www-form-urlencoded;charset=utf-8"
+	if len(params) > 0 && params[0] != "" {
+		contentType = params[0]
+	}
+	req.Header.Set("Content-Type", contentType)
+	req.Header.Set("authorization", utils.MD5(utils.APP_EDB_LIB_NAME_EN+utils.EDB_LIB_Md5_KEY))
+	resp, err := client.Do(req)
+	if err != nil {
+		fmt.Println("client.Do err:" + err.Error())
+		return nil, err
+	}
+	defer resp.Body.Close()
+	b, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		fmt.Println("HttpPost:" + string(b))
+	}
+	return b, err
+}
+
+func HttpGet(url string) ([]byte, error) {
+	res, err := http.Get(url)
+	if err != nil {
+		return nil, err
+	}
+	defer res.Body.Close()
+	return ioutil.ReadAll(res.Body)
+}

+ 622 - 0
services/commodity_coal.go

@@ -0,0 +1,622 @@
+package services
+
+import (
+	"encoding/json"
+	"eta/eta_crawler/models"
+	"eta/eta_crawler/utils"
+	"fmt"
+
+	"github.com/tealeg/xlsx"
+)
+
+func JsmHistory(path string) (err error) {
+	defer func() {
+		if err != nil {
+			fmt.Println("RefreshDataFromCoalCoastal  Err:" + err.Error())
+			utils.FileLog.Info(fmt.Sprintf("RefreshDataFromCoalCoastal, Err: %s", err))
+		}
+	}()
+
+	var xlFile *xlsx.File
+	exist, err := PathExists(path)
+	if err != nil {
+		fmt.Println(err)
+		return
+	}
+	if exist {
+		xlFile, err = xlsx.OpenFile(path)
+		if err != nil {
+			fmt.Println("OpenFile err:", err)
+			return
+		}
+	} else {
+		fmt.Println("Not Exist")
+		return
+	}
+
+	sheetDatas := make([]models.CoalSheetData, 0)
+	data := *xlFile.Sheets[0]
+	sheetData := models.CoalSheetData{
+		Name:     data.Name,
+		MaxRow:   data.MaxRow,
+		MaxCol:   data.MaxCol,
+		Hidden:   data.Hidden,
+		Selected: data.Selected,
+	}
+	rows := make([]models.Row, 0)
+	for _, v := range data.Rows {
+		cells := make([]models.Cell, 0)
+		for _, cell := range v.Cells {
+			cells = append(cells, models.Cell{
+				Value: cell.String(),
+			})
+		}
+		row := models.Row{
+			Cells: cells,
+		}
+		rows = append(rows, row)
+	}
+	sheetData.Rows = rows
+	sheetDatas = append(sheetDatas, sheetData)
+
+	params := make(map[string]interface{})
+	params["SheetData"] = sheetDatas
+
+	result, e := PostEdbLib(params, utils.LIB_ROUTE_COAL_MINE_JSM_HISTORY)
+	if e != nil {
+		b, _ := json.Marshal(params)
+		fmt.Println(e)
+		utils.FileLog.Info(fmt.Sprintf("PostEdbLib err: %s, params: %s", e.Error(), string(b)))
+		return
+	}
+	resp := new(models.BaseResponse)
+	if e := json.Unmarshal(result, &resp); e != nil {
+		utils.FileLog.Info(fmt.Sprintf("json.Unmarshal err: %s", e))
+		return
+	}
+	if resp.Ret != 200 {
+		utils.FileLog.Info(fmt.Sprintf("Msg: %s, ErrMsg: %s", resp.Msg, resp.ErrMsg))
+		return
+	}
+
+	return
+}
+
+func CoastalHistory(path string) (err error) {
+	defer func() {
+		if err != nil {
+			fmt.Println("RefreshDataFromCoalCoastal  Err:" + err.Error())
+			utils.FileLog.Info(fmt.Sprintf("RefreshDataFromCoalCoastal, Err: %s", err))
+		}
+	}()
+	//path := "/Users/xi/Desktop/瑞茂通-中国煤炭市场网数据/442家晋陕蒙、沿海8省、内陆17省历史数据/CⅢ-8-16 25省市库存和日耗情况(CCTD).xlsx"
+	//path := "D:\\瑞茂通-中国煤炭市场网数据\\442家晋陕蒙、沿海8省、内陆17省历史数据\\CⅢ-8-16 25省市库存和日耗情况(CCTD).xlsx"
+
+	var xlFile *xlsx.File
+	exist, err := PathExists(path)
+	if err != nil {
+		fmt.Println(err)
+		return
+	}
+	if exist {
+		xlFile, err = xlsx.OpenFile(path)
+		if err != nil {
+			fmt.Println("OpenFile err:", err)
+			return
+		}
+	} else {
+		fmt.Println("Not Exist")
+		return
+	}
+
+	sheetDatas := make([]models.CoalSheetData, 0)
+	for i, sheet := range xlFile.Sheets {
+		if i < 4 {
+			data := sheet
+			sheetData := models.CoalSheetData{
+				Name:     data.Name,
+				MaxRow:   data.MaxRow,
+				MaxCol:   data.MaxCol,
+				Hidden:   data.Hidden,
+				Selected: data.Selected,
+			}
+			rows := make([]models.Row, 0)
+			for _, v := range data.Rows {
+				cells := make([]models.Cell, 0)
+				for _, cell := range v.Cells {
+					cells = append(cells, models.Cell{
+						Value: cell.String(),
+					})
+				}
+				row := models.Row{
+					Cells: cells,
+				}
+				rows = append(rows, row)
+			}
+			sheetData.Rows = rows
+			sheetDatas = append(sheetDatas, sheetData)
+		}
+	}
+
+	params := make(map[string]interface{})
+	params["SheetData"] = sheetDatas
+	result, e := PostEdbLib(params, utils.LIB_ROUTE_COAL_MINE_COASTAL_HISTORY)
+	if e != nil {
+		b, _ := json.Marshal(params)
+		fmt.Println(e)
+		utils.FileLog.Info(fmt.Sprintf("PostEdbLib err: %s, params: %s", e.Error(), string(b)))
+		return
+	}
+	resp := new(models.BaseResponse)
+	if e := json.Unmarshal(result, &resp); e != nil {
+		utils.FileLog.Info(fmt.Sprintf("json.Unmarshal err: %s", e))
+		return
+	}
+	if resp.Ret != 200 {
+		utils.FileLog.Info(fmt.Sprintf("Msg: %s, ErrMsg: %s", resp.Msg, resp.ErrMsg))
+		return
+	}
+
+	return
+}
+
+func InlandHistory(path string) (err error) {
+	defer func() {
+		if err != nil {
+			fmt.Println("RefreshDataFromCoalCoastal  Err:" + err.Error())
+			utils.FileLog.Info(fmt.Sprintf("RefreshDataFromCoalCoastal, Err: %s", err))
+		}
+	}()
+	//path := "/Users/xi/Desktop/瑞茂通-中国煤炭市场网数据/442家晋陕蒙、沿海8省、内陆17省历史数据/CⅢ-8-16 25省市库存和日耗情况(CCTD).xlsx"
+	//path := "D:\\瑞茂通-中国煤炭市场网数据\\442家晋陕蒙、沿海8省、内陆17省历史数据\\CⅢ-8-16 25省市库存和日耗情况(CCTD).xlsx"
+
+	var xlFile *xlsx.File
+	exist, err := PathExists(path)
+	if err != nil {
+		fmt.Println(err)
+		return
+	}
+	if exist {
+		xlFile, err = xlsx.OpenFile(path)
+		if err != nil {
+			fmt.Println("OpenFile err:", err)
+			return
+		}
+	} else {
+		fmt.Println("Not Exist")
+		return
+	}
+
+	sheetDatas := make([]models.CoalSheetData, 0)
+	for i, sheet := range xlFile.Sheets {
+		if i > 3 {
+			data := sheet
+			sheetData := models.CoalSheetData{
+				Name:     data.Name,
+				MaxRow:   data.MaxRow,
+				MaxCol:   data.MaxCol,
+				Hidden:   data.Hidden,
+				Selected: data.Selected,
+			}
+			rows := make([]models.Row, 0)
+			for _, v := range data.Rows {
+				cells := make([]models.Cell, 0)
+				for _, cell := range v.Cells {
+					cells = append(cells, models.Cell{
+						Value: cell.String(),
+					})
+				}
+				row := models.Row{
+					Cells: cells,
+				}
+				rows = append(rows, row)
+			}
+			sheetData.Rows = rows
+			sheetDatas = append(sheetDatas, sheetData)
+		}
+	}
+
+	params := make(map[string]interface{})
+	params["SheetData"] = sheetDatas
+	result, e := PostEdbLib(params, utils.LIB_ROUTE_COAL_MINE_INLAND_HISTORY)
+	if e != nil {
+		b, _ := json.Marshal(params)
+		fmt.Println(e)
+		utils.FileLog.Info(fmt.Sprintf("PostEdbLib err: %s, params: %s", e.Error(), string(b)))
+		return
+	}
+	resp := new(models.BaseResponse)
+	if e := json.Unmarshal(result, &resp); e != nil {
+		utils.FileLog.Info(fmt.Sprintf("json.Unmarshal err: %s", e))
+		return
+	}
+	if resp.Ret != 200 {
+		utils.FileLog.Info(fmt.Sprintf("Msg: %s, ErrMsg: %s", resp.Msg, resp.ErrMsg))
+		return
+	}
+
+	return
+}
+
+func Jsm(path string) (err error) {
+	defer func() {
+		if err != nil {
+			fmt.Println("RefreshDataFromCoalCoastal  Err:" + err.Error())
+			utils.FileLog.Info(fmt.Sprintf("RefreshDataFromCoalCoastal, Err: %s", err))
+		}
+	}()
+	//path := "/home/code/python/coal_mail/emailFile/沿海八省动力煤终端用户供耗存数据更新.xlsx"
+	//path := "/Users/xi/Desktop/瑞茂通-中国煤炭市场网数据/442家晋陕蒙、沿海8省、内陆17省最新数据/442家晋陕蒙煤矿周度产量数据-20231201.xlsx"
+	//path := "D:\\瑞茂通-中国煤炭市场网数据\\442家晋陕蒙、沿海8省、内陆17省历史数据\\442家晋陕蒙历史数据.xlsx"
+
+	var xlFile *xlsx.File
+	exist, err := PathExists(path)
+	if err != nil {
+		fmt.Println(err)
+		return
+	}
+	if exist {
+		xlFile, err = xlsx.OpenFile(path)
+		if err != nil {
+			fmt.Println("OpenFile err:", err)
+			return
+		}
+	} else {
+		fmt.Println("Not Exist")
+		return
+	}
+
+	sheetDatas := make([]models.CoalSheetData, 0)
+	data := *xlFile.Sheets[0]
+	sheetData := models.CoalSheetData{
+		Name:     data.Name,
+		MaxRow:   data.MaxRow,
+		MaxCol:   data.MaxCol,
+		Hidden:   data.Hidden,
+		Selected: data.Selected,
+	}
+	rows := make([]models.Row, 0)
+	for _, v := range data.Rows {
+		cells := make([]models.Cell, 0)
+		for _, cell := range v.Cells {
+			cells = append(cells, models.Cell{
+				Value: cell.String(),
+			})
+		}
+		row := models.Row{
+			Cells: cells,
+		}
+		rows = append(rows, row)
+	}
+	sheetData.Rows = rows
+	sheetDatas = append(sheetDatas, sheetData)
+
+	params := make(map[string]interface{})
+	params["SheetData"] = sheetDatas
+	result, e := PostEdbLib(params, utils.LIB_ROUTE_COAL_MINE_JSM)
+	if e != nil {
+		b, _ := json.Marshal(params)
+		fmt.Println(e)
+		utils.FileLog.Info(fmt.Sprintf("PostEdbLib err: %s, params: %s", e.Error(), string(b)))
+		return
+	}
+	resp := new(models.BaseResponse)
+	if e := json.Unmarshal(result, &resp); e != nil {
+		utils.FileLog.Info(fmt.Sprintf("json.Unmarshal err: %s", e))
+		return
+	}
+	if resp.Ret != 200 {
+		utils.FileLog.Info(fmt.Sprintf("Msg: %s, ErrMsg: %s", resp.Msg, resp.ErrMsg))
+		return
+	}
+
+	return
+}
+
+func Coastal(path string) (err error) {
+	defer func() {
+		if err != nil {
+			fmt.Println("RefreshDataFromCoalCoastal  Err:" + err.Error())
+			utils.FileLog.Info(fmt.Sprintf("RefreshDataFromCoalCoastal, Err: %s", err))
+		}
+	}()
+	//path := "/Users/xi/Desktop/瑞茂通-中国煤炭市场网数据/442家晋陕蒙、沿海8省、内陆17省最新数据/内陆17省动力煤终端用户供耗存.xlsx"
+	//path := "D:\\瑞茂通-中国煤炭市场网数据\\442家晋陕蒙、沿海8省、内陆17省历史数据\\CⅢ-8-16 25省市库存和日耗情况(CCTD).xlsx"
+
+	fmt.Println("沿海开始")
+	var xlFile *xlsx.File
+	exist, err := PathExists(path)
+	if err != nil {
+		fmt.Println(err)
+		return
+	}
+	if exist {
+		xlFile, err = xlsx.OpenFile(path)
+		if err != nil {
+			fmt.Println("OpenFile err:", err)
+			return
+		}
+	} else {
+		fmt.Println("Not Exist")
+		return
+	}
+
+	sheetDatas := make([]models.CoalSheetData, 0)
+	for _, sheet := range xlFile.Sheets {
+		data := sheet
+		sheetData := models.CoalSheetData{
+			Name:     data.Name,
+			MaxRow:   data.MaxRow,
+			MaxCol:   data.MaxCol,
+			Hidden:   data.Hidden,
+			Selected: data.Selected,
+		}
+		rows := make([]models.Row, 0)
+		for _, v := range data.Rows {
+			cells := make([]models.Cell, 0)
+			for _, cell := range v.Cells {
+				cells = append(cells, models.Cell{
+					Value: cell.String(),
+				})
+			}
+			row := models.Row{
+				Cells: cells,
+			}
+			rows = append(rows, row)
+		}
+		sheetData.Rows = rows
+		fmt.Println("rows:", len(rows))
+		sheetDatas = append(sheetDatas, sheetData)
+	}
+	fmt.Println("sheetDatas:", len(sheetDatas))
+
+	params := make(map[string]interface{})
+	params["SheetData"] = sheetDatas
+	result, e := PostEdbLib(params, utils.LIB_ROUTE_COAL_MINE_COASTAL)
+	if e != nil {
+		b, _ := json.Marshal(params)
+		fmt.Println(e)
+		utils.FileLog.Info(fmt.Sprintf("PostEdbLib err: %s, params: %s", e.Error(), string(b)))
+		return
+	}
+	resp := new(models.BaseResponse)
+	if e := json.Unmarshal(result, &resp); e != nil {
+		utils.FileLog.Info(fmt.Sprintf("json.Unmarshal err: %s", e))
+		return
+	}
+	if resp.Ret != 200 {
+		utils.FileLog.Info(fmt.Sprintf("Msg: %s, ErrMsg: %s", resp.Msg, resp.ErrMsg))
+		return
+	}
+
+	return
+}
+
+func Inland(path string) (err error) {
+	defer func() {
+		if err != nil {
+			fmt.Println("RefreshDataFromCoalCoastal  Err:" + err.Error())
+			utils.FileLog.Info(fmt.Sprintf("RefreshDataFromCoalCoastal, Err: %s", err))
+		}
+	}()
+	//path := "/Users/xi/Desktop/瑞茂通-中国煤炭市场网数据/442家晋陕蒙、沿海8省、内陆17省最新数据/内陆17省动力煤终端用户供耗存.xlsx"
+	//path := "D:\\瑞茂通-中国煤炭市场网数据\\442家晋陕蒙、沿海8省、内陆17省历史数据\\CⅢ-8-16 25省市库存和日耗情况(CCTD).xlsx"
+
+	var xlFile *xlsx.File
+	exist, err := PathExists(path)
+	if err != nil {
+		fmt.Println(err)
+		return
+	}
+	if exist {
+		xlFile, err = xlsx.OpenFile(path)
+		if err != nil {
+			fmt.Println("OpenFile err:", err)
+			return
+		}
+	} else {
+		fmt.Println("Not Exist")
+		return
+	}
+
+	sheetDatas := make([]models.CoalSheetData, 0)
+	for _, sheet := range xlFile.Sheets {
+		data := sheet
+		sheetData := models.CoalSheetData{
+			Name:     data.Name,
+			MaxRow:   data.MaxRow,
+			MaxCol:   data.MaxCol,
+			Hidden:   data.Hidden,
+			Selected: data.Selected,
+		}
+		rows := make([]models.Row, 0)
+		for _, v := range data.Rows {
+			cells := make([]models.Cell, 0)
+			for _, cell := range v.Cells {
+				cells = append(cells, models.Cell{
+					Value: cell.String(),
+				})
+			}
+			row := models.Row{
+				Cells: cells,
+			}
+			rows = append(rows, row)
+		}
+		sheetData.Rows = rows
+		sheetDatas = append(sheetDatas, sheetData)
+	}
+
+	params := make(map[string]interface{})
+	params["SheetData"] = sheetDatas
+	result, e := PostEdbLib(params, utils.LIB_ROUTE_COAL_MINE_INLAND)
+	if e != nil {
+		b, _ := json.Marshal(params)
+		fmt.Println(e)
+		utils.FileLog.Info(fmt.Sprintf("PostEdbLib err: %s, params: %s", e.Error(), string(b)))
+		return
+	}
+	resp := new(models.BaseResponse)
+	if e := json.Unmarshal(result, &resp); e != nil {
+		utils.FileLog.Info(fmt.Sprintf("json.Unmarshal err: %s", e))
+		return
+	}
+	if resp.Ret != 200 {
+		utils.FileLog.Info(fmt.Sprintf("Msg: %s, ErrMsg: %s", resp.Msg, resp.ErrMsg))
+		return
+	}
+
+	return
+}
+
+func Mtjh(path string) (err error) {
+	defer func() {
+		if err != nil {
+			fmt.Println("RefreshDataFromCoalMtjh  Err:" + err.Error())
+			utils.FileLog.Info(fmt.Sprintf("RefreshDataFromCoalMtjh, Err: %s", err))
+		}
+	}()
+	//path = "/Users/xi/Desktop/煤炭江湖数据定制化服务——中国主流港口煤炭库存20231129.xlsx"
+
+	var xlFile *xlsx.File
+	exist, err := PathExists(path)
+	if err != nil {
+		fmt.Println(err)
+		return
+	}
+	if exist {
+		xlFile, err = xlsx.OpenFile(path)
+		if err != nil {
+			fmt.Println("OpenFile err:", err)
+			return
+		}
+	} else {
+		fmt.Println("Not Exist")
+		return
+	}
+
+	sheetDatas := make([]models.CoalSheetData, 0)
+	for i, sheet := range xlFile.Sheets {
+		if i > 0 {
+			break
+		}
+		data := sheet
+		sheetData := models.CoalSheetData{
+			Name:     data.Name,
+			MaxRow:   data.MaxRow,
+			MaxCol:   data.MaxCol,
+			Hidden:   data.Hidden,
+			Selected: data.Selected,
+		}
+		rows := make([]models.Row, 0)
+		for _, v := range data.Rows {
+			cells := make([]models.Cell, 0)
+			for _, cell := range v.Cells {
+				cells = append(cells, models.Cell{
+					Value: cell.String(),
+				})
+			}
+			row := models.Row{
+				Cells: cells,
+			}
+			rows = append(rows, row)
+		}
+		sheetData.Rows = rows
+		sheetDatas = append(sheetDatas, sheetData)
+	}
+
+	params := make(map[string]interface{})
+	params["SheetData"] = sheetDatas
+	result, e := PostEdbLib(params, utils.LIB_ROUTE_COAL_MINE_MTJH)
+	if e != nil {
+		b, _ := json.Marshal(params)
+		fmt.Println(e)
+		utils.FileLog.Info(fmt.Sprintf("PostEdbLib err: %s, params: %s", e.Error(), string(b)))
+		return
+	}
+	resp := new(models.BaseResponse)
+	if e := json.Unmarshal(result, &resp); e != nil {
+		utils.FileLog.Info(fmt.Sprintf("json.Unmarshal err: %s", e))
+		return
+	}
+	if resp.Ret != 200 {
+		utils.FileLog.Info(fmt.Sprintf("Msg: %s, ErrMsg: %s", resp.Msg, resp.ErrMsg))
+		return
+	}
+
+	return
+}
+
+func Firm(path string) (err error) {
+	defer func() {
+		if err != nil {
+			fmt.Println("RefreshDataFromCoalFirm  Err:" + err.Error())
+			utils.FileLog.Info(fmt.Sprintf("RefreshDataFromCoalFirm, Err: %s", err))
+		}
+	}()
+	//path = "/Users/xi/Desktop/煤炭江湖数据定制化服务——中国主流港口煤炭库存20231129.xlsx"
+
+	var xlFile *xlsx.File
+	exist, err := PathExists(path)
+	if err != nil {
+		fmt.Println(err)
+		return
+	}
+	if exist {
+		xlFile, err = xlsx.OpenFile(path)
+		if err != nil {
+			fmt.Println("OpenFile err:", err)
+			return
+		}
+	} else {
+		fmt.Println("Not Exist")
+		return
+	}
+
+	sheetDatas := make([]models.CoalSheetData, 0)
+	for _, sheet := range xlFile.Sheets {
+		data := sheet
+		sheetData := models.CoalSheetData{
+			Name:     data.Name,
+			MaxRow:   data.MaxRow,
+			MaxCol:   data.MaxCol,
+			Hidden:   data.Hidden,
+			Selected: data.Selected,
+		}
+		rows := make([]models.Row, 0)
+		for _, v := range data.Rows {
+			cells := make([]models.Cell, 0)
+			for _, cell := range v.Cells {
+				cells = append(cells, models.Cell{
+					Value: cell.String(),
+				})
+			}
+			row := models.Row{
+				Cells: cells,
+			}
+			rows = append(rows, row)
+		}
+		sheetData.Rows = rows
+		sheetDatas = append(sheetDatas, sheetData)
+	}
+
+	params := make(map[string]interface{})
+	params["SheetData"] = sheetDatas
+	result, e := PostEdbLib(params, utils.LIB_ROUTE_COAL_MINE_FIRM)
+	if e != nil {
+		b, _ := json.Marshal(params)
+		fmt.Println(e)
+		utils.FileLog.Info(fmt.Sprintf("PostEdbLib err: %s, params: %s", e.Error(), string(b)))
+		return
+	}
+	resp := new(models.BaseResponse)
+	if e := json.Unmarshal(result, &resp); e != nil {
+		utils.FileLog.Info(fmt.Sprintf("json.Unmarshal err: %s", e))
+		return
+	}
+	if resp.Ret != 200 {
+		utils.FileLog.Info(fmt.Sprintf("Msg: %s, ErrMsg: %s", resp.Msg, resp.ErrMsg))
+		return
+	}
+
+	return
+}

+ 88 - 0
services/commodity_coal_watch.go

@@ -0,0 +1,88 @@
+package services
+
+import (
+	"context"
+	"eta/eta_crawler/utils"
+	"fmt"
+	"io/fs"
+	"os"
+	"path/filepath"
+	"strings"
+	"syscall"
+	"time"
+
+	"github.com/patrickmn/go-cache"
+)
+
+func CoalWatchTask(cont context.Context) (err error) {
+	ReadWatchIndexFile()
+	return
+}
+
+func ReadWatchIndexFile() {
+	fmt.Println("ReadWatchIndexFile start")
+	var err error
+	defer func() {
+		if err != nil {
+			fmt.Println("ReadWatchIndexFile Err:" + err.Error())
+		}
+	}()
+	var cacheClient *cache.Cache
+	if cacheClient == nil {
+		cacheClient = cache.New(365*24*time.Hour, 365*24*time.Hour)
+	}
+	err = filepath.Walk(utils.CoalFilePath, func(path string, info fs.FileInfo, err error) error {
+		if err != nil {
+			return err
+		}
+		if !info.IsDir() {
+			fileInfo, err := os.Stat(path)
+			if err != nil {
+				fmt.Println("os.Stat:", err.Error())
+			}
+			winFileAttr := fileInfo.Sys().(*syscall.Win32FileAttributeData)
+			modifyTimeStr := utils.SecondToTime(winFileAttr.LastWriteTime.Nanoseconds() / 1e9).Format(utils.FormatDateTime)
+
+			existModifyTime, ok := cacheClient.Get(path)
+			if ok {
+				existModifyTimeStr := existModifyTime.(string)
+				if existModifyTimeStr != modifyTimeStr {
+					if strings.Contains(path, "442家晋陕蒙煤矿周度产量数据") {
+						err = Jsm(path)
+					} else if strings.Contains(path, "内陆17省动力煤终端用户供耗存") {
+						err = Inland(path)
+					} else if strings.Contains(path, "沿海八省动力煤终端用户供耗存数据更新") {
+						err = Coastal(path)
+					} else if strings.Contains(path, "442家晋陕蒙历史数据") {
+						err = JsmHistory(path)
+					} else if strings.Contains(path, "CⅢ-8-16 25省市库存和日耗情况") {
+						err = CoastalHistory(path)
+						time.Sleep(time.Second * 10)
+						err = InlandHistory(path)
+					} else if strings.Contains(path, "分企业煤炭产量旬度数据") {
+						err = Firm(path)
+					}
+				}
+			} else {
+				if strings.Contains(path, "442家晋陕蒙煤矿周度产量数据") {
+					err = Jsm(path)
+				} else if strings.Contains(path, "内陆17省动力煤终端用户供耗存") {
+					err = Inland(path)
+				} else if strings.Contains(path, "沿海八省动力煤终端用户供耗存数据更新") {
+					err = Coastal(path)
+				} else if strings.Contains(path, "442家晋陕蒙历史数据") {
+					err = JsmHistory(path)
+				} else if strings.Contains(path, "CⅢ-8-16 25省市库存和日耗情况") {
+					err = CoastalHistory(path)
+					time.Sleep(time.Second * 10)
+					err = InlandHistory(path)
+				} else if strings.Contains(path, "分企业煤炭产量旬度数据") {
+					err = Firm(path)
+				}
+			}
+			cacheClient.Delete(path)
+			cacheClient.Set(path, modifyTimeStr, 24*time.Hour)
+		}
+		return nil
+	})
+}

+ 66 - 0
services/email/mail.go

@@ -0,0 +1,66 @@
+package email
+
+import (
+	"context"
+	"eta/eta_crawler/utils"
+	"eta/eta_crawler/utils/mail"
+	"fmt"
+	"io/fs"
+	"os"
+	"strconv"
+	"sync"
+)
+
+// 同步用户锁,防止重复同步,不管是全量还是增量,都是同一时间只能一个同步
+var lockListenEmail sync.Mutex
+
+func ListenMail(cont context.Context) (err error) {
+	defer func() {
+		lockListenEmail.Unlock()
+		if err != nil {
+			utils.FileLog.Error("监听邮件失败:%s", err.Error())
+		}
+	}()
+	lockListenEmail.Lock()
+	// 目录创建
+	_ = ensureDirExists(fmt.Sprintf("%s%s", utils.CoalFilePath, `file`))
+
+	fmt.Println("开始监听邮件")
+	utils.FileLog.Info("中国煤炭网开始监听邮件")
+	var lastNday int
+	if utils.CoalEmailNDay != "" {
+		lastNday, _ = strconv.Atoi(utils.CoalEmailNDay)
+		if err != nil {
+			lastNday = 1
+		}
+	}
+	if lastNday < 0 {
+		lastNday = 1
+	}
+
+	utils.FileLog.Info("中国煤炭监听配置为:CoalEmailAddress:%s, CoalEmailFolder:%s, lastNday:%d", utils.CoalEmailAddress, utils.CoalEmailFolder, lastNday)
+	var readBatch int
+	if utils.CoalEmailReadBatch != "" {
+		readBatch, err = strconv.Atoi(utils.CoalEmailReadBatch)
+		if err != nil {
+			readBatch = 10
+			utils.FileLog.Warning("读取邮件 MtjhEmailReadBatch 配置失败:%s, 默认改为:%d", err.Error(), readBatch)
+		}
+	}
+	mail.ListenMail(utils.CoalEmailAddress, utils.CoalEmailFolder, utils.CoalEmailUseName, utils.CoalEmailPassword, readBatch, lastNday)
+	return
+}
+
+func ensureDirExists(dirPath string) error {
+	info, err := os.Stat(dirPath)
+	if err == nil {
+		if info.IsDir() {
+			return nil // 目录已存在
+		}
+		return fmt.Errorf("path '%s' exists but is not a directory", dirPath)
+	}
+	if os.IsNotExist(err) {
+		return os.MkdirAll(dirPath, fs.ModePerm)
+	}
+	return err
+}

+ 10 - 0
services/task.go

@@ -2,6 +2,7 @@ package services
 
 import (
 	"context"
+	"eta/eta_crawler/services/email"
 	"eta/eta_crawler/services/liangyou"
 	"eta/eta_crawler/services/sci99"
 	"eta/eta_crawler/utils"
@@ -84,6 +85,15 @@ func Task() {
 	//task.AddTask("统计局数据爬取-季度", refreshNationalQuarter) // 每月15号1:25执行
 	//task.AddTask("统计局数据爬取-年度A", refreshNationalYearA)  // 每月20日1:45执行
 	//task.AddTask("统计局数据爬取-年度B", refreshNationalYearB)  // 每月25日1:45执行
+	if utils.CoalOpen == "1" {
+		mtjh := task.NewTask("refreshMtjh", "0 */2 * * * *", CoalWatchTask)
+		task.AddTask("启动中国煤炭网监听excel脚本", mtjh)
+	}
+
+	if utils.CoalMailAttachmentOpen == "1" {
+		coalMailTask := task.NewTask("MailAttachment", utils.CoalMailAttachmentTime, email.ListenMail)
+		task.AddTask("启动获取邮件附件脚本", coalMailTask)
+	}
 
 	task.StartTask()
 	//FileCoalJsm()

+ 9 - 2
utils/common.go

@@ -10,8 +10,6 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
-	"github.com/mozillazg/go-pinyin"
-	"github.com/shopspring/decimal"
 	"image"
 	"image/png"
 	"io"
@@ -30,6 +28,10 @@ import (
 	"strings"
 	"time"
 	"unicode"
+
+	"github.com/shopspring/decimal"
+
+	"github.com/mozillazg/go-pinyin"
 )
 
 // 随机数种子
@@ -351,6 +353,11 @@ func SaveBase64ToFileBySeek(content, path string) (err error) {
 	return nil
 }
 
+// 把秒级的时间戳转为time格式
+func SecondToTime(sec int64) time.Time {
+	return time.Unix(sec, 0)
+}
+
 func PathExists(path string) (bool, error) {
 	_, err := os.Stat(path)
 	if err == nil {

+ 30 - 0
utils/config.go

@@ -52,6 +52,21 @@ var (
 	OLD_EXCEL_PATH_JR string
 )
 
+// 中国煤炭网
+var (
+	CoalFilePath           string // excel文件地址
+	CoalOpen               string // 是否配置中国煤炭网数据源,1已配置
+	CoalMailAttachmentOpen string // 获取邮件附件功能,1已配置
+	CoalMailAttachmentTime string // 获取邮件附件功能时间
+	CoalEmailAddress       string // 中国煤炭网监听邮箱服务器地址
+	CoalEmailUseName       string // 中国煤炭网监听邮箱用户名
+	CoalEmailPassword      string // 中国煤炭网监听邮箱密码
+	CoalEmailFolder        string // 中国煤炭网监听邮箱文件夹
+	CoalEmailReadBatch     string // 中国煤炭网监听邮箱读取批次
+	CoalEmailFileExt       string // 中国煤炭网监听文件后缀
+	CoalEmailNDay          string // 中国煤炭网监听取最近N天数据
+)
+
 func init() {
 	tmpRunMode, err := web.AppConfig.String("run_mode")
 	if err != nil {
@@ -133,4 +148,19 @@ func init() {
 		LY_OPEN = config["ly_open"]
 
 	}
+	// 中国煤炭网
+	{
+		CoalFilePath = config["coal_file_path"]
+		CoalOpen = config["coal_open"]
+		CoalMailAttachmentOpen = config["coal_mail_attachment_open"]
+		CoalMailAttachmentTime = config["coal_mail_attachment_time"]
+		CoalEmailAddress = config["coal_email_address"]
+		CoalEmailUseName = config["coal_email_use_name"]
+		CoalEmailPassword = config["coal_email_password"]
+		CoalEmailFolder = config["coal_email_folder"]
+		CoalEmailReadBatch = config["coal_email_read_batch"]
+		CoalEmailFileExt = config["coal_email_file_ext"]
+		CoalEmailNDay = config["coal_email_n_day"]
+	}
+
 }

+ 12 - 0
utils/constants.go

@@ -31,3 +31,15 @@ const (
 	BusinessCodeFuBang  = "E2024020200"
 	BusinessCodeJinRui  = "E2023122901"
 )
+
+// eta_index_lib 的接口名称
+const (
+	LIB_ROUTE_COAL_MINE_MTJH            = "/mtjh/data"                 //煤炭江湖数据处理excel数据并入库 数据地址
+	LIB_ROUTE_COAL_MINE_JSM_HISTORY     = "/coal_mine/jsm/history"     //jsm三省煤炭网历史数据处理excel数据并入库 数据地址
+	LIB_ROUTE_COAL_MINE_COASTAL_HISTORY = "/coal_mine/coastal/history" //沿海煤炭网历史数据处理excel数据并入库 数据地址
+	LIB_ROUTE_COAL_MINE_INLAND_HISTORY  = "/coal_mine/inland/history"  //内陆三省煤炭网历史数据处理excel数据并入库 数据地址
+	LIB_ROUTE_COAL_MINE_JSM             = "/coal_mine/jsm"             //jsm三省煤炭网历史数据处理excel数据并入库 数据地址
+	LIB_ROUTE_COAL_MINE_COASTAL         = "/coal_mine/coastal"         //沿海煤炭网历史数据处理excel数据并入库 数据地址
+	LIB_ROUTE_COAL_MINE_INLAND          = "/coal_mine/inland"          //内陆三省煤炭网历史数据处理excel数据并入库 数据地址
+	LIB_ROUTE_COAL_MINE_FIRM            = "/coal_mine/firm"            //分公司旬度煤炭网数据处理excel数据并入库 数据地址
+)

+ 94 - 0
utils/mail/charset_reader.go

@@ -0,0 +1,94 @@
+package mail
+
+import (
+	"fmt"
+	"golang.org/x/text/encoding/charmap"
+	"golang.org/x/text/encoding/japanese"
+	"golang.org/x/text/encoding/korean"
+	"golang.org/x/text/encoding/simplifiedchinese"
+	"golang.org/x/text/transform"
+	"io"
+	"strings"
+)
+
+var charsetMap = map[string]transform.Transformer{
+	"gb2312":         simplifiedchinese.GBK.NewDecoder(),
+	"gbk":            simplifiedchinese.GBK.NewDecoder(),
+	"ibm037":         charmap.CodePage037.NewDecoder(),
+	"ibm437":         charmap.CodePage437.NewDecoder(),
+	"ibm850":         charmap.CodePage850.NewDecoder(),
+	"ibm852":         charmap.CodePage852.NewDecoder(),
+	"ibm855":         charmap.CodePage855.NewDecoder(),
+	"ibm858":         charmap.CodePage858.NewDecoder(),
+	"ibm860":         charmap.CodePage860.NewDecoder(),
+	"ibm862":         charmap.CodePage862.NewDecoder(),
+	"ibm863":         charmap.CodePage863.NewDecoder(),
+	"ibm865":         charmap.CodePage865.NewDecoder(),
+	"ibm866":         charmap.CodePage866.NewDecoder(),
+	"ibm1047":        charmap.CodePage1047.NewDecoder(),
+	"ibm1140":        charmap.CodePage1140.NewDecoder(),
+	"iso-8859-1":     charmap.ISO8859_1.NewDecoder(),
+	"iso-8859-2":     charmap.ISO8859_2.NewDecoder(),
+	"iso-8859-3":     charmap.ISO8859_3.NewDecoder(),
+	"iso-8859-4":     charmap.ISO8859_4.NewDecoder(),
+	"iso-8859-5":     charmap.ISO8859_5.NewDecoder(),
+	"iso-8859-6":     charmap.ISO8859_6.NewDecoder(),
+	"iso-8859-7":     charmap.ISO8859_7.NewDecoder(),
+	"iso-8859-8":     charmap.ISO8859_8.NewDecoder(),
+	"iso-8859-9":     charmap.ISO8859_9.NewDecoder(),
+	"iso-8859-10":    charmap.ISO8859_10.NewDecoder(),
+	"iso-8859-13":    charmap.ISO8859_13.NewDecoder(),
+	"iso-8859-14":    charmap.ISO8859_14.NewDecoder(),
+	"iso-8859-15":    charmap.ISO8859_15.NewDecoder(),
+	"iso-8859-16":    charmap.ISO8859_16.NewDecoder(),
+	"koi8-r":         charmap.KOI8R.NewDecoder(),
+	"koi8-u":         charmap.KOI8U.NewDecoder(),
+	"macintosh":      charmap.Macintosh.NewDecoder(),
+	"x-mac-cyrillic": charmap.MacintoshCyrillic.NewDecoder(),
+	"windows-874":    charmap.Windows874.NewDecoder(),
+	"windows-1250":   charmap.Windows1250.NewDecoder(),
+	"windows-1251":   charmap.Windows1251.NewDecoder(),
+	"windows-1252":   charmap.Windows1252.NewDecoder(),
+	"windows-1253":   charmap.Windows1253.NewDecoder(),
+	"windows-1254":   charmap.Windows1254.NewDecoder(),
+	"windows-1255":   charmap.Windows1255.NewDecoder(),
+	"windows-1257":   charmap.Windows1257.NewDecoder(),
+	"windows-1258":   charmap.Windows1258.NewDecoder(),
+	"x-user-defined": charmap.XUserDefined.NewDecoder(),
+	"euc-jp":         japanese.EUCJP.NewDecoder(),
+	"iso-2022-jp":    japanese.ISO2022JP.NewDecoder(),
+	"shift_jis":      japanese.ShiftJIS.NewDecoder(),
+	"ks_c_5601-1987": korean.EUCKR.NewDecoder(),
+	"euc-kr":         korean.EUCKR.NewDecoder(),
+}
+
+// 定义一个自定义的 CharsetReader 函数,它能够处理 gb2312 和 gbk 字符集
+func myCharsetReader(charset string, input io.Reader) (io.Reader, error) {
+	charset = strings.ToLower(charset)
+	newDecoder, ok := charsetMap[charset]
+	if ok {
+		reader := transform.NewReader(input, newDecoder)
+		return reader, nil
+	}
+	if charset == `utf-8` {
+		return input, nil
+	}
+
+	switch strings.ToLower(charset) {
+	case "gb2312", "gbk":
+		reader := transform.NewReader(input, simplifiedchinese.GBK.NewDecoder())
+		return reader, nil
+	case "utf-8":
+		return input, nil
+	case "iso-8859-1":
+		reader := transform.NewReader(input, charmap.ISO8859_1.NewDecoder())
+
+		return reader, nil
+	case "windows-1252":
+		reader := transform.NewReader(input, charmap.Windows1252.NewDecoder())
+		return reader, nil
+	default:
+	}
+	return input, fmt.Errorf("unsupported charset: %s", charset)
+
+}

+ 366 - 0
utils/mail/imap.go

@@ -0,0 +1,366 @@
+package mail
+
+import (
+	"eta/eta_crawler/utils"
+	"fmt"
+	"io"
+	"log"
+	"os"
+	"path"
+	"regexp"
+	"strings"
+	"time"
+
+	"github.com/emersion/go-imap"
+	"github.com/emersion/go-imap/client"
+	"github.com/emersion/go-message"
+	"github.com/emersion/go-message/mail"
+	"github.com/h2non/filetype"
+)
+
+type MailMessage struct {
+	Date        time.Time         `description:"收件时间"`
+	Uid         uint32            `description:"该邮件在邮箱中的唯一id"`
+	FromAddress string            `description:"发件人邮箱"`
+	From        string            `description:"发件人名称"`
+	Title       string            `description:"邮件标题"`
+	Content     string            `description:"邮件主体正文"`
+	Resources   map[string]string `description:"正文内嵌资源"`
+	Attachment  map[string][]byte `description:"附件资源"`
+}
+
+func ListenMail(mailAddress, folder, userName, password string, readBatchSize, lastNday int) (err error) { // 收件箱
+	defer func() {
+		// 处理结束
+		if err != nil {
+			fmt.Println("err:", err.Error())
+			utils.FileLog.Info("中国煤炭网邮件监听, err:%s", err.Error())
+		}
+	}()
+	// 建立与 IMAP 服务器的连接
+	c, err := client.DialTLS(mailAddress, nil)
+	if err != nil {
+		fmt.Printf("连接 IMAP 服务器失败: %+v \n", err)
+		return
+	}
+
+	// 最后一定不要忘记退出登录
+	defer func() {
+		_ = c.Logout()
+	}()
+
+	// 登录
+	if err = c.Login(userName, password); err != nil {
+		fmt.Printf("邮箱[%s] 登录失败: %v \n", fmt.Sprintf("%s:%s", userName, mailAddress), err)
+		return
+	}
+	// 列出当前邮箱中的文件夹
+	mailboxes := make(chan *imap.MailboxInfo, 10)
+	done := make(chan error, 1) // 记录错误的 chan
+	go func() {
+		done <- c.List("", "*", mailboxes)
+	}()
+	utils.FileLog.Info("-->当前邮箱的文件夹 Mailboxes:")
+
+	var folderExists bool
+	for m := range mailboxes {
+		utils.FileLog.Info("* %s", m.Name)
+		if m.Name == folder {
+			folderExists = true
+		}
+	}
+
+	err = <-done
+	if err != nil {
+		utils.FileLog.Error("列出邮箱列表时,出现错误:%v \n", err)
+		return
+	}
+
+	utils.FileLog.Info("-->列出邮箱列表完毕!")
+	if !folderExists {
+		err = fmt.Errorf("文件夹[%s] 不存在", folder)
+		return
+	}
+
+	message.CharsetReader = myCharsetReader
+
+	// 选择指定的文件夹
+	mbox, err := c.Select(folder, false)
+	if err != nil {
+		err = fmt.Errorf("选择邮件箱失败: %+v", err)
+		return
+	}
+	utils.FileLog.Info("当前文件夹[%s]中,总共有 %d 封邮件", folder, mbox.Messages)
+	if mbox.Messages == 0 {
+		return
+	}
+
+	// 创建一个序列集,用于批量读取邮件
+	seqSet := new(imap.SeqSet)
+	to := mbox.Messages // 此文件下的邮件总数
+
+	now := time.Now()
+	startTime := time.Date(now.Year(), now.Month(), now.Day()-lastNday, 0, 0, 0, 0, time.Local)
+
+	var isStopFor bool
+	step := uint32(1)
+	for i := to; i >= 1; {
+		start := i - step + 1
+
+		seqSet.Clear()
+		seqSet.AddRange(start, i) // 添加指定范围内的邮件编号
+
+		// 获取整个消息正文
+		// imap.FetchEnvelope:请求获取邮件的信封数据(例如发件人、收件人、主题等元数据)。
+		// imap.FetchRFC822:请求获取完整的邮件内容,包括所有头部和正文。
+		items := []imap.FetchItem{imap.FetchFlags, imap.FetchEnvelope, imap.FetchRFC822, imap.FetchBodyStructure}
+
+		// 获取邮件内容 Start
+		messages := make(chan *imap.Message, readBatchSize) // 创建一个通道,用于接收邮件消息
+		fetchDone := make(chan error, 1)                    // 创建一个通道,用于接收错误消息
+		go func() {
+			// Fetch方法用于从服务器获取邮件数据,这里请求了邮件的信封和完整内容
+			fetchDone <- c.Fetch(seqSet, items, messages)
+		}()
+
+		err = <-fetchDone
+		if err != nil {
+			utils.FileLog.Error("获取邮件信息出现错误:%v \n", err)
+			return
+		}
+		// 获取邮件内容 End
+		for msg := range messages {
+			// 如果需要终止,那么就不处理了
+			if isStopFor {
+				continue
+			}
+			// 判断当前邮件收件时间是否小于设定的时间,如果是,那么就不处理了
+			envelope := msg.Envelope
+			if envelope != nil {
+				if envelope.Date.Before(startTime) {
+					continue
+				}
+			} else {
+				continue
+			}
+			utils.FileLog.Info("正在读取邮件uid: %d", msg.Uid)
+
+			emailMessage, tmpErr := readEveryMsg(msg)
+			if tmpErr != nil {
+				// 移除本地文件
+				{
+					for _, v := range emailMessage.Resources {
+						os.Remove(v)
+					}
+				}
+				utils.FileLog.Error("读取邮件内容时出现错误:%v \n", tmpErr)
+				continue
+			}
+
+		}
+
+		if isStopFor {
+			// 已经找到了最小的邮件id,那么就退出循环了
+			break
+		}
+		i = i - step
+	}
+
+	utils.FileLog.Info("读取了所有邮件,完毕!")
+	return
+}
+
+// document link: https://github.com/emersion/go-imap/wiki/Fetching-messages
+func readEveryMsg(msg *imap.Message) (emailMessage MailMessage, err error) {
+	defer func() {
+		if err != nil {
+			utils.FileLog.Error("邮件读取失败;Err:%s", err.Error())
+		}
+	}()
+	message.CharsetReader = myCharsetReader
+	emailMessage.Resources = make(map[string]string)  // 内嵌资源
+	emailMessage.Attachment = make(map[string][]byte) // 附件
+
+	emailMessage.Uid = msg.Uid
+	htmlStr := ``
+	textStr := ``
+
+	// 获取邮件正文
+	r := msg.GetBody(&imap.BodySectionName{})
+	if r == nil {
+		utils.FileLog.Info("服务器没有返回消息内容")
+	}
+
+	mr, err := mail.CreateReader(r)
+	if err != nil {
+		err = fmt.Errorf("邮件读取时出现错误:%v", err)
+		return
+	}
+
+	// 收件时间
+	{
+		date, err := mr.Header.Date()
+		if err != nil {
+			log.Println("收件时间 异常:", err.Error())
+		}
+		emailMessage.Date = date
+	}
+
+	// 发件人
+	{
+		fromStr := mr.Header.Get("From")
+		// 处理无效地址的情况
+		if !strings.Contains(fromStr, "@") {
+			emailMessage.FromAddress = fromStr
+			emailMessage.From = fromStr
+		} else {
+			from, tmpErr := mr.Header.AddressList("From")
+			if tmpErr != nil {
+				log.Println("发件人 异常:", err.Error())
+			}
+			if len(from) > 0 {
+				emailMessage.FromAddress = from[0].Address
+				emailMessage.From = from[0].Name
+			}
+		}
+	}
+
+	// 邮件标题
+	subject, err := mr.Header.Subject()
+	if err != nil {
+		utils.FileLog.Warning("邮件主题 Subject ERR:%v", err)
+	}
+	emailMessage.Title = subject
+
+	for {
+		p, tmpErr := mr.NextPart()
+		if tmpErr == io.EOF {
+			break
+		} else if tmpErr != nil {
+			utils.FileLog.Error("读取邮件内容时出现错误:%v", tmpErr)
+			err = tmpErr
+			return
+		}
+
+		bodyBytes, _ := io.ReadAll(p.Body)
+		if err != nil {
+			err = fmt.Errorf("读取邮件部分时出现错误:%v", err)
+			return
+		}
+
+		switch h := p.Header.(type) {
+		case *mail.InlineHeader:
+			// 这是消息的文本(可以是纯文本或 HTML)
+			contentType := h.Get("Content-Type")
+			if strings.HasPrefix(contentType, "text/plain") {
+				textStr += string(bodyBytes)
+			} else if strings.HasPrefix(contentType, "text/html") {
+				htmlStr += string(bodyBytes)
+			}
+			// 这是内嵌资源
+			if cid := p.Header.Get("Content-ID"); cid != "" {
+
+				// 确定文件后缀
+				fileSuffix := determineFileSuffix(bodyBytes)
+				fileName := fmt.Sprintf("%s%s.%s", utils.CoalFilePath, cid[1:len(cid)-1], fileSuffix)
+
+				err = SaveToFile(bodyBytes, fileName)
+				if err != nil {
+					err = fmt.Errorf("保存文件时出现错误:%v", err)
+					return
+				}
+				emailMessage.Resources[cid] = fileName
+			}
+		case *mail.AttachmentHeader:
+			// 这是一个附件
+			filename, _ := h.Filename()
+			fmt.Printf("读取到到附件: %s \n", filename)
+			utils.FileLog.Info("读取到附件: %s ", filename)
+			if !IsMatchExt(filename) {
+				continue
+			}
+			filePath := fmt.Sprintf("%s%s%s%s", utils.CoalFilePath, `file`, string(os.PathSeparator), filename)
+			err = SaveToFile(bodyBytes, filePath)
+			if err != nil {
+				err = fmt.Errorf("保存文件时出现错误:%v", err)
+				return
+			}
+			fmt.Printf("保存到文件: %s \n", filePath)
+			utils.FileLog.Info("保存到文件: %s ", filePath)
+			// 这是附件资源
+			if contentDisposition := p.Header.Get("Content-Disposition"); contentDisposition != "" {
+				if strings.HasPrefix(contentDisposition, "attachment") {
+					emailMessage.Attachment[filename] = bodyBytes
+				}
+			} else if cid := p.Header.Get("Content-ID"); cid != "" {
+				// 这是内嵌资源
+				emailMessage.Resources[cid] = filePath
+			}
+
+		default:
+			utils.FileLog.Info("未知格式:", h)
+		}
+	}
+	emailMessage.Content = htmlStr
+	if emailMessage.Content == `` {
+		emailMessage.Content = textStr
+	}
+
+	return
+}
+
+// 根据文件内容确定文件后缀
+func determineFileSuffix(content []byte) string {
+	kind, err := filetype.Match(content)
+	if err != nil {
+		utils.FileLog.Error("无法确定文件类型:%v \n", err)
+		return ".bin"
+	}
+	return kind.Extension
+}
+
+func SaveToFile(content []byte, fileName string) error {
+	file, err := os.Create(fileName)
+	if err != nil {
+		return err
+	}
+	defer func() {
+		_ = file.Close()
+	}()
+
+	_, err = file.Write(content)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+// ContainsWholeWord 检查字符串 s 中是否包含完整的单词 word。
+// 该函数使用正则表达式来匹配整个单词,确保不会错误地匹配到单词的一部分。
+// 参数:
+//
+//	s: 要搜索的字符串
+//	word: 要查找的完整单词
+//
+// 返回值:
+//
+//	如果 s 中包含完整的单词 word,则返回 true;否则返回 false。
+func ContainsWholeWord(s string, word string) bool {
+	pattern := fmt.Sprintf(`\b%s\b`, regexp.QuoteMeta(word))
+	re := regexp.MustCompile(pattern)
+	return re.MatchString(s)
+}
+
+func IsMatchExt(filename string) (ok bool) {
+	exts := utils.CoalEmailFileExt
+	extArr := strings.Split(exts, "|")
+	for _, ext := range extArr {
+		ex := strings.ToLower(path.Ext(filename))
+		if ext == ex {
+			ok = true
+			return
+		}
+	}
+	return
+}