소스 검색

add:邮箱监听

zqbao 4 달 전
부모
커밋
dda0640791
14개의 변경된 파일1501개의 추가작업 그리고 9개의 파일을 삭제
  1. 15 0
      .vscode/launch.json
  2. 5 0
      go.mod
  3. 13 0
      go.sum
  4. 40 7
      models/base_from_coalmine.go
  5. 60 0
      services/base.go
  6. 622 0
      services/commodity_coal.go
  7. 158 0
      services/email/mail.go
  8. 67 0
      services/mtjh_watch.go
  9. 9 0
      services/task.go
  10. 9 2
      utils/common.go
  11. 28 0
      utils/config.go
  12. 12 0
      utils/constants.go
  13. 94 0
      utils/mail/charset_reader.go
  14. 369 0
      utils/mail/imap.go

+ 15 - 0
.vscode/launch.json

@@ -0,0 +1,15 @@
+{
+    // 使用 IntelliSense 了解相关属性。 
+    // 悬停以查看现有属性的描述。
+    // 欲了解更多信息,请访问: https://go.microsoft.com/fwlink/?linkid=830387
+    "version": "0.2.0",
+    "configurations": [
+        {
+            "name": "Launch Package",
+            "type": "go",
+            "request": "launch",
+            "mode": "auto",
+            "program": "${workspaceFolder}"
+        }
+    ]
+}

+ 5 - 0
go.mod

@@ -8,8 +8,12 @@ require (
 	github.com/beego/beego/v2 v2.1.0
 	github.com/chromedp/cdproto v0.0.0-20240312231614-1e5096e63154
 	github.com/chromedp/chromedp v0.9.5
+	github.com/emersion/go-imap v1.2.1
+	github.com/emersion/go-message v0.18.1
 	github.com/go-sql-driver/mysql v1.8.0
+	github.com/h2non/filetype v1.1.3
 	github.com/mozillazg/go-pinyin v0.20.0
+	github.com/patrickmn/go-cache v2.1.0+incompatible
 	github.com/rdlucklib/rdluck_tools v1.0.3
 	github.com/shopspring/decimal v1.3.1
 	github.com/tealeg/xlsx v1.0.5
@@ -23,6 +27,7 @@ require (
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/cespare/xxhash/v2 v2.2.0 // indirect
 	github.com/chromedp/sysutil v1.0.0 // indirect
+	github.com/emersion/go-sasl v0.0.0-20200509203442-7bfe0ed36a21 // indirect
 	github.com/gobwas/httphead v0.1.0 // indirect
 	github.com/gobwas/pool v0.2.1 // indirect
 	github.com/gobwas/ws v1.3.2 // indirect

+ 13 - 0
go.sum

@@ -48,6 +48,14 @@ github.com/elastic/go-elasticsearch/v6 v6.8.5/go.mod h1:UwaDJsD3rWLM5rKNFzv9hgox
 github.com/elazarl/go-bindata-assetfs v1.0.0/go.mod h1:v+YaWX3bdea5J/mo8dSETolEo7R71Vk1u8bnjau5yw4=
 github.com/elazarl/go-bindata-assetfs v1.0.1 h1:m0kkaHRKEu7tUIUFVwhGGGYClXvyl4RE03qmvRTNfbw=
 github.com/elazarl/go-bindata-assetfs v1.0.1/go.mod h1:v+YaWX3bdea5J/mo8dSETolEo7R71Vk1u8bnjau5yw4=
+github.com/emersion/go-imap v1.2.1 h1:+s9ZjMEjOB8NzZMVTM3cCenz2JrQIGGo5j1df19WjTA=
+github.com/emersion/go-imap v1.2.1/go.mod h1:Qlx1FSx2FTxjnjWpIlVNEuX+ylerZQNFE5NsmKFSejY=
+github.com/emersion/go-message v0.15.0/go.mod h1:wQUEfE+38+7EW8p8aZ96ptg6bAb1iwdgej19uXASlE4=
+github.com/emersion/go-message v0.18.1 h1:tfTxIoXFSFRwWaZsgnqS1DSZuGpYGzSmCZD8SK3QA2E=
+github.com/emersion/go-message v0.18.1/go.mod h1:XpJyL70LwRvq2a8rVbHXikPgKj8+aI0kGdHlg16ibYA=
+github.com/emersion/go-sasl v0.0.0-20200509203442-7bfe0ed36a21 h1:OJyUGMJTzHTd1XQp98QTaHernxMYzRaOasRir9hUlFQ=
+github.com/emersion/go-sasl v0.0.0-20200509203442-7bfe0ed36a21/go.mod h1:iL2twTeMvZnrg54ZoPDNfJaJaqy0xIQFuBdrLsmspwQ=
+github.com/emersion/go-textwrapper v0.0.0-20200911093747-65d896831594/go.mod h1:aqO8z8wPrjkscevZJFVE1wXJrLpC5LtJG7fqLOsPb2U=
 github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
 github.com/garyburd/redigo v1.6.3/go.mod h1:rTb6epsqigu3kYKBnaF028A7Tf/Aw5s0cqA47doKKqw=
 github.com/glendc/gopher-json v0.0.0-20170414221815-dc4743023d0c/go.mod h1:Gja1A+xZ9BoviGJNA2E9vFkPjjsl+CoJxSXiQM1UXtw=
@@ -91,6 +99,8 @@ github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/
 github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=
 github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
 github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
+github.com/h2non/filetype v1.1.3 h1:FKkx9QbD7HR/zjK1Ia5XiBsq9zdLi5Kf3zGyFTAFkGg=
+github.com/h2non/filetype v1.1.3/go.mod h1:319b3zT68BvV+WRj7cwy856M2ehB3HqNOt6sy1HndBY=
 github.com/hashicorp/golang-lru v0.5.4 h1:YDjusn29QI/Das2iO9M0BHnIbxPeyuCHsjMW+lJfyTc=
 github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4=
 github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
@@ -138,6 +148,8 @@ github.com/onsi/ginkgo v1.12.0/go.mod h1:oUhWkIvk5aDxtKvDDuw8gItl8pKl42LzjC9KZE0
 github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY=
 github.com/orisano/pixelmatch v0.0.0-20220722002657-fb0b55479cde h1:x0TT0RDC7UhAVbbWWBzr41ElhJx5tXPWkIHA2HWPRuw=
 github.com/orisano/pixelmatch v0.0.0-20220722002657-fb0b55479cde/go.mod h1:nZgzbfBr3hhjoZnS66nKrHmduYNpc34ny7RK4z5/HM0=
+github.com/patrickmn/go-cache v2.1.0+incompatible h1:HRMgzkcYKYpi3C8ajMPV8OFXaaRUnok+kx1WdO15EQc=
+github.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ=
 github.com/pelletier/go-toml v1.0.1/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
 github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
 github.com/peterh/liner v1.0.1-0.20171122030339-3681c2a91233/go.mod h1:xIteQHvHuaLYG9IFj6mSxM0fCKrs34IrEQUhOYuGPHc=
@@ -261,6 +273,7 @@ golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
 golang.org/x/term v0.7.0/go.mod h1:P32HKFT3hSsZrRxla30E9HqToFYAQPCMs/zFMBUFqPY=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
 golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
 golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=

+ 40 - 7
models/base_from_coalmine.go

@@ -1,10 +1,43 @@
 package models
 
 import (
-	"github.com/beego/beego/v2/client/orm"
 	"time"
+
+	"github.com/beego/beego/v2/client/orm"
 )
 
+type CoalSheetData struct {
+	Name     string
+	Rows     []Row
+	Cols     []*Col
+	MaxRow   int
+	MaxCol   int
+	Hidden   bool
+	Selected bool
+}
+
+type Row struct {
+	Cells        []Cell
+	Hidden       bool
+	Height       float64
+	OutlineLevel uint8
+	isCustom     bool
+}
+
+type Col struct {
+	Min          int
+	Max          int
+	Hidden       bool
+	Width        float64
+	Collapsed    bool
+	OutlineLevel uint8
+	numFmt       string
+}
+
+type Cell struct {
+	Value string
+}
+
 type BaseFromCoalmineMapping struct {
 	BaseFromCoalmineMappingId int       `orm:"column(base_from_coalmine_mapping_id);pk"`
 	IndexName                 string    `description:"持买单量指标名称"`
@@ -44,14 +77,14 @@ type BaseFromCoalmineCompanyIndex struct {
 	ModifyTime                     time.Time `description:"修改时间"`
 }
 
-//添加指标
+// 添加指标
 func AddBaseFromCoalmineMapping(item *BaseFromCoalmineMapping) (lastId int64, err error) {
 	o := orm.NewOrmUsingDB("data")
 	lastId, err = o.Insert(item)
 	return
 }
 
-//查询指标
+// 查询指标
 func GetBaseFromCoalmineMapping() (items []*BaseFromCoalmineMapping, err error) {
 	o := orm.NewOrmUsingDB("data")
 	sql := `SELECT * FROM base_from_coalmine_mapping`
@@ -59,7 +92,7 @@ func GetBaseFromCoalmineMapping() (items []*BaseFromCoalmineMapping, err error)
 	return
 }
 
-//查询数据
+// 查询数据
 func GetBaseFromCoalmineIndex() (items []*BaseFromCoalmineJsmIndex, err error) {
 	o := orm.NewOrmUsingDB("data")
 	sql := `SELECT * FROM base_from_coalmine_jsm_index`
@@ -74,21 +107,21 @@ func UpdateBaseFromCoalmineIndex(item *BaseFromCoalmineJsmIndex) (err error) {
 	return
 }
 
-//添加数据
+// 添加数据
 func AddBaseFromCoalmineIndex(item *BaseFromCoalmineJsmIndex) (lastId int64, err error) {
 	o := orm.NewOrmUsingDB("data")
 	lastId, err = o.Insert(item)
 	return
 }
 
-//添加公司指标
+// 添加公司指标
 func AddBaseFromCoalmineCompanyIndex(item *BaseFromCoalmineCompanyIndex) (lastId int64, err error) {
 	o := orm.NewOrmUsingDB("data")
 	lastId, err = o.Insert(item)
 	return
 }
 
-//查询公司指标
+// 查询公司指标
 func GetBaseFromCoalmineCompanyIndex() (items []*BaseFromCoalmineCompanyIndex, err error) {
 	o := orm.NewOrmUsingDB("data")
 	sql := `SELECT * FROM base_from_coalmine_company_index`

+ 60 - 0
services/base.go

@@ -0,0 +1,60 @@
+package services
+
+import (
+	"encoding/json"
+	"eta/eta_crawler/utils"
+	"fmt"
+	"io/ioutil"
+	"net/http"
+	"strings"
+)
+
+// PostEdbLib 调用指标接口
+func PostEdbLib(param map[string]interface{}, method string) (result []byte, err error) {
+	postUrl := utils.EDB_LIB_URL + method
+	postData, err := json.Marshal(param)
+	if err != nil {
+		return
+	}
+	result, err = HttpPost(postUrl, string(postData), "application/json")
+	if err != nil {
+		return
+	}
+	return
+}
+
+func HttpPost(url, postData string, params ...string) ([]byte, error) {
+	fmt.Println("HttpPost Url:" + url)
+	body := ioutil.NopCloser(strings.NewReader(postData))
+	client := &http.Client{}
+	req, err := http.NewRequest("POST", url, body)
+	if err != nil {
+		return nil, err
+	}
+	contentType := "application/x-www-form-urlencoded;charset=utf-8"
+	if len(params) > 0 && params[0] != "" {
+		contentType = params[0]
+	}
+	req.Header.Set("Content-Type", contentType)
+	req.Header.Set("authorization", utils.MD5(utils.APP_EDB_LIB_NAME_EN+utils.EDB_LIB_Md5_KEY))
+	resp, err := client.Do(req)
+	if err != nil {
+		fmt.Println("client.Do err:" + err.Error())
+		return nil, err
+	}
+	defer resp.Body.Close()
+	b, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		fmt.Println("HttpPost:" + string(b))
+	}
+	return b, err
+}
+
+func HttpGet(url string) ([]byte, error) {
+	res, err := http.Get(url)
+	if err != nil {
+		return nil, err
+	}
+	defer res.Body.Close()
+	return ioutil.ReadAll(res.Body)
+}

+ 622 - 0
services/commodity_coal.go

@@ -0,0 +1,622 @@
+package services
+
+import (
+	"encoding/json"
+	"eta/eta_crawler/models"
+	"eta/eta_crawler/utils"
+	"fmt"
+
+	"github.com/tealeg/xlsx"
+)
+
+func JsmHistory(path string) (err error) {
+	defer func() {
+		if err != nil {
+			fmt.Println("RefreshDataFromCoalCoastal  Err:" + err.Error())
+			utils.FileLog.Info(fmt.Sprintf("RefreshDataFromCoalCoastal, Err: %s", err))
+		}
+	}()
+
+	var xlFile *xlsx.File
+	exist, err := PathExists(path)
+	if err != nil {
+		fmt.Println(err)
+		return
+	}
+	if exist {
+		xlFile, err = xlsx.OpenFile(path)
+		if err != nil {
+			fmt.Println("OpenFile err:", err)
+			return
+		}
+	} else {
+		fmt.Println("Not Exist")
+		return
+	}
+
+	sheetDatas := make([]models.CoalSheetData, 0)
+	data := *xlFile.Sheets[0]
+	sheetData := models.CoalSheetData{
+		Name:     data.Name,
+		MaxRow:   data.MaxRow,
+		MaxCol:   data.MaxCol,
+		Hidden:   data.Hidden,
+		Selected: data.Selected,
+	}
+	rows := make([]models.Row, 0)
+	for _, v := range data.Rows {
+		cells := make([]models.Cell, 0)
+		for _, cell := range v.Cells {
+			cells = append(cells, models.Cell{
+				Value: cell.String(),
+			})
+		}
+		row := models.Row{
+			Cells: cells,
+		}
+		rows = append(rows, row)
+	}
+	sheetData.Rows = rows
+	sheetDatas = append(sheetDatas, sheetData)
+
+	params := make(map[string]interface{})
+	params["SheetData"] = sheetDatas
+
+	result, e := PostEdbLib(params, utils.LIB_ROUTE_COAL_MINE_JSM_HISTORY)
+	if e != nil {
+		b, _ := json.Marshal(params)
+		fmt.Println(e)
+		utils.FileLog.Info(fmt.Sprintf("PostEdbLib err: %s, params: %s", e.Error(), string(b)))
+		return
+	}
+	resp := new(models.BaseResponse)
+	if e := json.Unmarshal(result, &resp); e != nil {
+		utils.FileLog.Info(fmt.Sprintf("json.Unmarshal err: %s", e))
+		return
+	}
+	if resp.Ret != 200 {
+		utils.FileLog.Info(fmt.Sprintf("Msg: %s, ErrMsg: %s", resp.Msg, resp.ErrMsg))
+		return
+	}
+
+	return
+}
+
+func CoastalHistory(path string) (err error) {
+	defer func() {
+		if err != nil {
+			fmt.Println("RefreshDataFromCoalCoastal  Err:" + err.Error())
+			utils.FileLog.Info(fmt.Sprintf("RefreshDataFromCoalCoastal, Err: %s", err))
+		}
+	}()
+	//path := "/Users/xi/Desktop/瑞茂通-中国煤炭市场网数据/442家晋陕蒙、沿海8省、内陆17省历史数据/CⅢ-8-16 25省市库存和日耗情况(CCTD).xlsx"
+	//path := "D:\\瑞茂通-中国煤炭市场网数据\\442家晋陕蒙、沿海8省、内陆17省历史数据\\CⅢ-8-16 25省市库存和日耗情况(CCTD).xlsx"
+
+	var xlFile *xlsx.File
+	exist, err := PathExists(path)
+	if err != nil {
+		fmt.Println(err)
+		return
+	}
+	if exist {
+		xlFile, err = xlsx.OpenFile(path)
+		if err != nil {
+			fmt.Println("OpenFile err:", err)
+			return
+		}
+	} else {
+		fmt.Println("Not Exist")
+		return
+	}
+
+	sheetDatas := make([]models.CoalSheetData, 0)
+	for i, sheet := range xlFile.Sheets {
+		if i < 4 {
+			data := sheet
+			sheetData := models.CoalSheetData{
+				Name:     data.Name,
+				MaxRow:   data.MaxRow,
+				MaxCol:   data.MaxCol,
+				Hidden:   data.Hidden,
+				Selected: data.Selected,
+			}
+			rows := make([]models.Row, 0)
+			for _, v := range data.Rows {
+				cells := make([]models.Cell, 0)
+				for _, cell := range v.Cells {
+					cells = append(cells, models.Cell{
+						Value: cell.String(),
+					})
+				}
+				row := models.Row{
+					Cells: cells,
+				}
+				rows = append(rows, row)
+			}
+			sheetData.Rows = rows
+			sheetDatas = append(sheetDatas, sheetData)
+		}
+	}
+
+	params := make(map[string]interface{})
+	params["SheetData"] = sheetDatas
+	result, e := PostEdbLib(params, utils.LIB_ROUTE_COAL_MINE_COASTAL_HISTORY)
+	if e != nil {
+		b, _ := json.Marshal(params)
+		fmt.Println(e)
+		utils.FileLog.Info(fmt.Sprintf("PostEdbLib err: %s, params: %s", e.Error(), string(b)))
+		return
+	}
+	resp := new(models.BaseResponse)
+	if e := json.Unmarshal(result, &resp); e != nil {
+		utils.FileLog.Info(fmt.Sprintf("json.Unmarshal err: %s", e))
+		return
+	}
+	if resp.Ret != 200 {
+		utils.FileLog.Info(fmt.Sprintf("Msg: %s, ErrMsg: %s", resp.Msg, resp.ErrMsg))
+		return
+	}
+
+	return
+}
+
+func InlandHistory(path string) (err error) {
+	defer func() {
+		if err != nil {
+			fmt.Println("RefreshDataFromCoalCoastal  Err:" + err.Error())
+			utils.FileLog.Info(fmt.Sprintf("RefreshDataFromCoalCoastal, Err: %s", err))
+		}
+	}()
+	//path := "/Users/xi/Desktop/瑞茂通-中国煤炭市场网数据/442家晋陕蒙、沿海8省、内陆17省历史数据/CⅢ-8-16 25省市库存和日耗情况(CCTD).xlsx"
+	//path := "D:\\瑞茂通-中国煤炭市场网数据\\442家晋陕蒙、沿海8省、内陆17省历史数据\\CⅢ-8-16 25省市库存和日耗情况(CCTD).xlsx"
+
+	var xlFile *xlsx.File
+	exist, err := PathExists(path)
+	if err != nil {
+		fmt.Println(err)
+		return
+	}
+	if exist {
+		xlFile, err = xlsx.OpenFile(path)
+		if err != nil {
+			fmt.Println("OpenFile err:", err)
+			return
+		}
+	} else {
+		fmt.Println("Not Exist")
+		return
+	}
+
+	sheetDatas := make([]models.CoalSheetData, 0)
+	for i, sheet := range xlFile.Sheets {
+		if i > 3 {
+			data := sheet
+			sheetData := models.CoalSheetData{
+				Name:     data.Name,
+				MaxRow:   data.MaxRow,
+				MaxCol:   data.MaxCol,
+				Hidden:   data.Hidden,
+				Selected: data.Selected,
+			}
+			rows := make([]models.Row, 0)
+			for _, v := range data.Rows {
+				cells := make([]models.Cell, 0)
+				for _, cell := range v.Cells {
+					cells = append(cells, models.Cell{
+						Value: cell.String(),
+					})
+				}
+				row := models.Row{
+					Cells: cells,
+				}
+				rows = append(rows, row)
+			}
+			sheetData.Rows = rows
+			sheetDatas = append(sheetDatas, sheetData)
+		}
+	}
+
+	params := make(map[string]interface{})
+	params["SheetData"] = sheetDatas
+	result, e := PostEdbLib(params, utils.LIB_ROUTE_COAL_MINE_INLAND_HISTORY)
+	if e != nil {
+		b, _ := json.Marshal(params)
+		fmt.Println(e)
+		utils.FileLog.Info(fmt.Sprintf("PostEdbLib err: %s, params: %s", e.Error(), string(b)))
+		return
+	}
+	resp := new(models.BaseResponse)
+	if e := json.Unmarshal(result, &resp); e != nil {
+		utils.FileLog.Info(fmt.Sprintf("json.Unmarshal err: %s", e))
+		return
+	}
+	if resp.Ret != 200 {
+		utils.FileLog.Info(fmt.Sprintf("Msg: %s, ErrMsg: %s", resp.Msg, resp.ErrMsg))
+		return
+	}
+
+	return
+}
+
+func Jsm(path string) (err error) {
+	defer func() {
+		if err != nil {
+			fmt.Println("RefreshDataFromCoalCoastal  Err:" + err.Error())
+			utils.FileLog.Info(fmt.Sprintf("RefreshDataFromCoalCoastal, Err: %s", err))
+		}
+	}()
+	//path := "/home/code/python/coal_mail/emailFile/沿海八省动力煤终端用户供耗存数据更新.xlsx"
+	//path := "/Users/xi/Desktop/瑞茂通-中国煤炭市场网数据/442家晋陕蒙、沿海8省、内陆17省最新数据/442家晋陕蒙煤矿周度产量数据-20231201.xlsx"
+	//path := "D:\\瑞茂通-中国煤炭市场网数据\\442家晋陕蒙、沿海8省、内陆17省历史数据\\442家晋陕蒙历史数据.xlsx"
+
+	var xlFile *xlsx.File
+	exist, err := PathExists(path)
+	if err != nil {
+		fmt.Println(err)
+		return
+	}
+	if exist {
+		xlFile, err = xlsx.OpenFile(path)
+		if err != nil {
+			fmt.Println("OpenFile err:", err)
+			return
+		}
+	} else {
+		fmt.Println("Not Exist")
+		return
+	}
+
+	sheetDatas := make([]models.CoalSheetData, 0)
+	data := *xlFile.Sheets[0]
+	sheetData := models.CoalSheetData{
+		Name:     data.Name,
+		MaxRow:   data.MaxRow,
+		MaxCol:   data.MaxCol,
+		Hidden:   data.Hidden,
+		Selected: data.Selected,
+	}
+	rows := make([]models.Row, 0)
+	for _, v := range data.Rows {
+		cells := make([]models.Cell, 0)
+		for _, cell := range v.Cells {
+			cells = append(cells, models.Cell{
+				Value: cell.String(),
+			})
+		}
+		row := models.Row{
+			Cells: cells,
+		}
+		rows = append(rows, row)
+	}
+	sheetData.Rows = rows
+	sheetDatas = append(sheetDatas, sheetData)
+
+	params := make(map[string]interface{})
+	params["SheetData"] = sheetDatas
+	result, e := PostEdbLib(params, utils.LIB_ROUTE_COAL_MINE_JSM)
+	if e != nil {
+		b, _ := json.Marshal(params)
+		fmt.Println(e)
+		utils.FileLog.Info(fmt.Sprintf("PostEdbLib err: %s, params: %s", e.Error(), string(b)))
+		return
+	}
+	resp := new(models.BaseResponse)
+	if e := json.Unmarshal(result, &resp); e != nil {
+		utils.FileLog.Info(fmt.Sprintf("json.Unmarshal err: %s", e))
+		return
+	}
+	if resp.Ret != 200 {
+		utils.FileLog.Info(fmt.Sprintf("Msg: %s, ErrMsg: %s", resp.Msg, resp.ErrMsg))
+		return
+	}
+
+	return
+}
+
+func Coastal(path string) (err error) {
+	defer func() {
+		if err != nil {
+			fmt.Println("RefreshDataFromCoalCoastal  Err:" + err.Error())
+			utils.FileLog.Info(fmt.Sprintf("RefreshDataFromCoalCoastal, Err: %s", err))
+		}
+	}()
+	//path := "/Users/xi/Desktop/瑞茂通-中国煤炭市场网数据/442家晋陕蒙、沿海8省、内陆17省最新数据/内陆17省动力煤终端用户供耗存.xlsx"
+	//path := "D:\\瑞茂通-中国煤炭市场网数据\\442家晋陕蒙、沿海8省、内陆17省历史数据\\CⅢ-8-16 25省市库存和日耗情况(CCTD).xlsx"
+
+	fmt.Println("沿海开始")
+	var xlFile *xlsx.File
+	exist, err := PathExists(path)
+	if err != nil {
+		fmt.Println(err)
+		return
+	}
+	if exist {
+		xlFile, err = xlsx.OpenFile(path)
+		if err != nil {
+			fmt.Println("OpenFile err:", err)
+			return
+		}
+	} else {
+		fmt.Println("Not Exist")
+		return
+	}
+
+	sheetDatas := make([]models.CoalSheetData, 0)
+	for _, sheet := range xlFile.Sheets {
+		data := sheet
+		sheetData := models.CoalSheetData{
+			Name:     data.Name,
+			MaxRow:   data.MaxRow,
+			MaxCol:   data.MaxCol,
+			Hidden:   data.Hidden,
+			Selected: data.Selected,
+		}
+		rows := make([]models.Row, 0)
+		for _, v := range data.Rows {
+			cells := make([]models.Cell, 0)
+			for _, cell := range v.Cells {
+				cells = append(cells, models.Cell{
+					Value: cell.String(),
+				})
+			}
+			row := models.Row{
+				Cells: cells,
+			}
+			rows = append(rows, row)
+		}
+		sheetData.Rows = rows
+		fmt.Println("rows:", len(rows))
+		sheetDatas = append(sheetDatas, sheetData)
+	}
+	fmt.Println("sheetDatas:", len(sheetDatas))
+
+	params := make(map[string]interface{})
+	params["SheetData"] = sheetDatas
+	result, e := PostEdbLib(params, utils.LIB_ROUTE_COAL_MINE_COASTAL)
+	if e != nil {
+		b, _ := json.Marshal(params)
+		fmt.Println(e)
+		utils.FileLog.Info(fmt.Sprintf("PostEdbLib err: %s, params: %s", e.Error(), string(b)))
+		return
+	}
+	resp := new(models.BaseResponse)
+	if e := json.Unmarshal(result, &resp); e != nil {
+		utils.FileLog.Info(fmt.Sprintf("json.Unmarshal err: %s", e))
+		return
+	}
+	if resp.Ret != 200 {
+		utils.FileLog.Info(fmt.Sprintf("Msg: %s, ErrMsg: %s", resp.Msg, resp.ErrMsg))
+		return
+	}
+
+	return
+}
+
+func Inland(path string) (err error) {
+	defer func() {
+		if err != nil {
+			fmt.Println("RefreshDataFromCoalCoastal  Err:" + err.Error())
+			utils.FileLog.Info(fmt.Sprintf("RefreshDataFromCoalCoastal, Err: %s", err))
+		}
+	}()
+	//path := "/Users/xi/Desktop/瑞茂通-中国煤炭市场网数据/442家晋陕蒙、沿海8省、内陆17省最新数据/内陆17省动力煤终端用户供耗存.xlsx"
+	//path := "D:\\瑞茂通-中国煤炭市场网数据\\442家晋陕蒙、沿海8省、内陆17省历史数据\\CⅢ-8-16 25省市库存和日耗情况(CCTD).xlsx"
+
+	var xlFile *xlsx.File
+	exist, err := PathExists(path)
+	if err != nil {
+		fmt.Println(err)
+		return
+	}
+	if exist {
+		xlFile, err = xlsx.OpenFile(path)
+		if err != nil {
+			fmt.Println("OpenFile err:", err)
+			return
+		}
+	} else {
+		fmt.Println("Not Exist")
+		return
+	}
+
+	sheetDatas := make([]models.CoalSheetData, 0)
+	for _, sheet := range xlFile.Sheets {
+		data := sheet
+		sheetData := models.CoalSheetData{
+			Name:     data.Name,
+			MaxRow:   data.MaxRow,
+			MaxCol:   data.MaxCol,
+			Hidden:   data.Hidden,
+			Selected: data.Selected,
+		}
+		rows := make([]models.Row, 0)
+		for _, v := range data.Rows {
+			cells := make([]models.Cell, 0)
+			for _, cell := range v.Cells {
+				cells = append(cells, models.Cell{
+					Value: cell.String(),
+				})
+			}
+			row := models.Row{
+				Cells: cells,
+			}
+			rows = append(rows, row)
+		}
+		sheetData.Rows = rows
+		sheetDatas = append(sheetDatas, sheetData)
+	}
+
+	params := make(map[string]interface{})
+	params["SheetData"] = sheetDatas
+	result, e := PostEdbLib(params, utils.LIB_ROUTE_COAL_MINE_INLAND)
+	if e != nil {
+		b, _ := json.Marshal(params)
+		fmt.Println(e)
+		utils.FileLog.Info(fmt.Sprintf("PostEdbLib err: %s, params: %s", e.Error(), string(b)))
+		return
+	}
+	resp := new(models.BaseResponse)
+	if e := json.Unmarshal(result, &resp); e != nil {
+		utils.FileLog.Info(fmt.Sprintf("json.Unmarshal err: %s", e))
+		return
+	}
+	if resp.Ret != 200 {
+		utils.FileLog.Info(fmt.Sprintf("Msg: %s, ErrMsg: %s", resp.Msg, resp.ErrMsg))
+		return
+	}
+
+	return
+}
+
+func Mtjh(path string) (err error) {
+	defer func() {
+		if err != nil {
+			fmt.Println("RefreshDataFromCoalMtjh  Err:" + err.Error())
+			utils.FileLog.Info(fmt.Sprintf("RefreshDataFromCoalMtjh, Err: %s", err))
+		}
+	}()
+	//path = "/Users/xi/Desktop/煤炭江湖数据定制化服务——中国主流港口煤炭库存20231129.xlsx"
+
+	var xlFile *xlsx.File
+	exist, err := PathExists(path)
+	if err != nil {
+		fmt.Println(err)
+		return
+	}
+	if exist {
+		xlFile, err = xlsx.OpenFile(path)
+		if err != nil {
+			fmt.Println("OpenFile err:", err)
+			return
+		}
+	} else {
+		fmt.Println("Not Exist")
+		return
+	}
+
+	sheetDatas := make([]models.CoalSheetData, 0)
+	for i, sheet := range xlFile.Sheets {
+		if i > 0 {
+			break
+		}
+		data := sheet
+		sheetData := models.CoalSheetData{
+			Name:     data.Name,
+			MaxRow:   data.MaxRow,
+			MaxCol:   data.MaxCol,
+			Hidden:   data.Hidden,
+			Selected: data.Selected,
+		}
+		rows := make([]models.Row, 0)
+		for _, v := range data.Rows {
+			cells := make([]models.Cell, 0)
+			for _, cell := range v.Cells {
+				cells = append(cells, models.Cell{
+					Value: cell.String(),
+				})
+			}
+			row := models.Row{
+				Cells: cells,
+			}
+			rows = append(rows, row)
+		}
+		sheetData.Rows = rows
+		sheetDatas = append(sheetDatas, sheetData)
+	}
+
+	params := make(map[string]interface{})
+	params["SheetData"] = sheetDatas
+	result, e := PostEdbLib(params, utils.LIB_ROUTE_COAL_MINE_MTJH)
+	if e != nil {
+		b, _ := json.Marshal(params)
+		fmt.Println(e)
+		utils.FileLog.Info(fmt.Sprintf("PostEdbLib err: %s, params: %s", e.Error(), string(b)))
+		return
+	}
+	resp := new(models.BaseResponse)
+	if e := json.Unmarshal(result, &resp); e != nil {
+		utils.FileLog.Info(fmt.Sprintf("json.Unmarshal err: %s", e))
+		return
+	}
+	if resp.Ret != 200 {
+		utils.FileLog.Info(fmt.Sprintf("Msg: %s, ErrMsg: %s", resp.Msg, resp.ErrMsg))
+		return
+	}
+
+	return
+}
+
+func Firm(path string) (err error) {
+	defer func() {
+		if err != nil {
+			fmt.Println("RefreshDataFromCoalFirm  Err:" + err.Error())
+			utils.FileLog.Info(fmt.Sprintf("RefreshDataFromCoalFirm, Err: %s", err))
+		}
+	}()
+	//path = "/Users/xi/Desktop/煤炭江湖数据定制化服务——中国主流港口煤炭库存20231129.xlsx"
+
+	var xlFile *xlsx.File
+	exist, err := PathExists(path)
+	if err != nil {
+		fmt.Println(err)
+		return
+	}
+	if exist {
+		xlFile, err = xlsx.OpenFile(path)
+		if err != nil {
+			fmt.Println("OpenFile err:", err)
+			return
+		}
+	} else {
+		fmt.Println("Not Exist")
+		return
+	}
+
+	sheetDatas := make([]models.CoalSheetData, 0)
+	for _, sheet := range xlFile.Sheets {
+		data := sheet
+		sheetData := models.CoalSheetData{
+			Name:     data.Name,
+			MaxRow:   data.MaxRow,
+			MaxCol:   data.MaxCol,
+			Hidden:   data.Hidden,
+			Selected: data.Selected,
+		}
+		rows := make([]models.Row, 0)
+		for _, v := range data.Rows {
+			cells := make([]models.Cell, 0)
+			for _, cell := range v.Cells {
+				cells = append(cells, models.Cell{
+					Value: cell.String(),
+				})
+			}
+			row := models.Row{
+				Cells: cells,
+			}
+			rows = append(rows, row)
+		}
+		sheetData.Rows = rows
+		sheetDatas = append(sheetDatas, sheetData)
+	}
+
+	params := make(map[string]interface{})
+	params["SheetData"] = sheetDatas
+	result, e := PostEdbLib(params, utils.LIB_ROUTE_COAL_MINE_FIRM)
+	if e != nil {
+		b, _ := json.Marshal(params)
+		fmt.Println(e)
+		utils.FileLog.Info(fmt.Sprintf("PostEdbLib err: %s, params: %s", e.Error(), string(b)))
+		return
+	}
+	resp := new(models.BaseResponse)
+	if e := json.Unmarshal(result, &resp); e != nil {
+		utils.FileLog.Info(fmt.Sprintf("json.Unmarshal err: %s", e))
+		return
+	}
+	if resp.Ret != 200 {
+		utils.FileLog.Info(fmt.Sprintf("Msg: %s, ErrMsg: %s", resp.Msg, resp.ErrMsg))
+		return
+	}
+
+	return
+}

+ 158 - 0
services/email/mail.go

@@ -0,0 +1,158 @@
+package email
+
+import (
+	"context"
+	"eta/eta_crawler/utils"
+	"eta/eta_crawler/utils/mail"
+	"fmt"
+	"io/fs"
+	"log"
+	"os"
+	"strconv"
+	"sync"
+	"time"
+)
+
+// 同步用户锁,防止重复同步,不管是全量还是增量,都是同一时间只能一个同步
+var lockListenEmail sync.Mutex
+
+// 邮件改名规则
+// var ruleList []report.MailRule
+
+type MailRule struct {
+	Rule       string `json:"rule"`
+	Title      string `json:"title"`
+	Author     string `json:"author"`
+	ClassifyId int    `json:"classify_id"`
+	Abstract   string `json:"abstract"`
+}
+
+func ListenMail(cont context.Context) (err error) {
+	defer func() {
+		if err != nil {
+			utils.FileLog.Error("监听邮件失败:%s", err.Error())
+		}
+	}()
+	//Fix()
+	//return
+	lockListenEmail.Lock()
+	// 目录创建
+	_ = ensureDirExists(fmt.Sprintf("%s%s", utils.MtjhFilePath, `file`))
+
+	mailMessageChan := make(chan mail.MailMessage, 5) // 创建一个通道,用于接收邮件消息
+	mailMessageDoneChan := make(chan bool, 1)         // 创建一个通道,用于接收邮件消息
+
+	// 邮件监听后的处理函数
+	go afterByListen(mailMessageChan, mailMessageDoneChan)
+
+	fmt.Println("开始监听邮件")
+
+	var emailMessageUID int
+	if utils.MtjhEmailStarIndex != "" {
+		emailMessageUID, err = strconv.Atoi(utils.MtjhEmailStarIndex)
+		if err != nil {
+			emailMessageUID = -1
+			utils.FileLog.Warning("读取邮件 MtjhEmailStarIndex 配置失败:%s, 默认改为:%d", err.Error(), emailMessageUID)
+		}
+	}
+	if emailMessageUID <= 0 {
+		// 获取最大的邮件id
+		// emailMessageUID, err = report.GetMaxOutsideReportByEmailMessageId()
+		// 已经存在了,那么就返回
+		if err != nil {
+			utils.FileLog.Error("获取已入库的最大邮件id失败:%s", err.Error())
+		}
+	}
+	var readBatch int
+	if utils.MtjhEmailReadBatch != "" {
+		readBatch, err = strconv.Atoi(utils.MtjhEmailReadBatch)
+		if err != nil {
+			readBatch = 10
+			utils.FileLog.Warning("读取邮件 MtjhEmailReadBatch 配置失败:%s, 默认改为:%d", err.Error(), readBatch)
+		}
+	}
+	mail.ListenMail(utils.MtjhEmailAddress, utils.MtjhEmailFolder, utils.MtjhEmailUseName, utils.MtjhEmailPassword, readBatch, emailMessageUID, mailMessageChan, mailMessageDoneChan)
+	return
+}
+
+func afterByListen(mailMessageChan chan mail.MailMessage, mailMessageDoneChan chan bool) {
+	defer func() {
+		log.Println("监听读取结束")
+		lockListenEmail.Unlock()
+	}()
+	for {
+		select {
+		case emailMessage := <-mailMessageChan:
+			handleMailMessage(emailMessage)
+		case <-time.After(10 * time.Second):
+			return
+		case <-mailMessageDoneChan:
+			for len(mailMessageChan) > 0 {
+				emailMessage := <-mailMessageChan
+				handleMailMessage(emailMessage)
+			}
+			return
+		}
+	}
+}
+
+func handleMailMessage(emailMessage mail.MailMessage) (err error) {
+	return
+}
+
+// 	defer func() {
+// 		if err != nil {
+// 			utils.FileLog.Error("邮件处理失败,邮件标题:%s,错误原因:%v", emailMessage.Title, err)
+// 		}
+
+// 		for _, v := range emailMessage.Resources {
+// 			os.Remove(v)
+// 		}
+// 	}()
+// 	outsideReportAttachmentList := make([]*report.OutsideReportAttachment, 0)
+
+// 	emailMessageUID := int(emailMessage.Uid)
+// 	outReport, err := report.GetOutsideReportByEmailMessageId(emailMessageUID)
+// 	// 已经存在了,那么就返回
+// 	if err == nil {
+// 		utils.FileLog.Debug("已存在,就不处理了,报告标题:%s;;邮件下标:%d", outReport.Title, emailMessage.Uid)
+// 		return
+// 	}
+// 	fmt.Println("开始处理邮件,标题:", emailMessage.Title, ";邮件下标:", emailMessage.Uid)
+// 	// sql报错,那么就返回
+// 	if err != nil && err.Error() != utils.ErrNoRow() {
+// 		return
+// 	}
+// 	// 已经存在了,那么就返回
+// 	if outReport.Id > 0 {
+// 		return
+// 	}
+
+// 	title := emailMessage.Title
+// 	rule, err := MatchTitleRule(emailMessage.Title, emailMessage.From)
+// 	if err != nil {
+// 		return
+// 	}
+// 	if rule == nil {
+// 		return
+// 	}
+// 	// 处理附件
+
+// 	err = report.CreateReportLog(reportInfo, outsideReportAttachmentList)
+
+// 	return
+// }
+
+func ensureDirExists(dirPath string) error {
+	info, err := os.Stat(dirPath)
+	if err == nil {
+		if info.IsDir() {
+			return nil // 目录已存在
+		}
+		return fmt.Errorf("path '%s' exists but is not a directory", dirPath)
+	}
+	if os.IsNotExist(err) {
+		return os.MkdirAll(dirPath, fs.ModePerm)
+	}
+	return err
+}

+ 67 - 0
services/mtjh_watch.go

@@ -0,0 +1,67 @@
+package services
+
+import (
+	"context"
+	"eta/eta_crawler/utils"
+	"fmt"
+	"io/fs"
+	"os"
+	"path/filepath"
+	"strings"
+	"syscall"
+	"time"
+
+	"github.com/patrickmn/go-cache"
+)
+
+func MtjhWatchTask(cont context.Context) (err error) {
+	mtjhWatch()
+	return
+}
+
+func mtjhWatch() {
+	fmt.Println("mtjhWatch start")
+	var err error
+	defer func() {
+		if err != nil {
+			fmt.Println("mtjhWatch Err:" + err.Error())
+			utils.FileLog.Info("mtjhWatch Err:" + err.Error())
+		}
+	}()
+	var cacheClient *cache.Cache
+	if cacheClient == nil {
+		cacheClient = cache.New(365*24*time.Hour, 365*24*time.Hour)
+	}
+	err = filepath.Walk(utils.MtjhFilePath, func(path string, info fs.FileInfo, err error) error {
+		if err != nil {
+			return err
+		}
+		if !info.IsDir() {
+			fileInfo, err := os.Stat(path)
+			if err != nil {
+				fmt.Println("os.Stat:", err.Error())
+			}
+			winFileAttr := fileInfo.Sys().(*syscall.Win32FileAttributeData)
+			modifyTimeStr := utils.SecondToTime(winFileAttr.LastWriteTime.Nanoseconds() / 1e9).Format(utils.FormatDateTime)
+
+			existModifyTime, ok := cacheClient.Get(path)
+			if ok {
+				existModifyTimeStr := existModifyTime.(string)
+				if existModifyTimeStr != modifyTimeStr {
+					if strings.Contains(path, "煤炭江湖") {
+						err = Mtjh(path)
+						utils.FileLog.Warning("Mtjh:" + err.Error())
+					}
+				}
+			} else {
+				if strings.Contains(path, "煤炭江湖") {
+					err = Mtjh(path)
+					utils.FileLog.Warning("Mtjh:" + err.Error())
+				}
+			}
+			cacheClient.Delete(path)
+			cacheClient.Set(path, modifyTimeStr, 24*time.Hour)
+		}
+		return nil
+	})
+}

+ 9 - 0
services/task.go

@@ -2,6 +2,7 @@ package services
 
 import (
 	"context"
+	"eta/eta_crawler/services/email"
 	"eta/eta_crawler/services/liangyou"
 	"eta/eta_crawler/services/sci99"
 	"eta/eta_crawler/utils"
@@ -84,6 +85,14 @@ func Task() {
 	//task.AddTask("统计局数据爬取-季度", refreshNationalQuarter) // 每月15号1:25执行
 	//task.AddTask("统计局数据爬取-年度A", refreshNationalYearA)  // 每月20日1:45执行
 	//task.AddTask("统计局数据爬取-年度B", refreshNationalYearB)  // 每月25日1:45执行
+	// if utils.MtjhOpen == "1" {
+	// 	mtjh := task.NewTask("refreshMtjh", "0 */2 * * * *", MtjhWatchTask)
+	// 	task.AddTask("启动煤炭江湖监听excel脚本", mtjh)
+	// }
+	if utils.MtjhMailAttachmentOpen == "1" {
+		coalMailTask := task.NewTask("MailAttachment", utils.MtjhMailAttachmentTime, email.ListenMail)
+		task.AddTask("启动获取邮件附件脚本", coalMailTask)
+	}
 
 	task.StartTask()
 	//FileCoalJsm()

+ 9 - 2
utils/common.go

@@ -10,8 +10,6 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
-	"github.com/mozillazg/go-pinyin"
-	"github.com/shopspring/decimal"
 	"image"
 	"image/png"
 	"io"
@@ -30,6 +28,10 @@ import (
 	"strings"
 	"time"
 	"unicode"
+
+	"github.com/shopspring/decimal"
+
+	"github.com/mozillazg/go-pinyin"
 )
 
 // 随机数种子
@@ -351,6 +353,11 @@ func SaveBase64ToFileBySeek(content, path string) (err error) {
 	return nil
 }
 
+// 把秒级的时间戳转为time格式
+func SecondToTime(sec int64) time.Time {
+	return time.Unix(sec, 0)
+}
+
 func PathExists(path string) (bool, error) {
 	_, err := os.Stat(path)
 	if err == nil {

+ 28 - 0
utils/config.go

@@ -52,6 +52,20 @@ var (
 	OLD_EXCEL_PATH_JR string
 )
 
+// 煤炭江湖
+var (
+	MtjhFilePath           string // excel文件地址
+	MtjhOpen               string // 是否配置煤炭江湖数据源,1已配置
+	MtjhMailAttachmentOpen string // 获取邮件附件功能,1已配置
+	MtjhMailAttachmentTime string // 获取邮件附件功能时间
+	MtjhEmailAddress       string // 煤炭江湖监听邮箱服务器地址
+	MtjhEmailUseName       string // 煤炭江湖监听邮箱用户名
+	MtjhEmailPassword      string // 煤炭江湖监听邮箱密码
+	MtjhEmailFolder        string // 煤炭江湖监听邮箱文件夹
+	MtjhEmailReadBatch     string // 煤炭江湖监听邮箱读取批次
+	MtjhEmailStarIndex     string // 煤炭江湖监听邮箱索引
+)
+
 func init() {
 	tmpRunMode, err := web.AppConfig.String("run_mode")
 	if err != nil {
@@ -133,4 +147,18 @@ func init() {
 		LY_OPEN = config["ly_open"]
 
 	}
+	//煤炭江湖文件夹配置
+	{
+		MtjhFilePath = config["mtjh_file_path"]
+		MtjhOpen = config["mtjh_open"]
+		MtjhMailAttachmentOpen = config["mtjh_mail_attachment_open"]
+		MtjhMailAttachmentTime = config["mtjh_mail_attachment_time"]
+		MtjhEmailAddress = config["mtjh_email_address"]
+		MtjhEmailUseName = config["mtjh_email_use_name"]
+		MtjhEmailPassword = config["mtjh_email_password"]
+		MtjhEmailFolder = config["mtjh_email_folder"]
+		MtjhEmailReadBatch = config["mtjh_email_read_batch"]
+		MtjhEmailStarIndex = config["mtjh_email_star_index"]
+	}
+
 }

+ 12 - 0
utils/constants.go

@@ -31,3 +31,15 @@ const (
 	BusinessCodeFuBang  = "E2024020200"
 	BusinessCodeJinRui  = "E2023122901"
 )
+
+// eta_index_lib 的接口名称
+const (
+	LIB_ROUTE_COAL_MINE_MTJH            = "/mtjh/data"                 //煤炭江湖数据处理excel数据并入库 数据地址
+	LIB_ROUTE_COAL_MINE_JSM_HISTORY     = "/coal_mine/jsm/history"     //jsm三省煤炭网历史数据处理excel数据并入库 数据地址
+	LIB_ROUTE_COAL_MINE_COASTAL_HISTORY = "/coal_mine/coastal/history" //沿海煤炭网历史数据处理excel数据并入库 数据地址
+	LIB_ROUTE_COAL_MINE_INLAND_HISTORY  = "/coal_mine/inland/history"  //内陆三省煤炭网历史数据处理excel数据并入库 数据地址
+	LIB_ROUTE_COAL_MINE_JSM             = "/coal_mine/jsm"             //jsm三省煤炭网历史数据处理excel数据并入库 数据地址
+	LIB_ROUTE_COAL_MINE_COASTAL         = "/coal_mine/coastal"         //沿海煤炭网历史数据处理excel数据并入库 数据地址
+	LIB_ROUTE_COAL_MINE_INLAND          = "/coal_mine/inland"          //内陆三省煤炭网历史数据处理excel数据并入库 数据地址
+	LIB_ROUTE_COAL_MINE_FIRM            = "/coal_mine/firm"            //分公司旬度煤炭网数据处理excel数据并入库 数据地址
+)

+ 94 - 0
utils/mail/charset_reader.go

@@ -0,0 +1,94 @@
+package mail
+
+import (
+	"fmt"
+	"golang.org/x/text/encoding/charmap"
+	"golang.org/x/text/encoding/japanese"
+	"golang.org/x/text/encoding/korean"
+	"golang.org/x/text/encoding/simplifiedchinese"
+	"golang.org/x/text/transform"
+	"io"
+	"strings"
+)
+
+var charsetMap = map[string]transform.Transformer{
+	"gb2312":         simplifiedchinese.GBK.NewDecoder(),
+	"gbk":            simplifiedchinese.GBK.NewDecoder(),
+	"ibm037":         charmap.CodePage037.NewDecoder(),
+	"ibm437":         charmap.CodePage437.NewDecoder(),
+	"ibm850":         charmap.CodePage850.NewDecoder(),
+	"ibm852":         charmap.CodePage852.NewDecoder(),
+	"ibm855":         charmap.CodePage855.NewDecoder(),
+	"ibm858":         charmap.CodePage858.NewDecoder(),
+	"ibm860":         charmap.CodePage860.NewDecoder(),
+	"ibm862":         charmap.CodePage862.NewDecoder(),
+	"ibm863":         charmap.CodePage863.NewDecoder(),
+	"ibm865":         charmap.CodePage865.NewDecoder(),
+	"ibm866":         charmap.CodePage866.NewDecoder(),
+	"ibm1047":        charmap.CodePage1047.NewDecoder(),
+	"ibm1140":        charmap.CodePage1140.NewDecoder(),
+	"iso-8859-1":     charmap.ISO8859_1.NewDecoder(),
+	"iso-8859-2":     charmap.ISO8859_2.NewDecoder(),
+	"iso-8859-3":     charmap.ISO8859_3.NewDecoder(),
+	"iso-8859-4":     charmap.ISO8859_4.NewDecoder(),
+	"iso-8859-5":     charmap.ISO8859_5.NewDecoder(),
+	"iso-8859-6":     charmap.ISO8859_6.NewDecoder(),
+	"iso-8859-7":     charmap.ISO8859_7.NewDecoder(),
+	"iso-8859-8":     charmap.ISO8859_8.NewDecoder(),
+	"iso-8859-9":     charmap.ISO8859_9.NewDecoder(),
+	"iso-8859-10":    charmap.ISO8859_10.NewDecoder(),
+	"iso-8859-13":    charmap.ISO8859_13.NewDecoder(),
+	"iso-8859-14":    charmap.ISO8859_14.NewDecoder(),
+	"iso-8859-15":    charmap.ISO8859_15.NewDecoder(),
+	"iso-8859-16":    charmap.ISO8859_16.NewDecoder(),
+	"koi8-r":         charmap.KOI8R.NewDecoder(),
+	"koi8-u":         charmap.KOI8U.NewDecoder(),
+	"macintosh":      charmap.Macintosh.NewDecoder(),
+	"x-mac-cyrillic": charmap.MacintoshCyrillic.NewDecoder(),
+	"windows-874":    charmap.Windows874.NewDecoder(),
+	"windows-1250":   charmap.Windows1250.NewDecoder(),
+	"windows-1251":   charmap.Windows1251.NewDecoder(),
+	"windows-1252":   charmap.Windows1252.NewDecoder(),
+	"windows-1253":   charmap.Windows1253.NewDecoder(),
+	"windows-1254":   charmap.Windows1254.NewDecoder(),
+	"windows-1255":   charmap.Windows1255.NewDecoder(),
+	"windows-1257":   charmap.Windows1257.NewDecoder(),
+	"windows-1258":   charmap.Windows1258.NewDecoder(),
+	"x-user-defined": charmap.XUserDefined.NewDecoder(),
+	"euc-jp":         japanese.EUCJP.NewDecoder(),
+	"iso-2022-jp":    japanese.ISO2022JP.NewDecoder(),
+	"shift_jis":      japanese.ShiftJIS.NewDecoder(),
+	"ks_c_5601-1987": korean.EUCKR.NewDecoder(),
+	"euc-kr":         korean.EUCKR.NewDecoder(),
+}
+
+// 定义一个自定义的 CharsetReader 函数,它能够处理 gb2312 和 gbk 字符集
+func myCharsetReader(charset string, input io.Reader) (io.Reader, error) {
+	charset = strings.ToLower(charset)
+	newDecoder, ok := charsetMap[charset]
+	if ok {
+		reader := transform.NewReader(input, newDecoder)
+		return reader, nil
+	}
+	if charset == `utf-8` {
+		return input, nil
+	}
+
+	switch strings.ToLower(charset) {
+	case "gb2312", "gbk":
+		reader := transform.NewReader(input, simplifiedchinese.GBK.NewDecoder())
+		return reader, nil
+	case "utf-8":
+		return input, nil
+	case "iso-8859-1":
+		reader := transform.NewReader(input, charmap.ISO8859_1.NewDecoder())
+
+		return reader, nil
+	case "windows-1252":
+		reader := transform.NewReader(input, charmap.Windows1252.NewDecoder())
+		return reader, nil
+	default:
+	}
+	return input, fmt.Errorf("unsupported charset: %s", charset)
+
+}

+ 369 - 0
utils/mail/imap.go

@@ -0,0 +1,369 @@
+package mail
+
+import (
+	"errors"
+	"eta/eta_crawler/utils"
+	"fmt"
+	"io"
+	"log"
+	"os"
+	"path"
+	"regexp"
+	"strings"
+	"time"
+
+	"github.com/emersion/go-imap"
+	"github.com/emersion/go-imap/client"
+	"github.com/emersion/go-message"
+	"github.com/emersion/go-message/mail"
+	"github.com/h2non/filetype"
+)
+
+type MailMessage struct {
+	Date        time.Time         `description:"收件时间"`
+	Uid         uint32            `description:"该邮件在邮箱中的唯一id"`
+	FromAddress string            `description:"发件人邮箱"`
+	From        string            `description:"发件人名称"`
+	Title       string            `description:"邮件标题"`
+	Content     string            `description:"邮件主体正文"`
+	Resources   map[string]string `description:"正文内嵌资源"`
+	Attachment  map[string][]byte `description:"附件资源"`
+}
+
+func ListenMail(mailAddress, folder, userName, password string, readBatchSize, fromEmailIndex int, mailMessageChan chan MailMessage, mailMessageDoneChan chan bool) (err error) { // 收件箱
+	defer func() {
+		// 处理结束
+		mailMessageDoneChan <- true
+		if err != nil {
+			fmt.Println("err:", err.Error())
+		}
+	}()
+	// 建立与 IMAP 服务器的连接
+	c, err := client.DialTLS(mailAddress, nil)
+	if err != nil {
+		fmt.Printf("连接 IMAP 服务器失败: %+v \n", err)
+		return
+	}
+
+	// 最后一定不要忘记退出登录
+	defer func() {
+		_ = c.Logout()
+	}()
+
+	// 登录
+	if err = c.Login(userName, password); err != nil {
+		fmt.Printf("邮箱[%s] 登录失败: %v \n", fmt.Sprintf("%s:%s", userName, mailAddress), err)
+		return
+	}
+	// 列出当前邮箱中的文件夹
+	mailboxes := make(chan *imap.MailboxInfo, 10)
+	done := make(chan error, 1) // 记录错误的 chan
+	go func() {
+		done <- c.List("", "*", mailboxes)
+	}()
+	log.Println("-->当前邮箱的文件夹 Mailboxes:")
+
+	var folderExists bool
+	for m := range mailboxes {
+		log.Println("* ", m.Name)
+		if m.Name == folder {
+			folderExists = true
+		}
+	}
+
+	err = <-done
+	if err != nil {
+		utils.FileLog.Error("列出邮箱列表时,出现错误:%v \n", err)
+		return
+	}
+
+	log.Println("-->列出邮箱列表完毕!")
+	if !folderExists {
+		err = fmt.Errorf(fmt.Sprintf("文件夹[%s] 不存在 \n", folder))
+		return
+	}
+
+	message.CharsetReader = myCharsetReader
+
+	// 选择指定的文件夹
+	mbox, err := c.Select(folder, false)
+	if err != nil {
+		err = fmt.Errorf(fmt.Sprintf("选择邮件箱失败: %+v", err))
+		return
+	}
+	log.Printf("当前文件夹[%s]中,总共有 %d 封邮件 \n", folder, mbox.Messages)
+	if mbox.Messages == 0 {
+		return
+	}
+
+	// 创建一个序列集,用于批量读取邮件
+	seqSet := new(imap.SeqSet)
+	to := mbox.Messages // 此文件下的邮件总数
+
+	var isStopFor bool
+	step := uint32(1)
+	for i := to; i >= 1; {
+		start := i - step + 1
+		if start < 0 {
+			start = 1
+		}
+
+		seqSet.Clear()
+		seqSet.AddRange(start, i) // 添加指定范围内的邮件编号
+
+		// 获取整个消息正文
+		// imap.FetchEnvelope:请求获取邮件的信封数据(例如发件人、收件人、主题等元数据)。
+		// imap.FetchRFC822:请求获取完整的邮件内容,包括所有头部和正文。
+		items := []imap.FetchItem{imap.FetchFlags, imap.FetchEnvelope, imap.FetchRFC822, imap.FetchBodyStructure}
+
+		// 获取邮件内容 Start
+		messages := make(chan *imap.Message, readBatchSize) // 创建一个通道,用于接收邮件消息
+		fetchDone := make(chan error, 1)                    // 创建一个通道,用于接收错误消息
+		go func() {
+			// Fetch方法用于从服务器获取邮件数据,这里请求了邮件的信封和完整内容
+			fetchDone <- c.Fetch(seqSet, items, messages)
+		}()
+
+		err = <-fetchDone
+		if err != nil {
+			utils.FileLog.Error("获取邮件信息出现错误:%v \n", err)
+			return
+		}
+		// 获取邮件内容 End
+		for msg := range messages {
+			// 如果需要终止,那么就不处理了
+			if isStopFor {
+				continue
+			}
+
+			emailMessage, isRead, tmpErr := readEveryMsg(msg)
+			if tmpErr != nil {
+				// 移除本地文件
+				{
+					for _, v := range emailMessage.Resources {
+						os.Remove(v)
+					}
+				}
+				utils.FileLog.Error("读取邮件内容时出现错误:%v \n", tmpErr)
+				continue
+			}
+			// 如果没有取到,那么就过滤
+			if !isRead {
+				continue
+			}
+
+			// 判断当前邮件id是否小于等于已经监听到的最小id,如果是,那么就不处理了
+			if emailMessage.Uid <= uint32(fromEmailIndex) {
+				isStopFor = true
+				continue
+			}
+			// 如果取到了,那么写入待处理chan
+			// 写入邮件处理chan
+			mailMessageChan <- emailMessage
+
+		}
+
+		if isStopFor {
+			// 已经找到了最小的邮件id,那么就退出循环了
+		}
+		i = i - step
+	}
+
+	log.Println("读取了所有邮件,完毕!")
+	return
+}
+
+// document link: https://github.com/emersion/go-imap/wiki/Fetching-messages
+func readEveryMsg(msg *imap.Message) (emailMessage MailMessage, ok bool, err error) {
+	ok = true
+	defer func() {
+		if err != nil {
+			ok = false
+			utils.FileLog.Error("邮件读取失败;Err:%s", err.Error())
+		}
+	}()
+	message.CharsetReader = myCharsetReader
+	emailMessage.Resources = make(map[string]string)  // 内嵌资源
+	emailMessage.Attachment = make(map[string][]byte) // 附件
+
+	emailMessage.Uid = msg.Uid
+	htmlStr := ``
+	textStr := ``
+
+	// 获取邮件正文
+	r := msg.GetBody(&imap.BodySectionName{})
+	if r == nil {
+		utils.FileLog.Info("服务器没有返回消息内容")
+	}
+
+	mr, err := mail.CreateReader(r)
+	if err != nil {
+		err = errors.New(fmt.Sprintf("邮件读取时出现错误:%v \n", err))
+		return
+	}
+
+	// 收件时间
+	{
+		date, err := mr.Header.Date()
+		if err != nil {
+			log.Println("收件时间 异常:", err.Error())
+		}
+		emailMessage.Date = date
+	}
+
+	// 发件人
+	{
+		fromStr := mr.Header.Get("From")
+		// 处理无效地址的情况
+		if !strings.Contains(fromStr, "@") {
+			emailMessage.FromAddress = fromStr
+			emailMessage.From = fromStr
+		} else {
+			from, tmpErr := mr.Header.AddressList("From")
+			if tmpErr != nil {
+				log.Println("发件人 异常:", err.Error())
+			}
+			if len(from) > 0 {
+				emailMessage.FromAddress = from[0].Address
+				emailMessage.From = from[0].Name
+			}
+		}
+	}
+
+	// 邮件标题
+	subject, err := mr.Header.Subject()
+	if err != nil {
+		log.Println("邮件主题 Subject ERR:", err)
+	} else {
+		//log.Println("邮件主题 Subject:", subject)
+	}
+	emailMessage.Title = subject
+
+	// 过滤
+
+	for {
+		p, tmpErr := mr.NextPart()
+		if tmpErr == io.EOF {
+			break
+		} else if tmpErr != nil {
+			utils.FileLog.Error("读取邮件内容时出现错误:%v \n", tmpErr)
+			err = tmpErr
+			return
+		}
+
+		bodyBytes, _ := io.ReadAll(p.Body)
+		if err != nil {
+			//log.Fatalf("读取邮件部分时出现错误:%v \n", err)
+			err = errors.New(fmt.Sprintf("读取邮件部分时出现错误:%v \n", err))
+			return
+		}
+
+		switch h := p.Header.(type) {
+		case *mail.InlineHeader:
+			// 这是消息的文本(可以是纯文本或 HTML)
+			contentType := h.Get("Content-Type")
+			//log.Println("消息内容content-type:", contentType)
+			if strings.HasPrefix(contentType, "text/plain") {
+				//log.Printf("得到正文 -> TEXT: %v \n", string(bodyBytes))
+				textStr += string(bodyBytes)
+			} else if strings.HasPrefix(contentType, "text/html") {
+				//log.Printf("得到正文 -> HTML: %v \n", len(b))
+				//log.Printf("得到正文 -> HTML: %v \n", string(bodyBytes))
+				htmlStr += string(bodyBytes)
+			}
+			// 这是内嵌资源
+			if cid := p.Header.Get("Content-ID"); cid != "" {
+
+				// 确定文件后缀
+				fileSuffix := determineFileSuffix(bodyBytes)
+				fileName := fmt.Sprintf("%s%s.%s", utils.MtjhFilePath, cid[1:len(cid)-1], fileSuffix)
+
+				err = SaveToFile(bodyBytes, fileName)
+				if err != nil {
+					err = errors.New(fmt.Sprintf("保存文件时出现错误:%v \n", err))
+					return
+				}
+				emailMessage.Resources[cid] = fileName
+			}
+		case *mail.AttachmentHeader:
+			// 这是一个附件
+			filename, _ := h.Filename()
+			//log.Printf("得到附件: %v,content-type:%s \n", filename, p.Header.Get("Content-Type"))
+			saveName := fmt.Sprint(msg.SeqNum, utils.MD5(filename), time.Now().Format(utils.FormatDateTimeUnSpace), time.Now().Nanosecond(), path.Ext(filename))
+			filePath := fmt.Sprintf("%s%s%s%s", utils.MtjhFilePath, `file`, string(os.PathSeparator), saveName)
+			err = SaveToFile(bodyBytes, filePath)
+			if err != nil {
+				err = errors.New(fmt.Sprintf("保存文件时出现错误:%v \n", err))
+				return
+			}
+
+			// 这是附件资源
+			if contentDisposition := p.Header.Get("Content-Disposition"); contentDisposition != "" {
+				if strings.HasPrefix(contentDisposition, "attachment") {
+					emailMessage.Attachment[filename] = bodyBytes
+				}
+			} else if cid := p.Header.Get("Content-ID"); cid != "" {
+				// 这是内嵌资源
+				emailMessage.Resources[cid] = filePath
+			}
+			//else {
+			//	mailMessage.Attachment[filename] = filePath
+			//}
+		default:
+			utils.FileLog.Info("未知格式:", h)
+			//log.Println(h)
+		}
+	}
+	emailMessage.Content = htmlStr
+	if emailMessage.Content == `` {
+		emailMessage.Content = textStr
+	}
+
+	//log.Println("一封邮件读取完毕")
+	//log.Printf("------------------------- \n\n")
+
+	return
+}
+
+// 根据文件内容确定文件后缀
+func determineFileSuffix(content []byte) string {
+	kind, err := filetype.Match(content)
+	if err != nil {
+		utils.FileLog.Error("无法确定文件类型:%v \n", err)
+		return ".bin"
+	}
+	return kind.Extension
+}
+
+func SaveToFile(content []byte, fileName string) error {
+	file, err := os.Create(fileName)
+	if err != nil {
+		return err
+	}
+	defer func() {
+		_ = file.Close()
+	}()
+
+	_, err = file.Write(content)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+// ContainsWholeWord 检查字符串 s 中是否包含完整的单词 word。
+// 该函数使用正则表达式来匹配整个单词,确保不会错误地匹配到单词的一部分。
+// 参数:
+//
+//	s: 要搜索的字符串
+//	word: 要查找的完整单词
+//
+// 返回值:
+//
+//	如果 s 中包含完整的单词 word,则返回 true;否则返回 false。
+func ContainsWholeWord(s string, word string) bool {
+	pattern := fmt.Sprintf(`\b%s\b`, regexp.QuoteMeta(word))
+	re := regexp.MustCompile(pattern)
+	return re.MatchString(s)
+}