优化邮件监听功能和报告获取逻辑

- 在邮件监听前获取已入库的最大邮件ID，避免重复处理
- 增加发件人邮箱过滤功能，支持大小写敏感和不敏感的过滤
- 优化邮件标题过滤逻辑，支持大小写敏感和不敏感的过滤
- 重构部分代码结构，提高代码可读性和维护性

config/config.go

@@ -91,16 +91,18 @@ type Xiangyu struct {
 }
 
 type Email struct {
-	Host                          string   `mapstructure:"host" json:"host" yaml:"host" description:"邮件服务器"`
-	Port                          int      `mapstructure:"port" json:"port" yaml:"port" description:"邮件服务器端口"`
-	UserName                      string   `mapstructure:"username" json:"username" yaml:"username" description:"邮件服务器用户名"`
-	Password                      string   `mapstructure:"password" json:"password" yaml:"password" description:"邮件服务器密码"`
-	Folder                        string   `mapstructure:"folder" json:"folder" yaml:"folder" description:"邮件服务器文件夹"`
-	ReadBatchSize                 int      `mapstructure:"read-batch-size" json:"read-batch-size" yaml:"read-batch-size" description:"邮件读取批次"`
-	IgnoreEmail                   []string `mapstructure:"ignore-email" json:"ignore-email" yaml:"ignore-email" description:"邮件过滤，不读取这些邮箱"`
-	IgnoreEmailCaseSensitive      []string `mapstructure:"ignore-email-case-sensitive" json:"ignore-email-case-sensitive" yaml:"ignore-email-case-sensitive" description:"邮件过滤，不读取这些邮箱，大小写敏感"`
-	IgnoreEmailTitle              []string `mapstructure:"ignore-email-title" json:"ignore-email-title" yaml:"ignore-email-title" description:"邮件过滤，不读取这些标题"`
-	IgnoreEmailTitleCaseSensitive []string `mapstructure:"ignore-email-title-case-sensitive" json:"ignore-email-title-case-sensitive" yaml:"ignore-email-title-case-sensitive" description:"邮件过滤，不读取这些标题，大小写敏感"`
+	Host                            string   `mapstructure:"host" json:"host" yaml:"host" description:"邮件服务器"`
+	Port                            int      `mapstructure:"port" json:"port" yaml:"port" description:"邮件服务器端口"`
+	UserName                        string   `mapstructure:"username" json:"username" yaml:"username" description:"邮件服务器用户名"`
+	Password                        string   `mapstructure:"password" json:"password" yaml:"password" description:"邮件服务器密码"`
+	Folder                          string   `mapstructure:"folder" json:"folder" yaml:"folder" description:"邮件服务器文件夹"`
+	ReadBatchSize                   int      `mapstructure:"read-batch-size" json:"read-batch-size" yaml:"read-batch-size" description:"邮件读取批次"`
+	IgnoreEmail                     []string `mapstructure:"ignore-email" json:"ignore-email" yaml:"ignore-email" description:"邮件过滤，不读取这些邮箱"`
+	IgnoreEmailCaseSensitive        []string `mapstructure:"ignore-email-case-sensitive" json:"ignore-email-case-sensitive" yaml:"ignore-email-case-sensitive" description:"邮件过滤，不读取这些邮箱，大小写敏感"`
+	IgnoreEmailAddress              []string `mapstructure:"ignore-email-address" json:"ignore-email-address" yaml:"ignore-email-address" description:"邮件过滤，不读取这些邮箱，大小写敏感"`
+	IgnoreEmailAddressCaseSensitive []string `mapstructure:"ignore-email-address-case-sensitive" json:"ignore-email-address-case-sensitive" yaml:"ignore-email-address-case-sensitive" description:"邮件过滤，不读取这些邮箱，大小写敏感"`
+	IgnoreEmailTitle                []string `mapstructure:"ignore-email-title" json:"ignore-email-title" yaml:"ignore-email-title" description:"邮件过滤，不读取这些标题"`
+	IgnoreEmailTitleCaseSensitive   []string `mapstructure:"ignore-email-title-case-sensitive" json:"ignore-email-title-case-sensitive" yaml:"ignore-email-title-case-sensitive" description:"邮件过滤，不读取这些标题，大小写敏感"`
 }
 
 type Oss struct {

models/report/outside_report.go

@@ -116,3 +116,12 @@ func CreateOutsideReport(item *OutsideReport, outsideReportAttachmentList []*Out
 	return
 
 }
+
+// GetMaxOutsideReportByEmailMessageId
+// @Description: 获取最大的邮件id
+// @return maxEmailMessageUid
+// @return err
+func GetMaxOutsideReportByEmailMessageId() (maxEmailMessageUid int, err error) {
+	err = global.DEFAULT_MYSQL.Model(OutsideReport{}).Select("max(outside_report_id)").Scan(&maxEmailMessageUid).Error
+	return
+}

services/pcsg/mail.go

@@ -35,8 +35,15 @@ func ListenMail() (err error) {
 	go afterByListen(mailMessageChan, mailMessageDoneChan)
 
 	fmt.Println("开始监听邮件")
+
+	emailMessageUID, err := report.GetMaxOutsideReportByEmailMessageId()
+	// 已经存在了，那么就返回
+	if err != nil {
+		global.FILE_LOG.Errorf("获取已入库的最大邮件id失败：%s", err.Error())
+	}
+
 	mailAddress := fmt.Sprintf("%s:%d", global.CONFIG.Email.Host, global.CONFIG.Email.Port)
-	mail.ListenMail(mailAddress, global.CONFIG.Email.Folder, global.CONFIG.Email.UserName, global.CONFIG.Email.Password, global.CONFIG.Email.ReadBatchSize, mailMessageChan, mailMessageDoneChan)
+	mail.ListenMail(mailAddress, global.CONFIG.Email.Folder, global.CONFIG.Email.UserName, global.CONFIG.Email.Password, global.CONFIG.Email.ReadBatchSize, emailMessageUID, mailMessageChan, mailMessageDoneChan)
 
 	return
 }
@@ -48,18 +55,18 @@ func afterByListen(mailMessageChan chan mail.MailMessage, mailMessageDoneChan ch
 	}()
 	for {
 		select {
-		case mailMessage := <-mailMessageChan:
+		case emailMessage := <-mailMessageChan:
 			//fmt.Println("读取成功")
-			//fmt.Println(mailMessage.Title)
-			handleMailMessage(mailMessage)
+			//fmt.Println(emailMessage.Title)
+			handleMailMessage(emailMessage)
 		case <-time.After(10 * time.Second):
 			//fmt.Println("监听超时了")
 			break
 		case <-mailMessageDoneChan:
 			fmt.Println("读取完成一轮了")
 			for len(mailMessageChan) > 0 {
-				mailMessage := <-mailMessageChan
-				handleMailMessage(mailMessage)
+				emailMessage := <-mailMessageChan
+				handleMailMessage(emailMessage)
 			}
 			fmt.Println("结束了")
 			return
@@ -67,32 +74,33 @@ func afterByListen(mailMessageChan chan mail.MailMessage, mailMessageDoneChan ch
 	}
 }
 
-func handleMailMessage(mailMessage mail.MailMessage) (err error) {
+func handleMailMessage(emailMessage mail.MailMessage) (err error) {
 	defer func() {
 		if err != nil {
-			global.FILE_LOG.Errorf("邮件处理失败，邮件标题：%s，错误原因：%v", mailMessage.Title, err)
+			global.FILE_LOG.Errorf("邮件处理失败，邮件标题：%s，错误原因：%v", emailMessage.Title, err)
 		}
 
-		for _, v := range mailMessage.Resources {
+		for _, v := range emailMessage.Resources {
 			os.Remove(v)
 		}
 
-		for _, v := range mailMessage.Attachment {
+		for _, v := range emailMessage.Attachment {
 			os.Remove(v)
 		}
 	}()
 	//rootPath := `C:\Users\123\go\src\eta\eta_email_analysis\static\`
 	ossClient := oss.NewOssClient()
-	//fmt.Println(mailMessage.Title)
+	//fmt.Println(emailMessage.Title)
 	outsideReportAttachmentList := make([]*report.OutsideReportAttachment, 0)
 
-	emailMessageUID := int(mailMessage.Uid)
+	emailMessageUID := int(emailMessage.Uid)
 	outReport, err := report.GetOutsideReportByEmailMessageId(emailMessageUID)
 	// 已经存在了，那么就返回
 	if err == nil {
-		fmt.Println("已存在，就不处理了,报告标题：", outReport.Title)
+		global.FILE_LOG.Debugf("已存在，就不处理了,报告标题：%s", outReport.Title)
 		return
 	}
+	fmt.Println("开始处理邮件，标题：", emailMessage.Title, "；邮件下标：", emailMessage.Uid)
 	// sql报错，那么就返回
 	if err != nil && !utils.IsErrNoRow(err) {
 		return
@@ -102,7 +110,7 @@ func handleMailMessage(mailMessage mail.MailMessage) (err error) {
 		return
 	}
 
-	for k, v := range mailMessage.Resources {
+	for k, v := range emailMessage.Resources {
 		randStr := utils.GetRandStringNoSpecialChar(28)
 		ext := path.Ext(v)
 		fileName := randStr + ext
@@ -110,16 +118,16 @@ func handleMailMessage(mailMessage mail.MailMessage) (err error) {
 		resourceUrl, tmpErr := ossClient.UploadFile(fileName, v, "")
 		//os.Remove(v)
 		if tmpErr != nil {
-			fmt.Println("上传文件失败：", tmpErr)
+			global.FILE_LOG.Error(emailMessage.Title, "- 文件上传文件失败：", tmpErr)
 			continue
 		}
 		tmpK := strings.Replace(k, "<", "cid:", -1)
 		tmpK = strings.Replace(tmpK, ">", "", -1)
 
-		mailMessage.Content = strings.Replace(mailMessage.Content, fmt.Sprint(tmpK, `"`), fmt.Sprint(resourceUrl, `"`), -1)
+		emailMessage.Content = strings.Replace(emailMessage.Content, fmt.Sprint(tmpK, `"`), fmt.Sprint(resourceUrl, `"`), -1)
 	}
 
-	for name, v := range mailMessage.Attachment {
+	for name, v := range emailMessage.Attachment {
 		var fileSize int64
 		fileInfo, tmpErr := os.Stat(v)
 		if tmpErr != nil {
@@ -134,7 +142,7 @@ func handleMailMessage(mailMessage mail.MailMessage) (err error) {
 		resourceUrl, tmpErr := ossClient.UploadFile(fileName, v, "")
 		//defer os.Remove(v)
 		if tmpErr != nil {
-			fmt.Println("上传文件失败：", tmpErr)
+			global.FILE_LOG.Error(emailMessage.Title, " - 文件上传文件失败：", tmpErr)
 			continue
 		}
 		outsideReportAttachmentList = append(outsideReportAttachmentList, &report.OutsideReportAttachment{
@@ -147,19 +155,19 @@ func handleMailMessage(mailMessage mail.MailMessage) (err error) {
 		})
 	}
 
-	//fileName := fmt.Sprintf("%s%s.%s", rootPath, mailMessage.Title, "html")
+	//fileName := fmt.Sprintf("%s%s.%s", rootPath, emailMessage.Title, "html")
 
-	htmlEscapeBody := html.EscapeString(mailMessage.Content)
+	htmlEscapeBody := html.EscapeString(emailMessage.Content)
 	//fmt.Println(htmlEscapeBody)
-	//err = utils.SaveToFile([]byte(mailMessage.Content), fileName)
+	//err = utils.SaveToFile([]byte(emailMessage.Content), fileName)
 	//if err != nil {
 	//	fmt.Println(fileName, "生成失败；err:", err)
 	//}
 
 	var sysUserId int
-	sysUserName := mailMessage.FromEmail
+	sysUserName := emailMessage.FromEmail
 	// 查找用户
-	if mailMessage.From != `` {
+	if emailMessage.From != `` {
 		tmpEmailStrList := strings.Split(sysUserName, "<")
 		if len(tmpEmailStrList) >= 2 {
 			sysUserName = tmpEmailStrList[1]
@@ -179,14 +187,14 @@ func handleMailMessage(mailMessage mail.MailMessage) (err error) {
 		}
 	}
 
-	reportUpdateTime := mailMessage.Date
+	reportUpdateTime := emailMessage.Date
 	if reportUpdateTime.IsZero() {
 		reportUpdateTime = time.Now()
 	}
 	reportInfo := &report.OutsideReport{
 		OutsideReportID:  0,
 		Source:           3,
-		Title:            mailMessage.Title,
+		Title:            emailMessage.Title,
 		Abstract:         "",
 		ClassifyID:       0,
 		ClassifyName:     "",

utils/mail/imap.go

@@ -29,7 +29,7 @@ type MailMessage struct {
 	Attachment map[string]string `description:"附件资源"`
 }
 
-func ListenMail(mailAddress, folder, userName, password string, readBatchSize int, mailMessageChan chan MailMessage, mailMessageDoneChan chan bool) (err error) { // 收件箱
+func ListenMail(mailAddress, folder, userName, password string, readBatchSize, fromEmailIndex int, mailMessageChan chan MailMessage, mailMessageDoneChan chan bool) (err error) { // 收件箱
 	defer func() {
 		// 处理结束
 		mailMessageDoneChan <- true
@@ -99,17 +99,22 @@ func ListenMail(mailAddress, folder, userName, password string, readBatchSize in
 	// 创建一个序列集，用于批量读取邮件
 	seqSet := new(imap.SeqSet)
 
-	// 假设需要获取最后4封邮件时
-	from := uint32(1)
 	to := mbox.Messages // 此文件下的邮件总数
-	var maxNum uint32
-
-	// 该次监听获取的最大数量
-	maxNum = 20000
 
-	if to > maxNum {
-		from = to - maxNum + 1
+	from := uint32(1)
+	// 假设需要获取最后4封邮件时
+	if fromEmailIndex > 0 {
+		from = uint32(fromEmailIndex)
+	} else {
+		var maxNum uint32
+		//该次监听获取的最大数量
+		maxNum = 20000
+		//获取开始的邮件编号
+		if to > maxNum {
+			from = to - maxNum + 1
+		}
 	}
+
 	step := uint32(5)
 	for i := from; i <= to; {
 		end := i + step - 1
@@ -474,7 +479,7 @@ func isIgnore(emailMessage MailMessage) bool {
 	lowerFrom := strings.ToLower(emailMessage.From)
 	for _, email := range global.CONFIG.Email.IgnoreEmail {
 		if utils.ContainsWholeWord(lowerFrom, email) {
-			global.FILE_LOG.Infof("发件人包含%s，过滤掉，标题：%s", email, emailMessage.Title)
+			global.FILE_LOG.Infof("发件人包含%s，过滤掉，标题：%s", email, emailMessage.From)
 			return true
 		}
 	}
@@ -482,7 +487,23 @@ func isIgnore(emailMessage MailMessage) bool {
 	// 邮件标题中包含待过滤的字符串（大小写敏感的标题），那么就过滤
 	for _, email := range global.CONFIG.Email.IgnoreEmailCaseSensitive {
 		if utils.ContainsWholeWord(emailMessage.From, email) {
-			global.FILE_LOG.Infof("发件人包含%s，过滤掉，标题：%s", email, emailMessage.Title)
+			global.FILE_LOG.Infof("发件人包含%s，过滤掉，标题：%s", email, emailMessage.From)
+			return true
+		}
+	}
+	// 发件人中包含待过滤的字符串，那么就过滤
+	lowerFromAddress := strings.ToLower(emailMessage.FromEmail)
+	for _, emailAddress := range global.CONFIG.Email.IgnoreEmailAddress {
+		if utils.ContainsWholeWord(lowerFromAddress, emailAddress) {
+			global.FILE_LOG.Infof("发件人邮箱包含%s，过滤掉，标题：%s", emailAddress, emailMessage.FromEmail)
+			return true
+		}
+	}
+
+	// 邮件标题中包含待过滤的字符串（大小写敏感的标题），那么就过滤
+	for _, emailAddress := range global.CONFIG.Email.IgnoreEmailAddressCaseSensitive {
+		if utils.ContainsWholeWord(emailMessage.FromEmail, emailAddress) {
+			global.FILE_LOG.Infof("发件人邮箱包含%s，过滤掉，标题：%s", emailAddress, emailMessage.FromEmail)
 			return true
 		}
 	}