eta_email_analysis/utils/mail/imap.go

package mail

import (
	"errors"
	"eta/eta_email_analysis/global"
	"eta/eta_email_analysis/models/eta"
	"eta/eta_email_analysis/models/report"
	"eta/eta_email_analysis/utils"
	"fmt"
	"github.com/emersion/go-imap"
	"github.com/emersion/go-imap/client"
	"github.com/emersion/go-message"
	"github.com/emersion/go-message/mail"
	"github.com/goccy/go-json"
	"github.com/h2non/filetype"
	"io"
	"log"
	"os"
	"path"
	"strings"
	"time"
)

type MailMessage struct {
	Date        time.Time         `description:"收件时间"`
	Uid         uint32            `description:"该邮件在邮箱中的唯一id"`
	FromAddress string            `description:"发件人邮箱"`
	From        string            `description:"发件人名称"`
	Title       string            `description:"邮件标题"`
	Content     string            `description:"邮件主体正文"`
	Resources   map[string]string `description:"正文内嵌资源"`
	Attachment  map[string]string `description:"附件资源"`
}

func ListenMail(mailAddress, folder, userName, password string, readBatchSize, fromEmailIndex int, mailMessageChan chan MailMessage, mailMessageDoneChan chan bool) (err error) { // 收件箱
	defer func() {
		// 处理结束
		mailMessageDoneChan <- true
		if err != nil {
			fmt.Println("err:", err.Error())
		}
	}()
	// 建立与 IMAP 服务器的连接
	c, err := client.DialTLS(mailAddress, nil)
	if err != nil {
		fmt.Printf("连接 IMAP 服务器失败: %+v \n", err)
		return
	}
	// 最后一定不要忘记退出登录
	defer func() {
		_ = c.Logout()
	}()

	// 登录
	if err = c.Login(userName, password); err != nil {
		fmt.Printf("邮箱[%s] 登录失败: %v \n", fmt.Sprintf("%s:%s", userName, mailAddress), err)
		return
	}
	// 列出当前邮箱中的文件夹
	mailboxes := make(chan *imap.MailboxInfo, 10)
	done := make(chan error, 1) // 记录错误的 chan
	go func() {
		done <- c.List("", "*", mailboxes)
	}()
	log.Println("-->当前邮箱的文件夹 Mailboxes:")

	var folderExists bool
	for m := range mailboxes {
		log.Println("* ", m.Name)
		if m.Name == folder {
			folderExists = true
		}
	}

	err = <-done
	if err != nil {
		global.LOG.Errorf("列出邮箱列表时，出现错误：%v \n", err)
		return
	}

	log.Println("-->列出邮箱列表完毕！")
	if !folderExists {
		err = errors.New(fmt.Sprintf("文件夹[%s] 不存在 \n", folder))
		return
	}

	message.CharsetReader = myCharsetReader

	// 选择指定的文件夹
	mbox, err := c.Select(folder, false)
	if err != nil {
		err = errors.New(fmt.Sprintf("选择邮件箱失败: %+v", err))
		return
	}
	//log.Printf("mbox %+v \n", mbox)
	log.Printf("当前文件夹[%s]中，总共有 %d 封邮件 \n", folder, mbox.Messages)
	if mbox.Messages == 0 {
		//log.Fatalf("当前文件夹[%s]中没有邮件", folder)
		return
	}

	// 创建一个序列集，用于批量读取邮件
	seqSet := new(imap.SeqSet)

	to := mbox.Messages // 此文件下的邮件总数

	//minIndex := uint32(5)
	//// 假设需要获取最后4封邮件时
	//if fromEmailIndex > 0 {
	//	minIndex = uint32(fromEmailIndex)
	//} else {
	//	var maxNum uint32
	//	//该次监听获取的最大数量
	//	maxNum = 20000
	//	//获取开始的邮件编号
	//	if to > maxNum {
	//		minIndex = to - maxNum + 1
	//	}
	//}
	//from = 310

	var isStopFor bool
	step := uint32(1)
	for i := to; i >= 1; {
		start := i - step + 1
		if start < 0 {
			start = 1
		}

		//fmt.Printf("当前剩余%d封邮件待处理\n", i-minIndex+1)

		seqSet.Clear()
		seqSet.AddRange(start, i) // 添加指定范围内的邮件编号

		// 获取整个消息正文
		// imap.FetchEnvelope：请求获取邮件的信封数据（例如发件人、收件人、主题等元数据）。
		// imap.FetchRFC822：请求获取完整的邮件内容，包括所有头部和正文。
		items := []imap.FetchItem{imap.FetchFlags, imap.FetchEnvelope, imap.FetchRFC822}

		// 获取邮件内容 Start
		messages := make(chan *imap.Message, readBatchSize) // 创建一个通道，用于接收邮件消息
		fetchDone := make(chan error, 1)                    // 创建一个通道，用于接收错误消息
		go func() {
			// Fetch方法用于从服务器获取邮件数据，这里请求了邮件的信封和完整内容
			fetchDone <- c.Fetch(seqSet, items, messages)
		}()

		err = <-fetchDone
		if err != nil {
			global.LOG.Errorf("获取邮件信息出现错误：%v \n", err)
			return
		}
		// 获取邮件内容 End

		// 获取规则
		ruleJson, e := eta.GetBusinessConfByKey("MailCheckRule")
		if e != nil {
			err = e
			global.FILE_LOG.Error("获取规则配置失败：%v", err)
			return
		}
		var ruleList []report.MailRule
		err = json.Unmarshal([]byte(ruleJson.ConfVal), &ruleList)
		if err != nil {
			global.FILE_LOG.Error("解析规则配置失败：%v", err)
			return
		}

		//log.Println("开始读取邮件内容")
		for msg := range messages {
			// 如果需要终止，那么就不处理了
			if isStopFor {
				continue
			}

			emailMessage, isRead, tmpErr := readEveryMsg(ruleList, msg)
			if tmpErr != nil {
				// 移除本地文件
				{
					for _, v := range emailMessage.Attachment {
						os.Remove(v)
					}
					for _, v := range emailMessage.Resources {
						os.Remove(v)
					}
				}
				global.FILE_LOG.Errorf("读取邮件内容时出现错误：%v \n", tmpErr)
				continue
			}
			// 如果没有取到，那么就过滤
			if !isRead {
				continue
			}

			// 判断当前邮件id是否小于等于已经监听到的最小id，如果是，那么就不处理了
			if emailMessage.Uid <= uint32(fromEmailIndex) {
				isStopFor = true
				continue
			}
			// 如果取到了，那么写入待处理chan
			// 写入邮件处理chan
			mailMessageChan <- emailMessage

		}

		if isStopFor {
			// 已经找到了最小的邮件id，那么就退出循环了
		}
		//time.Sleep(time.Second * 5) // 休眠10秒

		i = i - step
	}

	log.Println("读取了所有邮件，完毕！")

	return
}

// document link: https://github.com/emersion/go-imap/wiki/Fetching-messages
func readEveryMsg(ruleList []report.MailRule, msg *imap.Message) (emailMessage MailMessage, ok bool, err error) {
	ok = true
	defer func() {
		if err != nil {
			ok = false
			global.FILE_LOG.Errorf("邮件读取失败；Err:%s", err.Error())
		}
	}()
	message.CharsetReader = myCharsetReader
	emailMessage.Resources = make(map[string]string)  // 内嵌资源
	emailMessage.Attachment = make(map[string]string) // 附件

	emailMessage.Uid = msg.Uid
	htmlStr := ``
	textStr := ``

	//log.Printf("当前邮件的消息序列号 %+v \n", msg.SeqNum)
	//log.Println("-------------------------")
	// 获取邮件正文
	r := msg.GetBody(&imap.BodySectionName{})
	if r == nil {
		global.FILE_LOG.Info("服务器没有返回消息内容")
	}

	mr, err := mail.CreateReader(r)
	if err != nil {
		//log.Fatalf("邮件读取时出现错误： %v \n", err)
		err = errors.New(fmt.Sprintf("邮件读取时出现错误：%v \n", err))
		return
	}

	// 收件时间
	{
		date, err := mr.Header.Date()
		if err != nil {
			log.Println("收件时间 异常:", err.Error())
		}
		emailMessage.Date = date
		//log.Println("收件时间 Date:", date)
	}

	// 发件人
	{
		fromStr := mr.Header.Get("From")
		//fmt.Println(fromStr)
		// 处理无效地址的情况
		if !strings.Contains(fromStr, "@") {
			emailMessage.FromAddress = fromStr
			emailMessage.From = fromStr
		} else {
			from, tmpErr := mr.Header.AddressList("From")
			if tmpErr != nil {
				log.Println("发件人 异常:", err.Error())
			}
			if len(from) > 0 {
				emailMessage.FromAddress = from[0].Address
				emailMessage.From = from[0].Name
				//mailMessage.From = from[0].String()
				//log.Println("发件人 From:", from)
			}
		}
	}

	//if to, err := mr.Header.AddressList("To"); err == nil {
	//	log.Println("收件人 To:", to)
	//}
	//log.Printf("抄送 Cc: %+v \n", msg.Envelope.Cc)

	// 邮件标题
	subject, err := mr.Header.Subject()
	if err != nil {
		log.Println("邮件主题 Subject ERR:", err)
	} else {
		//log.Println("邮件主题 Subject:", subject)
	}
	emailMessage.Title = subject

	if !RuleCheck(ruleList, emailMessage) {
		ok = false
		return
	}

	// 过滤
	if isIgnore(emailMessage) {
		ok = false
		return
	}
	//fmt.Println("当前邮件Uid：", emailMessage.Uid)
	//ok = false
	//return

	for {
		p, tmpErr := mr.NextPart()
		if tmpErr == io.EOF {
			break
		} else if tmpErr != nil {
			global.FILE_LOG.Errorf("读取邮件内容时出现错误：%v \n", tmpErr)
			err = tmpErr
			return
		}

		bodyBytes, _ := io.ReadAll(p.Body)
		if err != nil {
			//log.Fatalf("读取邮件部分时出现错误：%v \n", err)
			err = errors.New(fmt.Sprintf("读取邮件部分时出现错误：%v \n", err))
			return
		}

		switch h := p.Header.(type) {
		case *mail.InlineHeader:
			// 这是消息的文本（可以是纯文本或 HTML）
			contentType := h.Get("Content-Type")
			//log.Println("消息内容content-type：", contentType)
			if strings.HasPrefix(contentType, "text/plain") {
				//log.Printf("得到正文 -> TEXT: %v \n", string(bodyBytes))
				textStr += string(bodyBytes)
			} else if strings.HasPrefix(contentType, "text/html") {
				//log.Printf("得到正文 -> HTML: %v \n", len(b))
				//log.Printf("得到正文 -> HTML: %v \n", string(bodyBytes))
				htmlStr += string(bodyBytes)
			}
			// 这是内嵌资源
			if cid := p.Header.Get("Content-ID"); cid != "" {

				// 确定文件后缀
				fileSuffix := determineFileSuffix(bodyBytes)
				fileName := fmt.Sprintf("%s%s.%s", global.CONFIG.Serve.StaticDir, cid[1:len(cid)-1], fileSuffix)

				err = utils.SaveToFile(bodyBytes, fileName)
				if err != nil {
					//log.Fatalf("保存文件时出现错误：%v \n", err)
					err = errors.New(fmt.Sprintf("保存文件时出现错误：%v \n", err))
					return
				}
				emailMessage.Resources[cid] = fileName
			}
			break
		case *mail.AttachmentHeader:
			// 这是一个附件
			filename, _ := h.Filename()
			//log.Printf("得到附件: %v,content-type:%s \n", filename, p.Header.Get("Content-Type"))
			saveName := fmt.Sprint(msg.SeqNum, utils.MD5(filename), time.Now().Format(utils.FormatDateTimeUnSpace), time.Now().Nanosecond(), path.Ext(filename))
			filePath := fmt.Sprintf("%s%s%s%s", global.CONFIG.Serve.StaticDir, `file`, string(os.PathSeparator), saveName)
			err = utils.SaveToFile(bodyBytes, filePath)
			if err != nil {
				//log.Fatalf("保存文件时出现错误：%v \n", err)
				err = errors.New(fmt.Sprintf("保存文件时出现错误：%v \n", err))
				return
			}

			// 这是附件资源
			if contentDisposition := p.Header.Get("Content-Disposition"); contentDisposition != "" {
				if strings.HasPrefix(contentDisposition, "attachment") {
					emailMessage.Attachment[filename] = filePath
				}
			} else if cid := p.Header.Get("Content-ID"); cid != "" {
				// 这是内嵌资源
				emailMessage.Resources[cid] = filePath
			}
			//else {
			//	mailMessage.Attachment[filename] = filePath
			//}

			break
		default:
			global.FILE_LOG.Info("未知格式:", h)
			//log.Println(h)
		}
	}
	emailMessage.Content = htmlStr
	if emailMessage.Content == `` {
		emailMessage.Content = textStr
	}

	//log.Println("一封邮件读取完毕")
	//log.Printf("------------------------- \n\n")

	return
}

// 根据文件内容确定文件后缀
func determineFileSuffix(content []byte) string {
	kind, err := filetype.Match(content)
	if err != nil {
		global.FILE_LOG.Error("无法确定文件类型：%v \n", err)
		return ".bin"
	}
	return kind.Extension
}

// isIgnore
// @Description: 校验是否忽略的邮件
// @author: Roc
// @datetime 2024-09-30 16:09:34
// @param emailMessage MailMessage
// @return bool
func isIgnore(emailMessage MailMessage) bool {
	// 发件人中包含待过滤的字符串，那么就过滤
	lowerFrom := strings.ToLower(emailMessage.From)
	for _, email := range global.CONFIG.Email.IgnoreEmail {
		if utils.ContainsWholeWord(lowerFrom, email) {
			global.FILE_LOG.Infof("发件人包含%s，过滤掉，发件人：%s；标题：%s；所属下标：%d", email, emailMessage.From, emailMessage.Title, emailMessage.Uid)
			return true
		}
	}

	// 邮件标题中包含待过滤的字符串（大小写敏感的标题），那么就过滤
	for _, email := range global.CONFIG.Email.IgnoreEmailCaseSensitive {
		if utils.ContainsWholeWord(emailMessage.From, email) {
			global.FILE_LOG.Infof("发件人包含%s，过滤掉，发件人：%s；标题：%s；所属下标：%d", email, emailMessage.From, emailMessage.Title, emailMessage.Uid)
			return true
		}
	}
	// 发件人地址中包含待过滤的字符串，那么就过滤
	lowerFromAddress := strings.ToLower(emailMessage.FromAddress)
	for _, emailAddress := range global.CONFIG.Email.IgnoreEmailAddress {
		if utils.ContainsWholeWord(lowerFromAddress, emailAddress) {
			global.FILE_LOG.Infof("发件人邮箱包含%s，过滤掉，发件人邮箱地址：%s；标题：%s；所属下标：%d", emailAddress, emailMessage.FromAddress, emailMessage.Title, emailMessage.Uid)
			return true
		}
	}

	// 邮件地址中包含待过滤的字符串（大小写敏感的标题），那么就过滤
	for _, emailAddress := range global.CONFIG.Email.IgnoreEmailAddressCaseSensitive {
		if utils.ContainsWholeWord(emailMessage.FromAddress, emailAddress) {
			global.FILE_LOG.Infof("发件人邮箱包含%s，过滤掉，发件人邮箱地址：%s；标题：%s；所属下标：%d", emailAddress, emailMessage.FromAddress, emailMessage.Title, emailMessage.Uid)
			return true
		}
	}

	// 邮件标题中包含待过滤的字符串，那么就过滤
	lowerTitle := strings.ToLower(emailMessage.Title)
	for _, title := range global.CONFIG.Email.IgnoreEmailTitle {
		if utils.ContainsWholeWord(lowerTitle, title) {
			global.FILE_LOG.Infof("邮件标题包含%s，过滤掉，标题：%s，所属下标：%d", title, emailMessage.Title, emailMessage.Uid)
			return true
		}
	}

	// 邮件标题中包含待过滤的字符串（大小写敏感的标题），那么就过滤
	for _, title := range global.CONFIG.Email.IgnoreEmailTitleCaseSensitive {
		if utils.ContainsWholeWord(emailMessage.Title, title) {
			global.FILE_LOG.Infof("邮件标题包含%s，过滤掉，标题：%s，所属下标：%d", title, emailMessage.Title, emailMessage.Uid)
			return true
		}
	}

	return false
}

func RuleCheck(ruleList []report.MailRule, emailMessage MailMessage) bool {
	for _, v := range ruleList {
		title := strings.ToLower(emailMessage.Title)
		rule := strings.ToLower(v.Rule)
		if strings.Contains(title, rule){
			return true
		}
	}

	return false
}