From 9e532a60076df3b48dcde21e8902b873f232e070 Mon Sep 17 00:00:00 2001 From: Aine Date: Sat, 19 Nov 2022 16:41:53 +0200 Subject: [PATCH 1/6] initial cc support --- bot/email.go | 2 ++ bot/settings_room.go | 2 ++ smtp/server.go | 2 ++ smtp/session.go | 69 +++++++++++++++----------------------------- utils/email.go | 57 ++++++++++++++++++++++++++++++++++-- 5 files changed, 84 insertions(+), 48 deletions(-) diff --git a/bot/email.go b/bot/email.go index 8cb5a2e..d3e4d21 100644 --- a/bot/email.go +++ b/bot/email.go @@ -25,8 +25,10 @@ const ( eventReferencesKey = "cc.etke.postmoogle.references" eventInReplyToKey = "cc.etke.postmoogle.inReplyTo" eventSubjectKey = "cc.etke.postmoogle.subject" + eventRcptToKey = "cc.etke.postmoogle.rcptTo" eventFromKey = "cc.etke.postmoogle.from" eventToKey = "cc.etke.postmoogle.to" + eventCcKey = "cc.etke.postmoogle.cc" ) // SetSendmail sets mail sending func to the bot diff --git a/bot/settings_room.go b/bot/settings_room.go index f46c97a..91edda9 100644 --- a/bot/settings_room.go +++ b/bot/settings_room.go @@ -152,7 +152,9 @@ func (s roomSettings) ContentOptions() *utils.ContentOptions { Threads: !s.NoThreads(), ToKey: eventToKey, + CcKey: eventCcKey, FromKey: eventFromKey, + RcptToKey: eventRcptToKey, SubjectKey: eventSubjectKey, MessageIDKey: eventMessageIDkey, InReplyToKey: eventInReplyToKey, diff --git a/smtp/server.go b/smtp/server.go index 3301b87..cefdd9a 100644 --- a/smtp/server.go +++ b/smtp/server.go @@ -60,6 +60,7 @@ func (m *mailServer) Login(state *smtp.ConnectionState, username, password strin from: username, log: m.log, domains: m.domains, + tos: []string{}, }, nil } @@ -80,6 +81,7 @@ func (m *mailServer) AnonymousLogin(state *smtp.ConnectionState) (smtp.Session, log: m.log, domains: m.domains, addr: state.RemoteAddr, + tos: []string{}, }, nil } diff --git a/smtp/session.go b/smtp/session.go index ca75024..661ae6e 100644 --- a/smtp/session.go +++ b/smtp/session.go @@ -28,7 +28,7 @@ type incomingSession struct { ctx context.Context addr net.Addr - to string + tos []string from string } @@ -46,7 +46,7 @@ func (s *incomingSession) Mail(from string, opts smtp.MailOptions) error { func (s *incomingSession) Rcpt(to string) error { sentry.GetHubFromContext(s.ctx).Scope().SetTag("to", to) - s.to = to + s.tos = append(s.tos, to) var domainok bool for _, domain := range s.domains { if utils.Hostname(to) == domain { @@ -66,7 +66,7 @@ func (s *incomingSession) Rcpt(to string) error { } validations := s.getFilters(roomID) - if !validateEmail(s.from, s.to, s.log, validations) { + if !validateEmail(s.from, to, s.log, validations) { s.ban(s.addr) return ErrBanned } @@ -89,20 +89,15 @@ func (s *incomingSession) Data(r io.Reader) error { return err } - files := parseAttachments(eml.Attachments, s.log) - - email := utils.NewEmail( - eml.GetHeader("Message-Id"), - eml.GetHeader("In-Reply-To"), - eml.GetHeader("References"), - eml.GetHeader("Subject"), - s.from, - s.to, - eml.Text, - eml.HTML, - files) - - return s.receiveEmail(s.ctx, email) + email := utils.FromEnvelope(s.tos[0], eml) + for _, to := range s.tos { + email.RcptTo = to + err := s.receiveEmail(s.ctx, email) + if err != nil { + return err + } + } + return nil } func (s *incomingSession) Reset() {} func (s *incomingSession) Logout() error { return nil } @@ -115,7 +110,7 @@ type outgoingSession struct { domains []string ctx context.Context - to string + tos []string from string } @@ -129,7 +124,7 @@ func (s *outgoingSession) Mail(from string, opts smtp.MailOptions) error { func (s *outgoingSession) Rcpt(to string) error { sentry.GetHubFromContext(s.ctx).Scope().SetTag("to", to) - s.to = to + s.tos = append(s.tos, to) s.log.Debug("mail to %s", to) return nil @@ -141,21 +136,16 @@ func (s *outgoingSession) Data(r io.Reader) error { if err != nil { return err } + email := utils.FromEnvelope(s.tos[0], eml) + for _, to := range s.tos { + email.RcptTo = to + err := s.sendmail(email.From, to, email.Compose(s.privkey)) + if err != nil { + return err + } + } - files := parseAttachments(eml.Attachments, s.log) - - email := utils.NewEmail( - eml.GetHeader("Message-Id"), - eml.GetHeader("In-Reply-To"), - eml.GetHeader("References"), - eml.GetHeader("Subject"), - s.from, - s.to, - eml.Text, - eml.HTML, - files) - - return s.sendmail(email.From, email.To, email.Compose(s.privkey)) + return nil } func (s *outgoingSession) Reset() {} func (s *outgoingSession) Logout() error { return nil } @@ -170,16 +160,3 @@ func validateEmail(from, to string, log *logger.Logger, options utils.IncomingFi return v.Email(from) } - -func parseAttachments(parts []*enmime.Part, log *logger.Logger) []*utils.File { - files := make([]*utils.File, 0, len(parts)) - for _, attachment := range parts { - for _, err := range attachment.Errors { - log.Warn("attachment error: %v", err) - } - file := utils.NewFile(attachment.FileName, attachment.Content) - files = append(files, file) - } - - return files -} diff --git a/utils/email.go b/utils/email.go index 34b03e3..dda0025 100644 --- a/utils/email.go +++ b/utils/email.go @@ -34,6 +34,8 @@ type Email struct { References string From string To string + RcptTo string + CC string Subject string Text string HTML string @@ -56,6 +58,8 @@ type ContentOptions struct { SubjectKey string FromKey string ToKey string + CcKey string + RcptToKey string } // AddressValid checks if email address is valid @@ -64,6 +68,16 @@ func AddressValid(email string) bool { return err == nil } +// EmailDate returns Date in RFC1123 with numeric timezone +func EmailDate(original ...time.Time) string { + now := time.Now().UTC() + if len(original) > 0 && !original[0].IsZero() { + now = original[0] + } + + return now.Format(time.RFC1123Z) +} + // MessageID generates email Message-Id from matrix event ID func MessageID(eventID id.EventID, domain string) string { return fmt.Sprintf("<%s@%s>", eventID, domain) @@ -72,12 +86,13 @@ func MessageID(eventID id.EventID, domain string) string { // NewEmail constructs Email object func NewEmail(messageID, inReplyTo, references, subject, from, to, text, html string, files []*File) *Email { email := &Email{ - Date: time.Now().UTC().Format(time.RFC1123Z), + Date: EmailDate(), MessageID: messageID, InReplyTo: inReplyTo, References: references, From: from, To: to, + RcptTo: to, Subject: subject, Text: text, HTML: html, @@ -92,10 +107,46 @@ func NewEmail(messageID, inReplyTo, references, subject, from, to, text, html st return email } +func FromEnvelope(rcptto string, eml *enmime.Envelope) *Email { + datetime, _ := eml.Date() //nolint:errcheck // handled in EmailDate() + date := EmailDate(datetime) + + var html string + if eml.HTML != "" { + html = styleRegex.ReplaceAllString(eml.HTML, "") + } + + files := make([]*File, 0, len(eml.Attachments)) + for _, attachment := range eml.Attachments { + for _, err := range attachment.Errors { + log.Warn("attachment error: %v", err) + } + file := NewFile(attachment.FileName, attachment.Content) + files = append(files, file) + } + + email := &Email{ + Date: date, + MessageID: eml.GetHeader("Message-Id"), + InReplyTo: eml.GetHeader("In-Reply-To"), + References: eml.GetHeader("References"), + From: eml.GetHeader("From"), + To: eml.GetHeader("To"), + RcptTo: rcptto, + CC: eml.GetHeader("Cc"), + Subject: eml.GetHeader("Subject"), + Text: eml.Text, + HTML: html, + Files: files, + } + + return email +} + // Mailbox returns postmoogle's mailbox, parsing it from FROM (if incoming=false) or TO (incoming=true) func (e *Email) Mailbox(incoming bool) string { if incoming { - return Mailbox(e.To) + return Mailbox(e.RcptTo) } return Mailbox(e.From) } @@ -133,8 +184,10 @@ func (e *Email) Content(threadID id.EventID, options *ContentOptions) *event.Con options.InReplyToKey: e.InReplyTo, options.ReferencesKey: e.References, options.SubjectKey: e.Subject, + options.RcptToKey: e.RcptTo, options.FromKey: e.From, options.ToKey: e.To, + options.CcKey: e.CC, }, Parsed: &parsed, } From 052fd5bb2588bf6dddfd04a2eb66ccb7949dbb95 Mon Sep 17 00:00:00 2001 From: Aine Date: Sat, 19 Nov 2022 17:00:57 +0200 Subject: [PATCH 2/6] refactoring, created email package --- bot/command.go | 13 ++--- bot/email.go | 29 +++++----- bot/settings_room.go | 5 +- {utils => email}/email.go | 110 ++++++++++---------------------------- email/options.go | 28 ++++++++++ email/utils.go | 33 ++++++++++++ go.mod | 2 +- smtp/manager.go | 6 +-- smtp/server.go | 8 +-- smtp/session.go | 27 +++++----- utils/mail.go | 41 ++++++++++++++ utils/utils.go | 38 ------------- 12 files changed, 176 insertions(+), 164 deletions(-) rename {utils => email}/email.go (60%) create mode 100644 email/options.go create mode 100644 email/utils.go create mode 100644 utils/mail.go diff --git a/bot/command.go b/bot/command.go index aebfcc9..8233ef3 100644 --- a/bot/command.go +++ b/bot/command.go @@ -10,6 +10,7 @@ import ( "maunium.net/go/mautrix/format" "maunium.net/go/mautrix/id" + "gitlab.com/etke.cc/postmoogle/email" "gitlab.com/etke.cc/postmoogle/utils" ) @@ -415,7 +416,7 @@ func (b *Bot) runSend(ctx context.Context) { tos := strings.Split(to, ",") // validate first for _, to := range tos { - if !utils.AddressValid(to) { + if !email.AddressValid(to) { b.Error(ctx, evt.RoomID, "email address is not valid") return } @@ -426,10 +427,10 @@ func (b *Bot) runSend(ctx context.Context) { domain := utils.SanitizeDomain(cfg.Domain()) from := mailbox + "@" + domain - ID := utils.MessageID(evt.ID, domain) + ID := email.MessageID(evt.ID, domain) for _, to := range tos { - email := utils.NewEmail(ID, "", " "+ID, subject, from, to, body, htmlBody, nil) - data := email.Compose(b.getBotSettings().DKIMPrivateKey()) + eml := email.New(ID, "", " "+ID, subject, from, to, body, htmlBody, nil) + data := eml.Compose(b.getBotSettings().DKIMPrivateKey()) if data == "" { b.SendError(ctx, evt.RoomID, "email body is empty") return @@ -437,14 +438,14 @@ func (b *Bot) runSend(ctx context.Context) { queued, err := b.Sendmail(evt.ID, from, to, data) if queued { b.log.Error("cannot send email: %v", err) - b.saveSentMetadata(ctx, queued, evt.ID, email, &cfg) + b.saveSentMetadata(ctx, queued, evt.ID, eml, &cfg) continue } if err != nil { b.Error(ctx, evt.RoomID, "cannot send email to %s: %v", to, err) continue } - b.saveSentMetadata(ctx, false, evt.ID, email, &cfg) + b.saveSentMetadata(ctx, false, evt.ID, eml, &cfg) } if len(tos) > 1 { b.SendNotice(ctx, evt.RoomID, "All emails were sent.") diff --git a/bot/email.go b/bot/email.go index d3e4d21..ce9fec5 100644 --- a/bot/email.go +++ b/bot/email.go @@ -9,6 +9,7 @@ import ( "maunium.net/go/mautrix/format" "maunium.net/go/mautrix/id" + "gitlab.com/etke.cc/postmoogle/email" "gitlab.com/etke.cc/postmoogle/utils" ) @@ -85,7 +86,7 @@ func (b *Bot) GetMapping(mailbox string) (id.RoomID, bool) { } // GetIFOptions returns incoming email filtering options (room settings) -func (b *Bot) GetIFOptions(roomID id.RoomID) utils.IncomingFilteringOptions { +func (b *Bot) GetIFOptions(roomID id.RoomID) email.IncomingFilteringOptions { cfg, err := b.getRoomSettings(roomID) if err != nil { b.log.Error("cannot retrieve room settings: %v", err) @@ -96,7 +97,7 @@ func (b *Bot) GetIFOptions(roomID id.RoomID) utils.IncomingFilteringOptions { } // IncomingEmail sends incoming email to matrix room -func (b *Bot) IncomingEmail(ctx context.Context, email *utils.Email) error { +func (b *Bot) IncomingEmail(ctx context.Context, email *email.Email) error { roomID, ok := b.GetMapping(email.Mailbox(true)) if !ok { return errors.New("room not found") @@ -177,11 +178,11 @@ func (b *Bot) SendEmailReply(ctx context.Context) { body := content.Body htmlBody := content.FormattedBody - meta.MessageID = utils.MessageID(evt.ID, domain) + meta.MessageID = email.MessageID(evt.ID, domain) meta.References = meta.References + " " + meta.MessageID b.log.Debug("send email reply: %+v", meta) - email := utils.NewEmail(meta.MessageID, meta.InReplyTo, meta.References, meta.Subject, meta.From, meta.To, body, htmlBody, nil) - data := email.Compose(b.getBotSettings().DKIMPrivateKey()) + eml := email.New(meta.MessageID, meta.InReplyTo, meta.References, meta.Subject, meta.From, meta.To, body, htmlBody, nil) + data := eml.Compose(b.getBotSettings().DKIMPrivateKey()) if data == "" { b.SendError(ctx, evt.RoomID, "email body is empty") return @@ -190,7 +191,7 @@ func (b *Bot) SendEmailReply(ctx context.Context) { queued, err := b.Sendmail(evt.ID, meta.From, meta.To, data) if queued { b.log.Error("cannot send email: %v", err) - b.saveSentMetadata(ctx, queued, meta.ThreadID, email, &cfg) + b.saveSentMetadata(ctx, queued, meta.ThreadID, eml, &cfg) return } @@ -199,7 +200,7 @@ func (b *Bot) SendEmailReply(ctx context.Context) { return } - b.saveSentMetadata(ctx, queued, meta.ThreadID, email, &cfg) + b.saveSentMetadata(ctx, queued, meta.ThreadID, eml, &cfg) } type parentEmail struct { @@ -259,7 +260,7 @@ func (b *Bot) getParentEmail(evt *event.Event, domain string) parentEmail { return parent } - parent.MessageID = utils.MessageID(parentEvt.ID, domain) + parent.MessageID = email.MessageID(parentEvt.ID, domain) parent.From = utils.EventField[string](&parentEvt.Content, eventFromKey) parent.To = utils.EventField[string](&parentEvt.Content, eventToKey) parent.InReplyTo = utils.EventField[string](&parentEvt.Content, eventMessageIDkey) @@ -283,14 +284,14 @@ func (b *Bot) getParentEmail(evt *event.Event, domain string) parentEmail { // saveSentMetadata used to save metadata from !pm sent and thread reply events to a separate notice message // because that metadata is needed to determine email thread relations -func (b *Bot) saveSentMetadata(ctx context.Context, queued bool, threadID id.EventID, email *utils.Email, cfg *roomSettings) { - text := "Email has been sent to " + email.To +func (b *Bot) saveSentMetadata(ctx context.Context, queued bool, threadID id.EventID, eml *email.Email, cfg *roomSettings) { + text := "Email has been sent to " + eml.RcptTo if queued { - text = "Email to " + email.To + " has been queued" + text = "Email to " + eml.RcptTo + " has been queued" } evt := eventFromContext(ctx) - content := email.Content(threadID, cfg.ContentOptions()) + content := eml.Content(threadID, cfg.ContentOptions()) notice := format.RenderMarkdown(text, true, true) msgContent, ok := content.Parsed.(*event.MessageEventContent) if !ok { @@ -307,8 +308,8 @@ func (b *Bot) saveSentMetadata(ctx context.Context, queued bool, threadID id.Eve return } domain := utils.SanitizeDomain(cfg.Domain()) - b.setThreadID(evt.RoomID, utils.MessageID(evt.ID, domain), threadID) - b.setThreadID(evt.RoomID, utils.MessageID(msgID, domain), threadID) + b.setThreadID(evt.RoomID, email.MessageID(evt.ID, domain), threadID) + b.setThreadID(evt.RoomID, email.MessageID(msgID, domain), threadID) b.setLastEventID(evt.RoomID, threadID, msgID) } diff --git a/bot/settings_room.go b/bot/settings_room.go index 91edda9..54359c7 100644 --- a/bot/settings_room.go +++ b/bot/settings_room.go @@ -5,6 +5,7 @@ import ( "maunium.net/go/mautrix/id" + "gitlab.com/etke.cc/postmoogle/email" "gitlab.com/etke.cc/postmoogle/utils" ) @@ -143,8 +144,8 @@ func (s roomSettings) migrateSpamlistSettings() { } // ContentOptions converts room display settings to content options -func (s roomSettings) ContentOptions() *utils.ContentOptions { - return &utils.ContentOptions{ +func (s roomSettings) ContentOptions() *email.ContentOptions { + return &email.ContentOptions{ HTML: !s.NoHTML(), Sender: !s.NoSender(), Recipient: !s.NoRecipient(), diff --git a/utils/email.go b/email/email.go similarity index 60% rename from utils/email.go rename to email/email.go index dda0025..b54ecbd 100644 --- a/utils/email.go +++ b/email/email.go @@ -1,31 +1,20 @@ -package utils +package email import ( "crypto" "crypto/x509" "encoding/pem" - "fmt" - "net/mail" - "regexp" "strings" - "time" "github.com/emersion/go-msgauth/dkim" "github.com/jhillyerd/enmime" "maunium.net/go/mautrix/event" "maunium.net/go/mautrix/format" "maunium.net/go/mautrix/id" + + "gitlab.com/etke.cc/postmoogle/utils" ) -var styleRegex = regexp.MustCompile("") - -// IncomingFilteringOptions for incoming mail -type IncomingFilteringOptions interface { - SpamcheckSMTP() bool - SpamcheckMX() bool - Spamlist() []string -} - // Email object type Email struct { Date string @@ -39,54 +28,13 @@ type Email struct { Subject string Text string HTML string - Files []*File + Files []*utils.File } -// ContentOptions represents settings that specify how an email is to be converted to a Matrix message -type ContentOptions struct { - // On/Off - Sender bool - Recipient bool - Subject bool - HTML bool - Threads bool - - // Keys - MessageIDKey string - InReplyToKey string - ReferencesKey string - SubjectKey string - FromKey string - ToKey string - CcKey string - RcptToKey string -} - -// AddressValid checks if email address is valid -func AddressValid(email string) bool { - _, err := mail.ParseAddress(email) - return err == nil -} - -// EmailDate returns Date in RFC1123 with numeric timezone -func EmailDate(original ...time.Time) string { - now := time.Now().UTC() - if len(original) > 0 && !original[0].IsZero() { - now = original[0] - } - - return now.Format(time.RFC1123Z) -} - -// MessageID generates email Message-Id from matrix event ID -func MessageID(eventID id.EventID, domain string) string { - return fmt.Sprintf("<%s@%s>", eventID, domain) -} - -// NewEmail constructs Email object -func NewEmail(messageID, inReplyTo, references, subject, from, to, text, html string, files []*File) *Email { +// New constructs Email object +func New(messageID, inReplyTo, references, subject, from, to, text, html string, files []*utils.File) *Email { email := &Email{ - Date: EmailDate(), + Date: dateNow(), MessageID: messageID, InReplyTo: inReplyTo, References: references, @@ -107,35 +55,33 @@ func NewEmail(messageID, inReplyTo, references, subject, from, to, text, html st return email } -func FromEnvelope(rcptto string, eml *enmime.Envelope) *Email { - datetime, _ := eml.Date() //nolint:errcheck // handled in EmailDate() - date := EmailDate(datetime) +// FromEnvelope constructs Email object from envelope +func FromEnvelope(rcptto string, envelope *enmime.Envelope) *Email { + datetime, _ := envelope.Date() //nolint:errcheck // handled in dateNow() + date := dateNow(datetime) var html string - if eml.HTML != "" { - html = styleRegex.ReplaceAllString(eml.HTML, "") + if envelope.HTML != "" { + html = styleRegex.ReplaceAllString(envelope.HTML, "") } - files := make([]*File, 0, len(eml.Attachments)) - for _, attachment := range eml.Attachments { - for _, err := range attachment.Errors { - log.Warn("attachment error: %v", err) - } - file := NewFile(attachment.FileName, attachment.Content) + files := make([]*utils.File, 0, len(envelope.Attachments)) + for _, attachment := range envelope.Attachments { + file := utils.NewFile(attachment.FileName, attachment.Content) files = append(files, file) } email := &Email{ Date: date, - MessageID: eml.GetHeader("Message-Id"), - InReplyTo: eml.GetHeader("In-Reply-To"), - References: eml.GetHeader("References"), - From: eml.GetHeader("From"), - To: eml.GetHeader("To"), + MessageID: envelope.GetHeader("Message-Id"), + InReplyTo: envelope.GetHeader("In-Reply-To"), + References: envelope.GetHeader("References"), + From: envelope.GetHeader("From"), + To: envelope.GetHeader("To"), RcptTo: rcptto, - CC: eml.GetHeader("Cc"), - Subject: eml.GetHeader("Subject"), - Text: eml.Text, + CC: envelope.GetHeader("Cc"), + Subject: envelope.GetHeader("Subject"), + Text: envelope.Text, HTML: html, Files: files, } @@ -146,9 +92,9 @@ func FromEnvelope(rcptto string, eml *enmime.Envelope) *Email { // Mailbox returns postmoogle's mailbox, parsing it from FROM (if incoming=false) or TO (incoming=true) func (e *Email) Mailbox(incoming bool) string { if incoming { - return Mailbox(e.RcptTo) + return utils.Mailbox(e.RcptTo) } - return Mailbox(e.From) + return utils.Mailbox(e.From) } // Content converts the email object to a Matrix event content @@ -176,7 +122,7 @@ func (e *Email) Content(threadID id.EventID, options *ContentOptions) *event.Con } parsed := format.RenderMarkdown(text.String(), true, true) - parsed.RelatesTo = RelatesTo(options.Threads, threadID) + parsed.RelatesTo = utils.RelatesTo(options.Threads, threadID) content := event.Content{ Raw: map[string]interface{}{ @@ -222,13 +168,11 @@ func (e *Email) Compose(privkey string) string { root, err := mail.Build() if err != nil { - log.Error("cannot compose email: %v", err) return "" } var data strings.Builder err = root.Encode(&data) if err != nil { - log.Error("cannot encode email: %v", err) return "" } diff --git a/email/options.go b/email/options.go new file mode 100644 index 0000000..d7744a0 --- /dev/null +++ b/email/options.go @@ -0,0 +1,28 @@ +package email + +// IncomingFilteringOptions for incoming mail +type IncomingFilteringOptions interface { + SpamcheckSMTP() bool + SpamcheckMX() bool + Spamlist() []string +} + +// ContentOptions represents settings that specify how an email is to be converted to a Matrix message +type ContentOptions struct { + // On/Off + Sender bool + Recipient bool + Subject bool + HTML bool + Threads bool + + // Keys + MessageIDKey string + InReplyToKey string + ReferencesKey string + SubjectKey string + FromKey string + ToKey string + CcKey string + RcptToKey string +} diff --git a/email/utils.go b/email/utils.go new file mode 100644 index 0000000..3d844c6 --- /dev/null +++ b/email/utils.go @@ -0,0 +1,33 @@ +package email + +import ( + "fmt" + "net/mail" + "regexp" + "time" + + "maunium.net/go/mautrix/id" +) + +var styleRegex = regexp.MustCompile("") + +// AddressValid checks if email address is valid +func AddressValid(email string) bool { + _, err := mail.ParseAddress(email) + return err == nil +} + +// MessageID generates email Message-Id from matrix event ID +func MessageID(eventID id.EventID, domain string) string { + return fmt.Sprintf("<%s@%s>", eventID, domain) +} + +// dateNow returns Date in RFC1123 with numeric timezone +func dateNow(original ...time.Time) string { + now := time.Now().UTC() + if len(original) > 0 && !original[0].IsZero() { + now = original[0] + } + + return now.Format(time.RFC1123Z) +} diff --git a/go.mod b/go.mod index 30a43c0..7d309dc 100644 --- a/go.mod +++ b/go.mod @@ -22,7 +22,6 @@ require ( gitlab.com/etke.cc/go/trysmtp v1.0.0 gitlab.com/etke.cc/go/validator v1.0.4 gitlab.com/etke.cc/linkpearl v0.0.0-20221116205701-65547c5608e6 - golang.org/x/net v0.2.0 maunium.net/go/mautrix v0.12.3 ) @@ -49,6 +48,7 @@ require ( github.com/tidwall/sjson v1.2.5 // indirect github.com/yuin/goldmark v1.5.3 // indirect golang.org/x/crypto v0.3.0 // indirect + golang.org/x/net v0.2.0 // indirect golang.org/x/sys v0.2.0 // indirect golang.org/x/text v0.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect diff --git a/smtp/manager.go b/smtp/manager.go index 7b3ca36..4fafc4c 100644 --- a/smtp/manager.go +++ b/smtp/manager.go @@ -11,7 +11,7 @@ import ( "gitlab.com/etke.cc/go/logger" "maunium.net/go/mautrix/id" - "gitlab.com/etke.cc/postmoogle/utils" + "gitlab.com/etke.cc/postmoogle/email" ) type Config struct { @@ -45,8 +45,8 @@ type matrixbot interface { IsBanned(net.Addr) bool Ban(net.Addr) GetMapping(string) (id.RoomID, bool) - GetIFOptions(id.RoomID) utils.IncomingFilteringOptions - IncomingEmail(context.Context, *utils.Email) error + GetIFOptions(id.RoomID) email.IncomingFilteringOptions + IncomingEmail(context.Context, *email.Email) error SetSendmail(func(string, string, string) error) GetDKIMprivkey() string } diff --git a/smtp/server.go b/smtp/server.go index cefdd9a..c96e85f 100644 --- a/smtp/server.go +++ b/smtp/server.go @@ -10,7 +10,7 @@ import ( "gitlab.com/etke.cc/go/logger" "gitlab.com/etke.cc/go/trysmtp" - "gitlab.com/etke.cc/postmoogle/utils" + "gitlab.com/etke.cc/postmoogle/email" ) var ( @@ -41,7 +41,7 @@ func (m *mailServer) Login(state *smtp.ConnectionState, username, password strin return nil, ErrBanned } - if !utils.AddressValid(username) { + if !email.AddressValid(username) { m.log.Debug("address %s is invalid", username) m.bot.Ban(state.RemoteAddr) return nil, ErrBanned @@ -114,6 +114,6 @@ func (m *mailServer) SendEmail(from, to, data string) error { } // ReceiveEmail - incoming mail into matrix room -func (m *mailServer) ReceiveEmail(ctx context.Context, email *utils.Email) error { - return m.bot.IncomingEmail(ctx, email) +func (m *mailServer) ReceiveEmail(ctx context.Context, eml *email.Email) error { + return m.bot.IncomingEmail(ctx, eml) } diff --git a/smtp/session.go b/smtp/session.go index 661ae6e..3802198 100644 --- a/smtp/session.go +++ b/smtp/session.go @@ -13,6 +13,7 @@ import ( "gitlab.com/etke.cc/go/validator" "maunium.net/go/mautrix/id" + "gitlab.com/etke.cc/postmoogle/email" "gitlab.com/etke.cc/postmoogle/utils" ) @@ -20,8 +21,8 @@ import ( type incomingSession struct { log *logger.Logger getRoomID func(string) (id.RoomID, bool) - getFilters func(id.RoomID) utils.IncomingFilteringOptions - receiveEmail func(context.Context, *utils.Email) error + getFilters func(id.RoomID) email.IncomingFilteringOptions + receiveEmail func(context.Context, *email.Email) error greylisted func(net.Addr) bool ban func(net.Addr) domains []string @@ -34,7 +35,7 @@ type incomingSession struct { func (s *incomingSession) Mail(from string, opts smtp.MailOptions) error { sentry.GetHubFromContext(s.ctx).Scope().SetTag("from", from) - if !utils.AddressValid(from) { + if !email.AddressValid(from) { s.log.Debug("address %s is invalid", from) s.ban(s.addr) return ErrBanned @@ -84,15 +85,15 @@ func (s *incomingSession) Data(r io.Reader) error { } } parser := enmime.NewParser() - eml, err := parser.ReadEnvelope(r) + envelope, err := parser.ReadEnvelope(r) if err != nil { return err } - email := utils.FromEnvelope(s.tos[0], eml) + eml := email.FromEnvelope(s.tos[0], envelope) for _, to := range s.tos { - email.RcptTo = to - err := s.receiveEmail(s.ctx, email) + eml.RcptTo = to + err := s.receiveEmail(s.ctx, eml) if err != nil { return err } @@ -116,7 +117,7 @@ type outgoingSession struct { func (s *outgoingSession) Mail(from string, opts smtp.MailOptions) error { sentry.GetHubFromContext(s.ctx).Scope().SetTag("from", from) - if !utils.AddressValid(from) { + if !email.AddressValid(from) { return errors.New("please, provide email address") } return nil @@ -132,14 +133,14 @@ func (s *outgoingSession) Rcpt(to string) error { func (s *outgoingSession) Data(r io.Reader) error { parser := enmime.NewParser() - eml, err := parser.ReadEnvelope(r) + envelope, err := parser.ReadEnvelope(r) if err != nil { return err } - email := utils.FromEnvelope(s.tos[0], eml) + eml := email.FromEnvelope(s.tos[0], envelope) for _, to := range s.tos { - email.RcptTo = to - err := s.sendmail(email.From, to, email.Compose(s.privkey)) + eml.RcptTo = to + err := s.sendmail(eml.From, to, eml.Compose(s.privkey)) if err != nil { return err } @@ -150,7 +151,7 @@ func (s *outgoingSession) Data(r io.Reader) error { func (s *outgoingSession) Reset() {} func (s *outgoingSession) Logout() error { return nil } -func validateEmail(from, to string, log *logger.Logger, options utils.IncomingFilteringOptions) bool { +func validateEmail(from, to string, log *logger.Logger, options email.IncomingFilteringOptions) bool { enforce := validator.Enforce{ Email: true, MX: options.SpamcheckMX(), diff --git a/utils/mail.go b/utils/mail.go new file mode 100644 index 0000000..3be1b25 --- /dev/null +++ b/utils/mail.go @@ -0,0 +1,41 @@ +package utils + +import "strings" + +// Mailbox returns mailbox part from email address +func Mailbox(email string) string { + index := strings.LastIndex(email, "@") + if index == -1 { + return email + } + return email[:index] +} + +// EmailsList returns human-readable list of mailbox's emails for all available domains +func EmailsList(mailbox string, domain string) string { + var msg strings.Builder + domain = SanitizeDomain(domain) + msg.WriteString(mailbox) + msg.WriteString("@") + msg.WriteString(domain) + + count := len(domains) - 1 + for i, aliasDomain := range domains { + if i < count { + msg.WriteString(", ") + } + if aliasDomain == domain { + continue + } + msg.WriteString(mailbox) + msg.WriteString("@") + msg.WriteString(aliasDomain) + } + + return msg.String() +} + +// Hostname returns hostname part from email address +func Hostname(email string) string { + return email[strings.LastIndex(email, "@")+1:] +} diff --git a/utils/utils.go b/utils/utils.go index e937cf0..fd7af2d 100644 --- a/utils/utils.go +++ b/utils/utils.go @@ -22,44 +22,6 @@ func SetDomains(slice []string) { domains = slice } -// Mailbox returns mailbox part from email address -func Mailbox(email string) string { - index := strings.LastIndex(email, "@") - if index == -1 { - return email - } - return email[:index] -} - -// EmailsList returns human-readable list of mailbox's emails for all available domains -func EmailsList(mailbox string, domain string) string { - var msg strings.Builder - domain = SanitizeDomain(domain) - msg.WriteString(mailbox) - msg.WriteString("@") - msg.WriteString(domain) - - count := len(domains) - 1 - for i, aliasDomain := range domains { - if i < count { - msg.WriteString(", ") - } - if aliasDomain == domain { - continue - } - msg.WriteString(mailbox) - msg.WriteString("@") - msg.WriteString(aliasDomain) - } - - return msg.String() -} - -// Hostname returns hostname part from email address -func Hostname(email string) string { - return email[strings.LastIndex(email, "@")+1:] -} - // SanitizeDomain checks that input domain is available for use func SanitizeDomain(domain string) string { domain = strings.TrimSpace(domain) From 5fe8603506b4e2f773b933b2d5b47b22cb4a8f84 Mon Sep 17 00:00:00 2001 From: Aine Date: Sat, 19 Nov 2022 17:09:24 +0200 Subject: [PATCH 3/6] add `nocc` option --- README.md | 1 + bot/command.go | 9 +++++++++ bot/settings_room.go | 6 ++++++ email/email.go | 6 +++++- email/options.go | 1 + 5 files changed, 22 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 4e66550..f46bd9c 100644 --- a/README.md +++ b/README.md @@ -102,6 +102,7 @@ If you want to change them - check available options in the help message (`!pm h * **!pm nosender** - Get or set `nosender` of the room (`true` - hide email sender; `false` - show email sender) * **!pm norecipient** - Get or set `norecipient` of the room (`true` - hide recipient; `false` - show recipient) +* **!pm nocc** - Get or set `nocc` of the room (`true` - hide CC; `false` - show CC) * **!pm nosubject** - Get or set `nosubject` of the room (`true` - hide email subject; `false` - show email subject) * **!pm nohtml** - Get or set `nohtml` of the room (`true` - ignore HTML in email; `false` - parse HTML in emails) * **!pm nothreads** - Get or set `nothreads` of the room (`true` - ignore email threads; `false` - convert email threads into matrix threads) diff --git a/bot/command.go b/bot/command.go index 8233ef3..965017e 100644 --- a/bot/command.go +++ b/bot/command.go @@ -121,6 +121,15 @@ func (b *Bot) initCommands() commandList { sanitizer: utils.SanitizeBoolString, allowed: b.allowOwner, }, + { + key: roomOptionNoCC, + description: fmt.Sprintf( + "Get or set `%s` of the room (`true` - hide CC; `false` - show CC)", + roomOptionNoCC, + ), + sanitizer: utils.SanitizeBoolString, + allowed: b.allowOwner, + }, { key: roomOptionNoSubject, description: fmt.Sprintf( diff --git a/bot/settings_room.go b/bot/settings_room.go index 54359c7..bd62091 100644 --- a/bot/settings_room.go +++ b/bot/settings_room.go @@ -18,6 +18,7 @@ const ( roomOptionMailbox = "mailbox" roomOptionDomain = "domain" roomOptionNoSend = "nosend" + roomOptionNoCC = "nocc" roomOptionNoSender = "nosender" roomOptionNoRecipient = "norecipient" roomOptionNoSubject = "nosubject" @@ -62,6 +63,10 @@ func (s roomSettings) NoSend() bool { return utils.Bool(s.Get(roomOptionNoSend)) } +func (s roomSettings) NoCC() bool { + return utils.Bool(s.Get(roomOptionNoCC)) +} + func (s roomSettings) NoSender() bool { return utils.Bool(s.Get(roomOptionNoSender)) } @@ -146,6 +151,7 @@ func (s roomSettings) migrateSpamlistSettings() { // ContentOptions converts room display settings to content options func (s roomSettings) ContentOptions() *email.ContentOptions { return &email.ContentOptions{ + CC: !s.NoCC(), HTML: !s.NoHTML(), Sender: !s.NoSender(), Recipient: !s.NoRecipient(), diff --git a/email/email.go b/email/email.go index b54ecbd..026d76f 100644 --- a/email/email.go +++ b/email/email.go @@ -107,7 +107,11 @@ func (e *Email) Content(threadID id.EventID, options *ContentOptions) *event.Con text.WriteString(" ➡️ ") text.WriteString(e.To) } - if options.Sender || options.Recipient { + if options.CC && e.CC != "" { + text.WriteString("\ncc: ") + text.WriteString(e.CC) + } + if options.Sender || options.Recipient || options.CC { text.WriteString("\n\n") } if options.Subject && threadID == "" { diff --git a/email/options.go b/email/options.go index d7744a0..522ece8 100644 --- a/email/options.go +++ b/email/options.go @@ -10,6 +10,7 @@ type IncomingFilteringOptions interface { // ContentOptions represents settings that specify how an email is to be converted to a Matrix message type ContentOptions struct { // On/Off + CC bool Sender bool Recipient bool Subject bool From 8aac16aca8c577c02ff390182086ce971545356f Mon Sep 17 00:00:00 2001 From: Aine Date: Sat, 19 Nov 2022 17:38:13 +0200 Subject: [PATCH 4/6] make thread replies CC-aware and multi-domain aware --- bot/command.go | 2 +- bot/email.go | 49 ++++++++++++++++++++++++++++++++++++++++--------- email/email.go | 5 +++-- 3 files changed, 44 insertions(+), 12 deletions(-) diff --git a/bot/command.go b/bot/command.go index 965017e..cf3ce73 100644 --- a/bot/command.go +++ b/bot/command.go @@ -438,7 +438,7 @@ func (b *Bot) runSend(ctx context.Context) { from := mailbox + "@" + domain ID := email.MessageID(evt.ID, domain) for _, to := range tos { - eml := email.New(ID, "", " "+ID, subject, from, to, body, htmlBody, nil) + eml := email.New(ID, "", " "+ID, subject, from, to, to, "", body, htmlBody, nil) data := eml.Compose(b.getBotSettings().DKIMPrivateKey()) if data == "" { b.SendError(ctx, evt.RoomID, "email body is empty") diff --git a/bot/email.go b/bot/email.go index ce9fec5..d744587 100644 --- a/bot/email.go +++ b/bot/email.go @@ -155,13 +155,8 @@ func (b *Bot) SendEmailReply(ctx context.Context) { b.lock(evt.RoomID.String()) defer b.unlock(evt.RoomID.String()) - fromMailbox := mailbox + "@" + domain meta := b.getParentEmail(evt, domain) - // when email was sent from matrix and reply was sent from matrix again - if fromMailbox != meta.From { - meta.To = meta.From - } - meta.From = fromMailbox + meta.fixtofrom(mailbox, domain, b.domains) if meta.To == "" { b.Error(ctx, evt.RoomID, "cannot find parent email and continue the thread. Please, start a new email thread") @@ -181,7 +176,7 @@ func (b *Bot) SendEmailReply(ctx context.Context) { meta.MessageID = email.MessageID(evt.ID, domain) meta.References = meta.References + " " + meta.MessageID b.log.Debug("send email reply: %+v", meta) - eml := email.New(meta.MessageID, meta.InReplyTo, meta.References, meta.Subject, meta.From, meta.To, body, htmlBody, nil) + eml := email.New(meta.MessageID, meta.InReplyTo, meta.References, meta.Subject, meta.From, meta.To, meta.RcptTo, meta.CC, body, htmlBody, nil) data := eml.Compose(b.getBotSettings().DKIMPrivateKey()) if data == "" { b.SendError(ctx, evt.RoomID, "email body is empty") @@ -208,11 +203,45 @@ type parentEmail struct { ThreadID id.EventID From string To string + RcptTo string + CC string InReplyTo string References string Subject string } +// fixtofrom attempts to "fix" or rather reverse the To, From and CC headers +// of parent email by using parent email as metadata source for a new email +// that will be sent from postmoogle. +// To do so, we need to reverse From and To headers, but Cc should be adjusted as well, +// thus that hacky workaround below: +func (e *parentEmail) fixtofrom(newSenderMailbox string, preferredDomain string, domains []string) { + newSenders := make(map[string]struct{}, len(domains)) + newSenderPref := newSenderMailbox + "@" + preferredDomain + for _, domain := range domains { + sender := newSenderMailbox + "@" + domain + newSenders[sender] = struct{}{} + } + + originalFrom := e.From + // reverse From if needed + _, ok := newSenders[e.From] + if !ok { + e.From = newSenderPref + } + // reverse To if needed + _, ok = newSenders[e.To] + if ok { + e.To = originalFrom + } + // replace previous recipient of the email which is sender now with the original From + for newSender := range newSenders { + if strings.Contains(e.CC, newSender) { + e.CC = strings.ReplaceAll(e.CC, newSender, originalFrom) + } + } +} + func (b *Bot) getParentEvent(evt *event.Event) (id.EventID, *event.Event) { content := evt.Content.AsMessage() threadID := utils.EventParent(evt.ID, content) @@ -249,8 +278,8 @@ func (b *Bot) getParentEvent(evt *event.Event) (id.EventID, *event.Event) { return threadID, decrypted } -func (b *Bot) getParentEmail(evt *event.Event, domain string) parentEmail { - var parent parentEmail +func (b *Bot) getParentEmail(evt *event.Event, domain string) *parentEmail { + parent := &parentEmail{} threadID, parentEvt := b.getParentEvent(evt) parent.ThreadID = threadID if parentEvt == nil { @@ -263,6 +292,8 @@ func (b *Bot) getParentEmail(evt *event.Event, domain string) parentEmail { parent.MessageID = email.MessageID(parentEvt.ID, domain) parent.From = utils.EventField[string](&parentEvt.Content, eventFromKey) parent.To = utils.EventField[string](&parentEvt.Content, eventToKey) + parent.CC = utils.EventField[string](&parentEvt.Content, eventCcKey) + parent.RcptTo = utils.EventField[string](&parentEvt.Content, eventRcptToKey) parent.InReplyTo = utils.EventField[string](&parentEvt.Content, eventMessageIDkey) parent.References = utils.EventField[string](&parentEvt.Content, eventReferencesKey) if parent.InReplyTo == "" { diff --git a/email/email.go b/email/email.go index 026d76f..4d503ae 100644 --- a/email/email.go +++ b/email/email.go @@ -32,7 +32,7 @@ type Email struct { } // New constructs Email object -func New(messageID, inReplyTo, references, subject, from, to, text, html string, files []*utils.File) *Email { +func New(messageID, inReplyTo, references, subject, from, to, rcptto, cc, text, html string, files []*utils.File) *Email { email := &Email{ Date: dateNow(), MessageID: messageID, @@ -40,7 +40,8 @@ func New(messageID, inReplyTo, references, subject, from, to, text, html string, References: references, From: from, To: to, - RcptTo: to, + CC: cc, + RcptTo: rcptto, Subject: subject, Text: text, HTML: html, From 128d2b595ad11e1f92a741a46010a55431571a80 Mon Sep 17 00:00:00 2001 From: Aine Date: Sat, 19 Nov 2022 17:41:38 +0200 Subject: [PATCH 5/6] use the same sender's domain on thread reply as in parent email --- bot/email.go | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/bot/email.go b/bot/email.go index d744587..e3dc04a 100644 --- a/bot/email.go +++ b/bot/email.go @@ -156,7 +156,7 @@ func (b *Bot) SendEmailReply(ctx context.Context) { defer b.unlock(evt.RoomID.String()) meta := b.getParentEmail(evt, domain) - meta.fixtofrom(mailbox, domain, b.domains) + meta.fixtofrom(mailbox, b.domains) if meta.To == "" { b.Error(ctx, evt.RoomID, "cannot find parent email and continue the thread. Please, start a new email thread") @@ -215,19 +215,18 @@ type parentEmail struct { // that will be sent from postmoogle. // To do so, we need to reverse From and To headers, but Cc should be adjusted as well, // thus that hacky workaround below: -func (e *parentEmail) fixtofrom(newSenderMailbox string, preferredDomain string, domains []string) { - newSenders := make(map[string]struct{}, len(domains)) - newSenderPref := newSenderMailbox + "@" + preferredDomain +func (e *parentEmail) fixtofrom(newSenderMailbox string, domains []string) { + newSenders := make(map[string]string, len(domains)) for _, domain := range domains { sender := newSenderMailbox + "@" + domain - newSenders[sender] = struct{}{} + newSenders[sender] = sender } originalFrom := e.From // reverse From if needed - _, ok := newSenders[e.From] + newSender, ok := newSenders[e.From] if !ok { - e.From = newSenderPref + e.From = newSender } // reverse To if needed _, ok = newSenders[e.To] From 1bcf9bb0503010a3039eb54088c6ffcb3383cd9a Mon Sep 17 00:00:00 2001 From: Aine Date: Sat, 19 Nov 2022 18:05:26 +0200 Subject: [PATCH 6/6] set correct Message-Id, From, To, Cc, based on previous emails (and used domain) in the thread --- bot/email.go | 40 +++++++++++++++++++++++++++++----------- 1 file changed, 29 insertions(+), 11 deletions(-) diff --git a/bot/email.go b/bot/email.go index e3dc04a..7aca141 100644 --- a/bot/email.go +++ b/bot/email.go @@ -150,13 +150,11 @@ func (b *Bot) SendEmailReply(ctx context.Context) { b.Error(ctx, evt.RoomID, "mailbox is not configured, kupo") return } - domain := utils.SanitizeDomain(cfg.Domain()) b.lock(evt.RoomID.String()) defer b.unlock(evt.RoomID.String()) - meta := b.getParentEmail(evt, domain) - meta.fixtofrom(mailbox, b.domains) + meta := b.getParentEmail(evt, mailbox) if meta.To == "" { b.Error(ctx, evt.RoomID, "cannot find parent email and continue the thread. Please, start a new email thread") @@ -173,7 +171,7 @@ func (b *Bot) SendEmailReply(ctx context.Context) { body := content.Body htmlBody := content.FormattedBody - meta.MessageID = email.MessageID(evt.ID, domain) + meta.MessageID = email.MessageID(evt.ID, meta.FromDomain) meta.References = meta.References + " " + meta.MessageID b.log.Debug("send email reply: %+v", meta) eml := email.New(meta.MessageID, meta.InReplyTo, meta.References, meta.Subject, meta.From, meta.To, meta.RcptTo, meta.CC, body, htmlBody, nil) @@ -202,6 +200,7 @@ type parentEmail struct { MessageID string ThreadID id.EventID From string + FromDomain string To string RcptTo string CC string @@ -222,15 +221,33 @@ func (e *parentEmail) fixtofrom(newSenderMailbox string, domains []string) { newSenders[sender] = sender } + // try to determine previous email of the room mailbox + // by matching RCPT TO, To and From fields + // why? Because of possible multi-domain setup and we won't leak information + var previousSender string + rcptToSender, ok := newSenders[e.RcptTo] + if ok { + previousSender = rcptToSender + } + toSender, ok := newSenders[e.To] + if ok { + previousSender = toSender + } + fromSender, ok := newSenders[e.From] + if ok { + previousSender = fromSender + } + + // Message-Id should not leak information either + e.FromDomain = utils.SanitizeDomain(utils.Hostname(previousSender)) + originalFrom := e.From // reverse From if needed - newSender, ok := newSenders[e.From] - if !ok { - e.From = newSender + if fromSender == "" { + e.From = previousSender } // reverse To if needed - _, ok = newSenders[e.To] - if ok { + if toSender != "" { e.To = originalFrom } // replace previous recipient of the email which is sender now with the original From @@ -277,7 +294,7 @@ func (b *Bot) getParentEvent(evt *event.Event) (id.EventID, *event.Event) { return threadID, decrypted } -func (b *Bot) getParentEmail(evt *event.Event, domain string) *parentEmail { +func (b *Bot) getParentEmail(evt *event.Event, newFromMailbox string) *parentEmail { parent := &parentEmail{} threadID, parentEvt := b.getParentEvent(evt) parent.ThreadID = threadID @@ -288,13 +305,14 @@ func (b *Bot) getParentEmail(evt *event.Event, domain string) *parentEmail { return parent } - parent.MessageID = email.MessageID(parentEvt.ID, domain) parent.From = utils.EventField[string](&parentEvt.Content, eventFromKey) parent.To = utils.EventField[string](&parentEvt.Content, eventToKey) parent.CC = utils.EventField[string](&parentEvt.Content, eventCcKey) parent.RcptTo = utils.EventField[string](&parentEvt.Content, eventRcptToKey) parent.InReplyTo = utils.EventField[string](&parentEvt.Content, eventMessageIDkey) parent.References = utils.EventField[string](&parentEvt.Content, eventReferencesKey) + parent.fixtofrom(newFromMailbox, b.domains) + parent.MessageID = email.MessageID(parentEvt.ID, parent.FromDomain) if parent.InReplyTo == "" { parent.InReplyTo = parent.MessageID }