From f9cf94c914b6c51f7d1585067bd8a494f86a442b Mon Sep 17 00:00:00 2001 From: Aine Date: Wed, 24 Aug 2022 21:28:30 +0300 Subject: [PATCH] threads --- README.md | 2 +- bot/bot.go | 58 ++++++++++++++----- bot/data.go | 130 +++++++++++-------------------------------- bot/settings.go | 116 ++++++++++++++++++++++++++++++++++++++ e2e/simple.eml | 2 +- e2e/simple.reply.eml | 55 ++++++++++++++++++ smtp/session.go | 14 ++++- smtp/smtp.go | 2 +- utils/email.go | 13 +++++ 9 files changed, 279 insertions(+), 113 deletions(-) create mode 100644 bot/settings.go create mode 100644 e2e/simple.reply.eml create mode 100644 utils/email.go diff --git a/README.md b/README.md index 8769c19..bd0419f 100644 --- a/README.md +++ b/README.md @@ -13,7 +13,7 @@ An Email to Matrix bridge - [x] Configuration in room's account data - [x] Receive emails to matrix rooms - [x] Receive attachments -- [ ] Map email threads to matrix threads +- [x] Map email threads to matrix threads ### Send diff --git a/bot/bot.go b/bot/bot.go index d09fdf1..49121be 100644 --- a/bot/bot.go +++ b/bot/bot.go @@ -90,8 +90,8 @@ func (b *Bot) Start() error { } // Send email to matrix room -func (b *Bot) Send(ctx context.Context, from, to, subject, plaintext, html string, files []*utils.File) error { - roomID, ok := b.GetMapping(ctx, utils.Mailbox(to)) +func (b *Bot) Send(ctx context.Context, email *utils.Email) error { + roomID, ok := b.GetMapping(ctx, utils.Mailbox(email.To)) if !ok { return errors.New("room not found") } @@ -104,26 +104,56 @@ func (b *Bot) Send(ctx context.Context, from, to, subject, plaintext, html strin var text strings.Builder if !settings.NoSender() { text.WriteString("From: ") - text.WriteString(from) + text.WriteString(email.From) text.WriteString("\n\n") } if !settings.NoSubject() { text.WriteString("# ") - text.WriteString(subject) + text.WriteString(email.Subject) text.WriteString("\n\n") } - if html != "" { - text.WriteString(format.HTMLToMarkdown(html)) + if email.HTML != "" { + text.WriteString(format.HTMLToMarkdown(email.HTML)) } else { - text.WriteString(plaintext) + text.WriteString(email.Text) } - content := format.RenderMarkdown(text.String(), true, true) - _, err = b.lp.Send(roomID, content) - if err != nil { - return err + contentParsed := format.RenderMarkdown(text.String(), true, true) + + var threadID id.EventID + if email.InReplyTo != "" { + threadID = b.getThreadID(ctx, roomID, email.InReplyTo) + if threadID != "" { + contentParsed.SetRelatesTo(&event.RelatesTo{ + Type: event.RelThread, + EventID: threadID, + }) + b.setThreadID(ctx, roomID, email.MessageID, threadID) + } } + content := &event.Content{ + Raw: map[string]interface{}{ + eventMessageIDkey: email.MessageID, + eventInReplyToKey: email.InReplyTo, + }, + Parsed: contentParsed, + } + eventID, serr := b.lp.Send(roomID, content) + if serr != nil { + return serr + } + + if threadID == "" { + b.setThreadID(ctx, roomID, email.MessageID, eventID) + threadID = eventID + } + + b.sendFiles(ctx, roomID, email.Files, threadID) + return nil +} + +func (b *Bot) sendFiles(ctx context.Context, roomID id.RoomID, files []*utils.File, threadID id.EventID) { for _, file := range files { req := file.Convert() resp, err := b.lp.GetClient().UploadMedia(req) @@ -135,13 +165,15 @@ func (b *Bot) Send(ctx context.Context, from, to, subject, plaintext, html strin MsgType: event.MsgFile, Body: req.FileName, URL: resp.ContentURI.CUString(), + RelatesTo: &event.RelatesTo{ + Type: event.RelThread, + EventID: threadID, + }, }) if err != nil { b.Error(ctx, roomID, "cannot send uploaded file %s: %v", req.FileName, err) } } - - return nil } // GetMappings returns mapping of mailbox = room diff --git a/bot/data.go b/bot/data.go index 3b23a67..2f370cb 100644 --- a/bot/data.go +++ b/bot/data.go @@ -2,17 +2,25 @@ package bot import ( "context" - "strconv" "strings" "github.com/getsentry/sentry-go" "maunium.net/go/mautrix/id" - - "gitlab.com/etke.cc/postmoogle/utils" ) -const settingskey = "cc.etke.postmoogle.settings" +// account data keys +const ( + messagekey = "cc.etke.postmoogle.message" + settingskey = "cc.etke.postmoogle.settings" +) +// event keys +const ( + eventMessageIDkey = "cc.etke.postmoogle.messageID" + eventInReplyToKey = "cc.etke.postmoogle.inReplyTo" +) + +// option keys const ( optionOwner = "owner" optionMailbox = "mailbox" @@ -22,62 +30,6 @@ const ( var migrations = []string{} -// settings of a room -type settings map[string]string - -// settingsOld of a room -type settingsOld struct { - Mailbox string - Owner id.UserID - NoSender bool -} - -// Allowed checks if change is allowed -func (s settings) Allowed(noowner bool, userID id.UserID) bool { - if noowner { - return true - } - - owner := s.Owner() - if owner == "" { - return true - } - - return owner == userID.String() -} - -// Get option -func (s settings) Get(key string) string { - value := s[strings.ToLower(strings.TrimSpace(key))] - - sanitizer, exists := sanitizers[key] - if exists { - return sanitizer(value) - } - return value -} - -func (s settings) Mailbox() string { - return s.Get(optionMailbox) -} - -func (s settings) Owner() string { - return s.Get(optionOwner) -} - -func (s settings) NoSender() bool { - return utils.Bool(s.Get(optionNoSender)) -} - -func (s settings) NoSubject() bool { - return utils.Bool(s.Get(optionNoSubject)) -} - -// Set option -func (s settings) Set(key, value string) { - s[strings.ToLower(strings.TrimSpace(key))] = value -} - func (b *Bot) migrate() error { b.log.Debug("migrating database...") tx, beginErr := b.lp.GetDB().Begin() @@ -134,50 +86,36 @@ func (b *Bot) syncRooms(ctx context.Context) error { return nil } -// TODO: remove after migration -func (b *Bot) migrateSettings(ctx context.Context, roomID id.RoomID) { - var config settingsOld - err := b.lp.GetClient().GetRoomAccountData(roomID, settingskey, &config) - if err != nil { - // any error = no need to migrate - return - } - - if config.Mailbox == "" { - return - } - cfg := settings{} - cfg.Set(optionMailbox, config.Mailbox) - cfg.Set(optionOwner, config.Owner.String()) - cfg.Set(optionNoSender, strconv.FormatBool(config.NoSender)) - - err = b.setSettings(ctx, roomID, cfg) - if err != nil { - b.log.Error("cannot migrate settings: %v", err) - } -} - -func (b *Bot) getSettings(ctx context.Context, roomID id.RoomID) (settings, error) { - span := sentry.StartSpan(ctx, "http.server", sentry.TransactionName("getSettings")) +func (b *Bot) getThreadID(ctx context.Context, roomID id.RoomID, messageID string) id.EventID { + span := sentry.StartSpan(ctx, "http.server", sentry.TransactionName("getThreadID")) defer span.Finish() - config := settings{} - err := b.lp.GetClient().GetRoomAccountData(roomID, settingskey, &config) + key := messagekey + "." + messageID + data := map[string]id.EventID{} + err := b.lp.GetClient().GetRoomAccountData(roomID, key, &data) if err != nil { - if strings.Contains(err.Error(), "M_NOT_FOUND") { - // Suppress `M_NOT_FOUND (HTTP 404): Room account data not found` errors. - // Until some settings are explicitly set, we don't store any. - // In such cases, just return a default (empty) settings object. - err = nil + if !strings.Contains(err.Error(), "M_NOT_FOUND") { + b.log.Error("cannot retrieve account data %s: %v", key, err) + return "" } } - return config, err + return data["eventID"] } -func (b *Bot) setSettings(ctx context.Context, roomID id.RoomID, cfg settings) error { - span := sentry.StartSpan(ctx, "http.server", sentry.TransactionName("setSettings")) +func (b *Bot) setThreadID(ctx context.Context, roomID id.RoomID, messageID string, eventID id.EventID) { + span := sentry.StartSpan(ctx, "http.server", sentry.TransactionName("setThreadID")) defer span.Finish() - return b.lp.GetClient().SetRoomAccountData(roomID, settingskey, cfg) + key := messagekey + "." + messageID + data := map[string]id.EventID{ + "eventID": eventID, + } + + err := b.lp.GetClient().SetRoomAccountData(roomID, key, data) + if err != nil { + if !strings.Contains(err.Error(), "M_NOT_FOUND") { + b.log.Error("cannot save account data %s: %v", key, err) + } + } } diff --git a/bot/settings.go b/bot/settings.go new file mode 100644 index 0000000..01b0955 --- /dev/null +++ b/bot/settings.go @@ -0,0 +1,116 @@ +package bot + +import ( + "context" + "strconv" + "strings" + + "github.com/getsentry/sentry-go" + "maunium.net/go/mautrix/id" + + "gitlab.com/etke.cc/postmoogle/utils" +) + +// settings of a room +type settings map[string]string + +// settingsOld of a room +type settingsOld struct { + Mailbox string + Owner id.UserID + NoSender bool +} + +// Allowed checks if change is allowed +func (s settings) Allowed(noowner bool, userID id.UserID) bool { + if noowner { + return true + } + + owner := s.Owner() + if owner == "" { + return true + } + + return owner == userID.String() +} + +// Get option +func (s settings) Get(key string) string { + value := s[strings.ToLower(strings.TrimSpace(key))] + + sanitizer, exists := sanitizers[key] + if exists { + return sanitizer(value) + } + return value +} + +func (s settings) Mailbox() string { + return s.Get(optionMailbox) +} + +func (s settings) Owner() string { + return s.Get(optionOwner) +} + +func (s settings) NoSender() bool { + return utils.Bool(s.Get(optionNoSender)) +} + +func (s settings) NoSubject() bool { + return utils.Bool(s.Get(optionNoSubject)) +} + +// Set option +func (s settings) Set(key, value string) { + s[strings.ToLower(strings.TrimSpace(key))] = value +} + +// TODO: remove after migration +func (b *Bot) migrateSettings(ctx context.Context, roomID id.RoomID) { + var config settingsOld + err := b.lp.GetClient().GetRoomAccountData(roomID, settingskey, &config) + if err != nil { + // any error = no need to migrate + return + } + + if config.Mailbox == "" { + return + } + cfg := settings{} + cfg.Set(optionMailbox, config.Mailbox) + cfg.Set(optionOwner, config.Owner.String()) + cfg.Set(optionNoSender, strconv.FormatBool(config.NoSender)) + + err = b.setSettings(ctx, roomID, cfg) + if err != nil { + b.log.Error("cannot migrate settings: %v", err) + } +} + +func (b *Bot) getSettings(ctx context.Context, roomID id.RoomID) (settings, error) { + span := sentry.StartSpan(ctx, "http.server", sentry.TransactionName("getSettings")) + defer span.Finish() + + config := settings{} + err := b.lp.GetClient().GetRoomAccountData(roomID, settingskey, &config) + if err != nil { + if strings.Contains(err.Error(), "M_NOT_FOUND") { + // Suppress `M_NOT_FOUND (HTTP 404): Room account data not found` errors. + // Until some settings are explicitly set, we don't store any. + // In such cases, just return a default (empty) settings object. + err = nil + } + } + + return config, err +} + +func (b *Bot) setSettings(ctx context.Context, roomID id.RoomID, cfg settings) error { + span := sentry.StartSpan(ctx, "http.server", sentry.TransactionName("setSettings")) + defer span.Finish() + + return b.lp.GetClient().SetRoomAccountData(roomID, settingskey, cfg) +} diff --git a/e2e/simple.eml b/e2e/simple.eml index 8d4f24a..56eb757 100644 --- a/e2e/simple.eml +++ b/e2e/simple.eml @@ -2,7 +2,7 @@ From: James Hillyerd Content-Type: multipart/alternative; boundary="Apple-Mail=_E091454E-BCFA-43B4-99C0-678AEC9868D6" Subject: MIME test 1 Date: Sat, 13 Oct 2012 15:33:07 -0700 -Message-Id: <4E2E5A48-1A2C-4450-8663-D41B451DA93E@makita.skynet> +Message-Id: <4E2E5A48-1A2C-4450-8663-D41B451DA93A@makita.skynet> To: test@localhost Mime-Version: 1.0 (Apple Message framework v1283) X-Mailer: Apple Mail (2.1283) diff --git a/e2e/simple.reply.eml b/e2e/simple.reply.eml new file mode 100644 index 0000000..bb2b4f0 --- /dev/null +++ b/e2e/simple.reply.eml @@ -0,0 +1,55 @@ +From: James Hillyerd +Content-Type: multipart/alternative; boundary="Apple-Mail=_E091454E-BCFA-43B4-99C0-678AEC9868D6" +Subject: MIME test 2 with reply +Date: Sat, 13 Oct 2012 15:33:07 -0700 +Message-Id: <4E2E5A48-1A2C-4450-8663-D41B451DA93E@makita.skynet> +In-Reply-To: <4E2E5A48-1A2C-4450-8663-D41B451DA93A@makita.skynet> +To: test@localhost +Mime-Version: 1.0 (Apple Message framework v1283) +X-Mailer: Apple Mail (2.1283) + + +--Apple-Mail=_E091454E-BCFA-43B4-99C0-678AEC9868D6 +Content-Transfer-Encoding: 7bit +Content-Type: text/plain; + charset=us-ascii + +Test of text section +--Apple-Mail=_E091454E-BCFA-43B4-99C0-678AEC9868D6 +Content-Type: multipart/related; + type="text/html"; + boundary="Apple-Mail=_D2ABE25A-F0FE-404E-94EE-D98BD23448D5" + + +--Apple-Mail=_D2ABE25A-F0FE-404E-94EE-D98BD23448D5 +Content-Transfer-Encoding: 7bit +Content-Type: text/html; + charset=us-ascii + +Test of HTML section +--Apple-Mail=_D2ABE25A-F0FE-404E-94EE-D98BD23448D5 +Content-Transfer-Encoding: base64 +Content-Disposition: inline; + filename=favicon.png +Content-Type: image/png; + x-unix-mode=0644; + name="favicon.png" +Content-Id: <8B8481A2-25CA-4886-9B5A-8EB9115DD064@skynet> + +iVBORw0KGgoAAAANSUhEUgAAABAAAAAQCAYAAAAf8/9hAAAAGXRFWHRTb2Z0d2FyZQBBZG9iZSBJ +bWFnZVJlYWR5ccllPAAAAlFJREFUeNqUU8tOFEEUPVVdNV3dPe8xYRBnjGhmBgKjKzCIiQvBoIaN +bly5Z+PSv3Aj7DSiP2B0rwkLGVdGgxITSCRIJGSMEQWZR3eVt5sEFBgTb/dN1yvnnHtPNTPG4Pqd +HgCMXnPRSZrpSuH8vUJu4DE4rYHDGAZDX62BZttHqTiIayM3gGiXQsgYLEvATaqxU+dy1U13YXap +XptpNHY8iwn8KyIAzm1KBdtRZWErpI5lEWTXp5Z/vHpZ3/wyKKwYGGOdAYwR0EZwoezTYApBEIOb +yELl/aE1/83cp40Pt5mxqCKrE4Ck+mVWKKcI5tA8BLEhRBKJLjez6a7MLq7XZtp+yyOawwCBtkiB +VZDKzRk4NN7NQBMYPHiZDFhXY+p9ff7F961vVcnl4R5I2ykJ5XFN7Ab7Gc61VoipNBKF+PDyztu5 +lfrSLT/wIwCxq0CAGtXHZTzqR2jtwQiXONma6hHpj9sLT7YaPxfTXuZdBGA02Wi7FS48YiTfj+i2 +NhqtdhP5RC8mh2/Op7y0v6eAcWVLFT8D7kWX5S9mepp+C450MV6aWL1cGnvkxbwHtLW2B9AOkLeU +d9KEDuh9fl/7CEj7YH5g+3r/lWfF9In7tPz6T4IIwBJOr1SJyIGQMZQbsh5P9uBq5VJtqHh2mo49 +pdw5WFoEwKWqWHacaWOjQXWGcifKo6vj5RGS6zykI587XeUIQDqJSmAp+lE4qt19W5P9o8+Lma5D +cjsC8JiT607lMVkdqQ0Vyh3lHhmh52tfNy78ajXv0rgYzv8nfwswANuk+7sD/Q0aAAAAAElFTkSu +QmCC + +--Apple-Mail=_D2ABE25A-F0FE-404E-94EE-D98BD23448D5-- + +--Apple-Mail=_E091454E-BCFA-43B4-99C0-678AEC9868D6-- diff --git a/smtp/session.go b/smtp/session.go index 4c6a9be..a2c06ad 100644 --- a/smtp/session.go +++ b/smtp/session.go @@ -73,7 +73,19 @@ func (s *session) Data(r io.Reader) error { files := make([]*utils.File, 0, len(attachments)+len(inlines)) files = append(files, attachments...) files = append(files, inlines...) - return s.client.Send(s.ctx, s.from, s.to, eml.GetHeader("Subject"), eml.Text, eml.HTML, files) + + email := &utils.Email{ + MessageID: eml.GetHeader("Message-Id"), + InReplyTo: eml.GetHeader("In-Reply-To"), + Subject: eml.GetHeader("Subject"), + From: s.from, + To: s.to, + Text: eml.Text, + HTML: eml.HTML, + Files: files, + } + + return s.client.Send(s.ctx, email) } func (s *session) Reset() {} diff --git a/smtp/smtp.go b/smtp/smtp.go index 5c1e409..c2ae716 100644 --- a/smtp/smtp.go +++ b/smtp/smtp.go @@ -11,5 +11,5 @@ import ( // Client interface to send emails type Client interface { GetMapping(context.Context, string) (id.RoomID, bool) - Send(ctx context.Context, from, mailbox, subject, text, html string, files []*utils.File) error + Send(ctx context.Context, email *utils.Email) error } diff --git a/utils/email.go b/utils/email.go new file mode 100644 index 0000000..de3bfd9 --- /dev/null +++ b/utils/email.go @@ -0,0 +1,13 @@ +package utils + +// Email object +type Email struct { + MessageID string + InReplyTo string + From string + To string + Subject string + Text string + HTML string + Files []*File +}