add missing References email header, fix Message-Id composing, fix email reply bugs

bot/command.go

@@ -364,15 +364,15 @@ func (b *Bot) runSend(ctx context.Context) {
 	defer b.unlock(evt.RoomID)
 
 	from := mailbox + "@" + b.domains[0]
-	ID := fmt.Sprintf("<%s@%s>", evt.ID, b.domains[0])
+	ID := utils.MessageID(evt.ID, b.domains[0])
 	for _, to := range tos {
-		email := utils.NewEmail(ID, "", subject, from, to, body, "", nil)
+		email := utils.NewEmail(ID, "", " "+ID, subject, from, to, body, "", nil)
 		data := email.Compose(b.getBotSettings().DKIMPrivateKey())
 		err = b.sendmail(from, to, data)
 		if err != nil {
 			b.Error(ctx, evt.RoomID, "cannot send email to %s: %v", to, err)
 		} else {
-			b.forgeSentMetadata(ctx, email, &cfg)
+			b.saveSentMetadata(ctx, email, &cfg)
 		}
 	}
 	if len(tos) > 1 {
@@ -380,9 +380,9 @@ func (b *Bot) runSend(ctx context.Context) {
 	}
 }
 
-// forgeSentMetadata used to save metadata from !pm sent event to a separate notice message
+// saveSentMetadata used to save metadata from !pm sent event to a separate notice message
 // because that metadata is needed to determine email thread relations
-func (b *Bot) forgeSentMetadata(ctx context.Context, email *utils.Email, cfg *roomSettings) {
+func (b *Bot) saveSentMetadata(ctx context.Context, email *utils.Email, cfg *roomSettings) {
 	evt := eventFromContext(ctx)
 	threadID := utils.EventParent(evt.ID, evt.Content.AsMessage())
 	content := email.Content(threadID, cfg.ContentOptions())
@@ -402,7 +402,7 @@ func (b *Bot) forgeSentMetadata(ctx context.Context, email *utils.Email, cfg *ro
 		return
 	}
 	if threadID != "" {
-		b.setThreadID(evt.RoomID, fmt.Sprintf("<%s@%s>", msgID, b.domains[0]), threadID)
+		b.setThreadID(evt.RoomID, utils.MessageID(msgID, b.domains[0]), threadID)
 	}
 	b.setLastEventID(evt.RoomID, threadID, msgID)
 }

bot/email.go

@@ -20,6 +20,7 @@ const (
 // event keys
 const (
 	eventMessageIDkey  = "cc.etke.postmoogle.messageID"
+	eventReferencesKey = "cc.etke.postmoogle.references"
 	eventInReplyToKey  = "cc.etke.postmoogle.inReplyTo"
 	eventSubjectKey    = "cc.etke.postmoogle.subject"
 	eventFromKey       = "cc.etke.postmoogle.from"
@@ -115,46 +116,60 @@ func (b *Bot) IncomingEmail(ctx context.Context, email *utils.Email) error {
 	return nil
 }
 
-func (b *Bot) getParentEmail(evt *event.Event) (string, string, string, string) {
+type parentEmail struct {
+	MessageID  string
+	From       string
+	To         string
+	InReplyTo  string
+	References string
+	Subject    string
+}
+
+func (b *Bot) getParentEmail(evt *event.Event) parentEmail {
+	var parent parentEmail
 	content := evt.Content.AsMessage()
 	parentID := utils.EventParent(evt.ID, content)
 	if parentID == evt.ID {
-		return "", "", "", ""
+		return parent
 	}
 	parentID = b.getLastEventID(evt.RoomID, parentID)
 	parentEvt, err := b.lp.GetClient().GetEvent(evt.RoomID, parentID)
 	if err != nil {
 		b.log.Error("cannot get parent event: %v", err)
-		return "", "", "", ""
+		return parent
 	}
 	if parentEvt.Content.Parsed == nil {
 		perr := parentEvt.Content.ParseRaw(event.EventMessage)
 		if perr != nil {
 			b.log.Error("cannot parse event content: %v", perr)
-			return "", "", "", ""
+			return parent
 		}
 	}
 
-	from := utils.EventField[string](&parentEvt.Content, eventFromKey)
+	parent.MessageID = utils.MessageID(parentID, b.domains[0])
-	to := utils.EventField[string](&parentEvt.Content, eventToKey)
+	parent.From = utils.EventField[string](&parentEvt.Content, eventFromKey)
-	inReplyTo := utils.EventField[string](&parentEvt.Content, eventMessageIDkey)
+	parent.To = utils.EventField[string](&parentEvt.Content, eventToKey)
-	if inReplyTo == "" {
+	parent.InReplyTo = utils.EventField[string](&parentEvt.Content, eventMessageIDkey)
-		inReplyTo = parentID.String()
+	parent.References = utils.EventField[string](&parentEvt.Content, eventReferencesKey)
+	if parent.InReplyTo == "" {
+		parent.InReplyTo = parent.MessageID
+	}
+	if parent.References == "" {
+		parent.References = " " + parent.MessageID
 	}
 
-	subject := utils.EventField[string](&parentEvt.Content, eventSubjectKey)
+	parent.Subject = utils.EventField[string](&parentEvt.Content, eventSubjectKey)
-	if subject != "" {
+	if parent.Subject != "" {
-		subject = "Re: " + subject
+		parent.Subject = "Re: " + parent.Subject
 	} else {
-		subject = strings.SplitN(content.Body, "\n", 1)[0]
+		parent.Subject = strings.SplitN(content.Body, "\n", 1)[0]
 	}
 
-	return from, to, inReplyTo, subject
+	return parent
 }
 
 // SendEmailReply sends replies from matrix thread to email thread
 func (b *Bot) SendEmailReply(ctx context.Context) {
-	var inReplyTo string
 	evt := eventFromContext(ctx)
 	cfg, err := b.getRoomSettings(evt.RoomID)
 	if err != nil {
@@ -169,35 +184,37 @@ func (b *Bot) SendEmailReply(ctx context.Context) {
 
 	b.lock(evt.RoomID)
 	defer b.unlock(evt.RoomID)
+
 	fromMailbox := mailbox + "@" + b.domains[0]
-	from, to, inReplyTo, subject := b.getParentEmail(evt)
+	meta := b.getParentEmail(evt)
 	// when email was sent from matrix and reply was sent from matrix again
-	if fromMailbox != from {
+	if fromMailbox != meta.From {
-		to = from
+		meta.To = meta.From
 	}
 
-	if to == "" {
+	if meta.To == "" {
 		b.Error(ctx, evt.RoomID, "cannot find parent email and continue the thread. Please, start a new email thread")
 		return
 	}
 
 	content := evt.Content.AsMessage()
-	if subject == "" {
+	if meta.Subject == "" {
-		subject = strings.SplitN(content.Body, "\n", 1)[0]
+		meta.Subject = strings.SplitN(content.Body, "\n", 1)[0]
 	}
 	body := content.Body
 
-	ID := evt.ID.String()[1:] + "@" + b.domains[0]
+	ID := utils.MessageID(evt.ID, b.domains[0])
-	b.log.Debug("send email reply ID=%s from=%s to=%s inReplyTo=%s subject=%s body=%s", ID, from, to, inReplyTo, subject, body)
+	meta.References = meta.References + " " + ID
-	data := utils.
+	b.log.Debug("send email reply ID=%s meta=%+v", ID, meta)
-		NewEmail(ID, inReplyTo, subject, from, to, body, "", nil).
+	email := utils.NewEmail(ID, meta.InReplyTo, meta.References, meta.Subject, meta.From, meta.To, body, "", nil)
-		Compose(b.getBotSettings().DKIMPrivateKey())
+	data := email.Compose(b.getBotSettings().DKIMPrivateKey())
 
-	err = b.sendmail(from, to, data)
+	err = b.sendmail(meta.From, meta.To, data)
 	if err != nil {
 		b.Error(ctx, evt.RoomID, "cannot send email: %v", err)
 		return
 	}
+	b.saveSentMetadata(ctx, email, &cfg)
 }
 
 func (b *Bot) sendFiles(ctx context.Context, roomID id.RoomID, files []*utils.File, noThreads bool, parentID id.EventID) {

bot/settings_room.go

@@ -151,6 +151,7 @@ func (s roomSettings) ContentOptions() *utils.ContentOptions {
 		SubjectKey:    eventSubjectKey,
 		MessageIDKey:  eventMessageIDkey,
 		InReplyToKey:  eventInReplyToKey,
+		ReferencesKey: eventReferencesKey,
 	}
 }
 

smtp/session.go

@@ -80,6 +80,7 @@ func (s *incomingSession) Data(r io.Reader) error {
 	email := utils.NewEmail(
 		eml.GetHeader("Message-Id"),
 		eml.GetHeader("In-Reply-To"),
+		eml.GetHeader("References"),
 		eml.GetHeader("Subject"),
 		s.from,
 		s.to,
@@ -132,6 +133,7 @@ func (s *outgoingSession) Data(r io.Reader) error {
 	email := utils.NewEmail(
 		eml.GetHeader("Message-Id"),
 		eml.GetHeader("In-Reply-To"),
+		eml.GetHeader("References"),
 		eml.GetHeader("Subject"),
 		s.from,
 		s.to,

utils/email.go

@@ -4,6 +4,7 @@ import (
 	"crypto"
 	"crypto/x509"
 	"encoding/pem"
+	"fmt"
 	"net/mail"
 	"strings"
 	"time"
@@ -26,6 +27,7 @@ type Email struct {
 	Date       string
 	MessageID  string
 	InReplyTo  string
+	References string
 	From       string
 	To         string
 	Subject    string
@@ -46,6 +48,7 @@ type ContentOptions struct {
 	// Keys
 	MessageIDKey  string
 	InReplyToKey  string
+	ReferencesKey string
 	SubjectKey    string
 	FromKey       string
 	ToKey         string
@@ -57,12 +60,18 @@ func AddressValid(email string) bool {
 	return err == nil
 }
 
+// MessageID generates email Message-Id from matrix event ID
+func MessageID(eventID id.EventID, domain string) string {
+	return fmt.Sprintf("<%s@%s>", eventID, domain)
+}
+
 // NewEmail constructs Email object
-func NewEmail(messageID, inReplyTo, subject, from, to, text, html string, files []*File) *Email {
+func NewEmail(messageID, inReplyTo, references, subject, from, to, text, html string, files []*File) *Email {
 	email := &Email{
 		Date:       time.Now().UTC().Format(time.RFC1123Z),
 		MessageID:  messageID,
 		InReplyTo:  inReplyTo,
+		References: references,
 		From:       from,
 		To:         to,
 		Subject:    subject,
@@ -121,6 +130,7 @@ func (e *Email) Content(threadID id.EventID, options *ContentOptions) *event.Con
 		Raw: map[string]interface{}{
 			options.MessageIDKey:  e.MessageID,
 			options.InReplyToKey:  e.InReplyTo,
+			options.ReferencesKey: e.References,
 			options.SubjectKey:    e.Subject,
 			options.FromKey:       e.From,
 			options.ToKey:         e.To,
@@ -167,6 +177,12 @@ func (e *Email) Compose(privkey string) string {
 		data.WriteString("\r\n")
 	}
 
+	if e.References != "" {
+		data.WriteString("References: ")
+		data.WriteString(e.References)
+		data.WriteString("\r\n")
+	}
+
 	data.WriteString("Subject: ")
 	data.WriteString(e.Subject)
 	data.WriteString("\r\n")