From 42cb5221a17d84da46cc6e1479f32e0b538403de Mon Sep 17 00:00:00 2001 From: Aine Date: Sun, 16 Oct 2022 20:14:54 +0300 Subject: [PATCH 1/2] spamlist wildcards --- README.md | 4 +-- bot/command.go | 24 ++----------- bot/settings_room.go | 86 +++++++++++++++++++++++++++++++------------- go.mod | 2 +- go.sum | 4 +-- smtp/msasession.go | 12 +++---- utils/email.go | 4 +-- 7 files changed, 73 insertions(+), 63 deletions(-) diff --git a/README.md b/README.md index 40800a7..dcb1ef8 100644 --- a/README.md +++ b/README.md @@ -256,9 +256,7 @@ If you want to change them - check available options in the help message (`!pm h * **!pm spamcheck:mx** - only accept email from servers which seem prepared to receive it (those having valid MX records) (`true` - enable, `false` - disable) * **!pm spamcheck:smtp** - only accept email from servers which seem prepared to receive it (those listening on an SMTP port) (`true` - enable, `false` - disable) -* **!pm spamlist:emails** - Get or set `spamlist:emails` of the room (comma-separated list), eg: `spammer@example.com,sspam@example.org` -* **!pm spamlist:hosts** - Get or set `spamlist:hosts` of the room (comma-separated list), eg: `spammer.com,scammer.com,morespam.com` -* **!pm spamlist:mailboxes** - Get or set `spamlist:mailboxes` of the room (comma-separated list), eg: `notspam,noreply,no-reply` +* **!pm spamlist** - Get or set `spamlist` of the room (comma-separated list), eg: `spammer@example.com,*@spammer.org,noreply@*` --- diff --git a/bot/command.go b/bot/command.go index df135b3..2c48a1f 100644 --- a/bot/command.go +++ b/bot/command.go @@ -157,28 +157,10 @@ func (b *Bot) initCommands() commandList { allowed: b.allowOwner, }, { - key: roomOptionSpamlistEmails, + key: roomOptionSpamlist, description: fmt.Sprintf( - "Get or set `%s` of the room (comma-separated list), eg: `spammer@example.com,sspam@example.org`", - roomOptionSpamlistEmails, - ), - sanitizer: utils.SanitizeStringSlice, - allowed: b.allowOwner, - }, - { - key: roomOptionSpamlistHosts, - description: fmt.Sprintf( - "Get or set `%s` of the room (comma-separated list), eg: `spammer.com,scammer.com,morespam.com`", - roomOptionSpamlistHosts, - ), - sanitizer: utils.SanitizeStringSlice, - allowed: b.allowOwner, - }, - { - key: roomOptionSpamlistLocalparts, - description: fmt.Sprintf( - "Get or set `%s` of the room (comma-separated list), eg: `notspam,noreply,no-reply`", - roomOptionSpamlistLocalparts, + "Get or set `%s` of the room (comma-separated list), eg: `spammer@example.com,*@spammer.org,spam@*`", + roomOptionSpamlist, ), sanitizer: utils.SanitizeStringSlice, allowed: b.allowOwner, diff --git a/bot/settings_room.go b/bot/settings_room.go index fcf9c8f..abf86b1 100644 --- a/bot/settings_room.go +++ b/bot/settings_room.go @@ -13,21 +13,19 @@ const acRoomSettingsKey = "cc.etke.postmoogle.settings" // option keys const ( - roomOptionOwner = "owner" - roomOptionMailbox = "mailbox" - roomOptionNoSend = "nosend" - roomOptionNoSender = "nosender" - roomOptionNoRecipient = "norecipient" - roomOptionNoSubject = "nosubject" - roomOptionNoHTML = "nohtml" - roomOptionNoThreads = "nothreads" - roomOptionNoFiles = "nofiles" - roomOptionPassword = "password" - roomOptionSpamcheckSMTP = "spamcheck:smtp" - roomOptionSpamcheckMX = "spamcheck:mx" - roomOptionSpamlistEmails = "spamlist:emails" - roomOptionSpamlistHosts = "spamlist:hosts" - roomOptionSpamlistLocalparts = "spamlist:mailboxes" + roomOptionOwner = "owner" + roomOptionMailbox = "mailbox" + roomOptionNoSend = "nosend" + roomOptionNoSender = "nosender" + roomOptionNoRecipient = "norecipient" + roomOptionNoSubject = "nosubject" + roomOptionNoHTML = "nohtml" + roomOptionNoThreads = "nothreads" + roomOptionNoFiles = "nofiles" + roomOptionPassword = "password" + roomOptionSpamcheckSMTP = "spamcheck:smtp" + roomOptionSpamcheckMX = "spamcheck:mx" + roomOptionSpamlist = "spamlist" ) type roomSettings map[string]string @@ -90,16 +88,9 @@ func (s roomSettings) SpamcheckMX() bool { return utils.Bool(s.Get(roomOptionSpamcheckMX)) } -func (s roomSettings) SpamlistEmails() []string { - return utils.StringSlice(s.Get(roomOptionSpamlistEmails)) -} - -func (s roomSettings) SpamlistHosts() []string { - return utils.StringSlice(s.Get(roomOptionSpamlistHosts)) -} - -func (s roomSettings) SpamlistLocalparts() []string { - return utils.StringSlice(s.Get(roomOptionSpamlistLocalparts)) +func (s roomSettings) Spamlist() []string { + s.migrateSpamlist() + return utils.StringSlice(s.Get(roomOptionSpamlist)) } // ContentOptions converts room display settings to content options @@ -130,3 +121,48 @@ func (b *Bot) getRoomSettings(roomID id.RoomID) (roomSettings, error) { func (b *Bot) setRoomSettings(roomID id.RoomID, cfg roomSettings) error { return utils.UnwrapError(b.lp.SetRoomAccountData(roomID, acRoomSettingsKey, cfg)) } + +func (s roomSettings) migrateSpamlist() { + uniq := map[string]struct{}{} + emails := utils.StringSlice(s.Get("spamlist:emails")) + localparts := utils.StringSlice(s.Get("spamlist:localparts")) + hosts := utils.StringSlice(s.Get("spamlist:hosts")) + list := utils.StringSlice(s.Get(roomOptionSpamlist)) + delete(s, "spamlist:emails") + delete(s, "spamlist:localparts") + delete(s, "spamlist:hosts") + + for _, email := range emails { + if email == "" { + continue + } + uniq[email] = struct{}{} + } + + for _, localpart := range localparts { + if localpart == "" { + continue + } + uniq[localpart+"@*"] = struct{}{} + } + + for _, host := range hosts { + if host == "" { + continue + } + uniq["*@"+host] = struct{}{} + } + + for _, item := range list { + if item == "" { + continue + } + uniq[item] = struct{}{} + } + + spamlist := make([]string, 0, len(uniq)) + for item := range uniq { + spamlist = append(spamlist, item) + } + s.Set(roomOptionSpamlist, strings.Join(spamlist, ",")) +} diff --git a/go.mod b/go.mod index 8f22472..745e6cf 100644 --- a/go.mod +++ b/go.mod @@ -19,7 +19,7 @@ require ( gitlab.com/etke.cc/go/mxidwc v1.0.0 gitlab.com/etke.cc/go/secgen v1.1.1 gitlab.com/etke.cc/go/trysmtp v1.0.0 - gitlab.com/etke.cc/go/validator v1.0.1 + gitlab.com/etke.cc/go/validator v1.0.2 gitlab.com/etke.cc/linkpearl v0.0.0-20221012104738-a977907db8b9 golang.org/x/net v0.0.0-20221014081412-f15817d10f9b maunium.net/go/mautrix v0.12.2 diff --git a/go.sum b/go.sum index b1d53a1..0888ec6 100644 --- a/go.sum +++ b/go.sum @@ -97,8 +97,8 @@ gitlab.com/etke.cc/go/secgen v1.1.1 h1:RmKOki725HIhWJHzPtAc9X4YvBneczndchpMgoDkE gitlab.com/etke.cc/go/secgen v1.1.1/go.mod h1:3pJqRGeWApzx7qXjABqz2o2SMCNpKSZao/gXVdasqE8= gitlab.com/etke.cc/go/trysmtp v1.0.0 h1:f/7gSmzohKniVeLSLevI+ZsySYcPUGkT9cRlOTwjOr8= gitlab.com/etke.cc/go/trysmtp v1.0.0/go.mod h1:KqRuIB2IPElEEbAxXmFyKtm7S5YiuEb4lxwWthccqyE= -gitlab.com/etke.cc/go/validator v1.0.1 h1:xp1tAzgCu9A1pga8rFUo7hODaEcCR1nkkodw96+dYuA= -gitlab.com/etke.cc/go/validator v1.0.1/go.mod h1:3vdssRG4LwgdTr9IHz9MjGSEO+3/FO9hXPGMuSeweJ8= +gitlab.com/etke.cc/go/validator v1.0.2 h1:7iVHG9sh1Hz6YcNT+tTLDm60B2PVSz6eh9nh6KOx7LI= +gitlab.com/etke.cc/go/validator v1.0.2/go.mod h1:3vdssRG4LwgdTr9IHz9MjGSEO+3/FO9hXPGMuSeweJ8= gitlab.com/etke.cc/linkpearl v0.0.0-20221012104738-a977907db8b9 h1:CJyYRf4KGmaFJDBJS5NXkt9v5ICi/AHrJIIOinQD/os= gitlab.com/etke.cc/linkpearl v0.0.0-20221012104738-a977907db8b9/go.mod h1:HkUHUkhbkDueEJVc7h/zBfz2hjhl4xxjQKv9Itrdf9k= golang.org/x/crypto v0.0.0-20220518034528-6f7dac969898/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= diff --git a/smtp/msasession.go b/smtp/msasession.go index 51d98be..c4cfca9 100644 --- a/smtp/msasession.go +++ b/smtp/msasession.go @@ -82,16 +82,12 @@ func (s *msasession) parseAttachments(parts []*enmime.Part) []*utils.File { } func (s *msasession) validate(options utils.IncomingFilteringOptions) bool { - spam := validator.Spam{ - Emails: options.SpamlistEmails(), - Hosts: options.SpamlistHosts(), - Localparts: options.SpamlistLocalparts(), - } enforce := validator.Enforce{ - MX: options.SpamcheckMX(), - SMTP: options.SpamcheckMX(), + Email: true, + MX: options.SpamcheckMX(), + SMTP: options.SpamcheckMX(), } - v := validator.New(spam, enforce, s.to, s.log) + v := validator.New(options.Spamlist(), enforce, s.to, s.log) return v.Email(s.from) } diff --git a/utils/email.go b/utils/email.go index 12c9814..e099cd1 100644 --- a/utils/email.go +++ b/utils/email.go @@ -23,9 +23,7 @@ type MTA interface { type IncomingFilteringOptions interface { SpamcheckSMTP() bool SpamcheckMX() bool - SpamlistEmails() []string - SpamlistHosts() []string - SpamlistLocalparts() []string + Spamlist() []string } // Email object From 4dd09dacb46cfb9e4c8d7704db3431435e4b124e Mon Sep 17 00:00:00 2001 From: Aine Date: Mon, 17 Oct 2022 19:01:21 +0300 Subject: [PATCH 2/2] move migration to the postmoogle inital room sync --- bot/data.go | 1 + bot/settings_room.go | 78 ++++++++++++++++++++++++++------------------ 2 files changed, 48 insertions(+), 31 deletions(-) diff --git a/bot/data.go b/bot/data.go index e29f645..f100ae8 100644 --- a/bot/data.go +++ b/bot/data.go @@ -41,6 +41,7 @@ func (b *Bot) syncRooms() error { if serr != nil { continue } + b.migrateRoomSettings(roomID) mailbox := cfg.Mailbox() if mailbox != "" { b.rooms.Store(mailbox, roomID) diff --git a/bot/settings_room.go b/bot/settings_room.go index abf86b1..349ffeb 100644 --- a/bot/settings_room.go +++ b/bot/settings_room.go @@ -89,40 +89,10 @@ func (s roomSettings) SpamcheckMX() bool { } func (s roomSettings) Spamlist() []string { - s.migrateSpamlist() return utils.StringSlice(s.Get(roomOptionSpamlist)) } -// ContentOptions converts room display settings to content options -func (s roomSettings) ContentOptions() *utils.ContentOptions { - return &utils.ContentOptions{ - HTML: !s.NoHTML(), - Sender: !s.NoSender(), - Recipient: !s.NoRecipient(), - Subject: !s.NoSubject(), - Threads: !s.NoThreads(), - - FromKey: eventFromKey, - SubjectKey: eventSubjectKey, - MessageIDKey: eventMessageIDkey, - InReplyToKey: eventInReplyToKey, - } -} - -func (b *Bot) getRoomSettings(roomID id.RoomID) (roomSettings, error) { - config, err := b.lp.GetRoomAccountData(roomID, acRoomSettingsKey) - if config == nil { - config = map[string]string{} - } - - return config, utils.UnwrapError(err) -} - -func (b *Bot) setRoomSettings(roomID id.RoomID, cfg roomSettings) error { - return utils.UnwrapError(b.lp.SetRoomAccountData(roomID, acRoomSettingsKey, cfg)) -} - -func (s roomSettings) migrateSpamlist() { +func (s roomSettings) migrateSpamlistSettings() { uniq := map[string]struct{}{} emails := utils.StringSlice(s.Get("spamlist:emails")) localparts := utils.StringSlice(s.Get("spamlist:localparts")) @@ -166,3 +136,49 @@ func (s roomSettings) migrateSpamlist() { } s.Set(roomOptionSpamlist, strings.Join(spamlist, ",")) } + +// ContentOptions converts room display settings to content options +func (s roomSettings) ContentOptions() *utils.ContentOptions { + return &utils.ContentOptions{ + HTML: !s.NoHTML(), + Sender: !s.NoSender(), + Recipient: !s.NoRecipient(), + Subject: !s.NoSubject(), + Threads: !s.NoThreads(), + + FromKey: eventFromKey, + SubjectKey: eventSubjectKey, + MessageIDKey: eventMessageIDkey, + InReplyToKey: eventInReplyToKey, + } +} + +func (b *Bot) getRoomSettings(roomID id.RoomID) (roomSettings, error) { + config, err := b.lp.GetRoomAccountData(roomID, acRoomSettingsKey) + if config == nil { + config = map[string]string{} + } + + return config, utils.UnwrapError(err) +} + +func (b *Bot) setRoomSettings(roomID id.RoomID, cfg roomSettings) error { + return utils.UnwrapError(b.lp.SetRoomAccountData(roomID, acRoomSettingsKey, cfg)) +} + +func (b *Bot) migrateRoomSettings(roomID id.RoomID) { + cfg, err := b.getRoomSettings(roomID) + if err != nil { + b.log.Error("cannot retrieve room settings: %v", err) + return + } + + if cfg["spamlist:emails"] == "" && cfg["spamlist:localparts"] == "" && cfg["spamlist:hosts"] == "" { + return + } + cfg.migrateSpamlistSettings() + err = b.setRoomSettings(roomID, cfg) + if err != nil { + b.log.Error("cannot migrate room settings: %v", err) + } +}