From 271a4a0e3198593807b08b0a19a415a3e644b628 Mon Sep 17 00:00:00 2001 From: Aine Date: Mon, 26 Feb 2024 20:42:37 +0200 Subject: [PATCH] add !pm stripify option --- README.md | 2 + bot/command.go | 9 + bot/config/room.go | 6 + bot/email.go | 22 ++ email/email.go | 11 +- email/options.go | 1 + go.mod | 5 +- go.sum | 10 +- .../kvannotten/mailstrip/.gitignore | 0 .../github.com/kvannotten/mailstrip/LICENSE | 52 ++++ .../github.com/kvannotten/mailstrip/README.md | 35 +++ .../kvannotten/mailstrip/mailstrip.go | 268 ++++++++++++++++++ vendor/gitlab.com/etke.cc/go/psd/client.go | 27 +- vendor/modules.txt | 7 +- 14 files changed, 442 insertions(+), 13 deletions(-) create mode 100644 vendor/github.com/kvannotten/mailstrip/.gitignore create mode 100644 vendor/github.com/kvannotten/mailstrip/LICENSE create mode 100644 vendor/github.com/kvannotten/mailstrip/README.md create mode 100644 vendor/github.com/kvannotten/mailstrip/mailstrip.go diff --git a/README.md b/README.md index afb5ab2..1564f13 100644 --- a/README.md +++ b/README.md @@ -20,6 +20,7 @@ so you can use it to send emails from your apps and scripts as well. - [x] Receive attachments - [x] Subaddressing support - [x] Catch-all mailbox +- [x] Strip forwarding, signatures, and other noise from emails if configured - [x] Map email threads to matrix threads - [x] Multi-domain support - [x] SMTP verification @@ -126,6 +127,7 @@ If you want to change them - check available options in the help message (`!pm h * **`!pm autoreply`** - Get or set autoreply of the room (markdown supported) that will be sent on any new incoming email thread * **`!pm signature`** - Get or set signature of the room (markdown supported) * **`!pm threadify`** - Get or set `threadify` of the room (`true` - send incoming email body in thread; `false` - send incoming email body as part of the message) +* **`!pm stripify`** - Get or set `threadify` of the room (`true` - strip incoming email's reply quotes and signatures; `false` - send incoming email as-is) * **`!pm nosend`** - Get or set `nosend` of the room (`true` - disable email sending; `false` - enable email sending) * **`!pm noreplies`** - Get or set `noreplies` of the room (`true` - ignore matrix replies; `false` - parse matrix replies) * **`!pm nosender`** - Get or set `nosender` of the room (`true` - hide email sender; `false` - show email sender) diff --git a/bot/command.go b/bot/command.go index fe99354..4453866 100644 --- a/bot/command.go +++ b/bot/command.go @@ -125,6 +125,15 @@ func (b *Bot) initCommands() commandList { sanitizer: utils.SanitizeBoolString, allowed: b.allowOwner, }, + { + key: config.RoomStripify, + description: fmt.Sprintf( + "Get or set `%s` of the room (`true` - strip incoming email reply quotes and signatures; `false` - send incoming email as-is)", + config.RoomStripify, + ), + sanitizer: utils.SanitizeBoolString, + allowed: b.allowOwner, + }, { key: config.RoomNoSend, description: fmt.Sprintf( diff --git a/bot/config/room.go b/bot/config/room.go index af9929e..104bc9e 100644 --- a/bot/config/room.go +++ b/bot/config/room.go @@ -23,6 +23,7 @@ const ( RoomAutoreply = "autoreply" RoomThreadify = "threadify" + RoomStripify = "stripify" RoomNoCC = "nocc" RoomNoFiles = "nofiles" RoomNoHTML = "nohtml" @@ -84,6 +85,10 @@ func (s Room) Threadify() bool { return utils.Bool(s.Get(RoomThreadify)) } +func (s Room) Stripify() bool { + return utils.Bool(s.Get(RoomStripify)) +} + func (s Room) NoSend() bool { return utils.Bool(s.Get(RoomNoSend)) } @@ -198,6 +203,7 @@ func (s Room) ContentOptions() *email.ContentOptions { Recipient: !s.NoRecipient(), Subject: !s.NoSubject(), Threads: !s.NoThreads(), + Stripify: s.Stripify(), Threadify: s.Threadify(), ToKey: "cc.etke.postmoogle.to", diff --git a/bot/email.go b/bot/email.go index 4032c8c..bc46369 100644 --- a/bot/email.go +++ b/bot/email.go @@ -146,6 +146,18 @@ func (b *Bot) IncomingEmail(ctx context.Context, eml *email.Email) error { b.setThreadID(ctx, roomID, eml.MessageID, threadID) } } + + // if automatic stripping is enabled, there is a chance something important may be stripped out + // to prevent that, we use a hacky way to generate content without stripping and save it as a file fist + if cfg.Stripify() && !cfg.Threadify() { + contentOpts := cfg.ContentOptions() + contentOpts.Stripify = false + content := eml.Content(threadID, contentOpts, b.psdc) + eml.Files = append(eml.Files, //nolint:forcetypeassert // that's ok + utils.NewFile("original.md", []byte(content.Parsed.(*event.MessageEventContent).Body)), + ) + } + content := eml.Content(threadID, cfg.ContentOptions(), b.psdc) eventID, serr := b.lp.Send(ctx, roomID, content) if serr != nil { @@ -164,6 +176,16 @@ func (b *Bot) IncomingEmail(ctx context.Context, eml *email.Email) error { b.setLastEventID(ctx, roomID, threadID, eventID) if newThread && cfg.Threadify() { + // if automatic stripping is enabled, there is a chance something important may be stripped out + // to prevent that, we use a hacky way to generate content without stripping and save it as a file fist + if cfg.Stripify() { + contentOpts := cfg.ContentOptions() + contentOpts.Stripify = false + content := eml.ContentBody(threadID, contentOpts) + eml.Files = append(eml.Files, //nolint:forcetypeassert // that's ok + utils.NewFile("original.md", []byte(content.Parsed.(*event.MessageEventContent).Body)), + ) + } _, berr := b.lp.Send(ctx, roomID, eml.ContentBody(threadID, cfg.ContentOptions())) if berr != nil { return berr diff --git a/email/email.go b/email/email.go index ebfce0d..ac29a46 100644 --- a/email/email.go +++ b/email/email.go @@ -8,6 +8,7 @@ import ( "github.com/emersion/go-msgauth/dkim" "github.com/jhillyerd/enmime" + "github.com/kvannotten/mailstrip" "gitlab.com/etke.cc/go/psd" "gitlab.com/etke.cc/linkpearl" "maunium.net/go/mautrix/event" @@ -173,7 +174,11 @@ func (e *Email) Content(threadID id.EventID, options *ContentOptions, psdc *psd. } } - parsed := format.RenderMarkdown(text.String(), true, true) + body := text.String() + if options.Stripify && threadID != "" { // strip only in thread replies + body = mailstrip.Parse(body).String() + } + parsed := format.RenderMarkdown(body, true, true) parsed.RelatesTo = linkpearl.RelatesTo(threadID, !options.Threads) var cc string @@ -210,6 +215,10 @@ func (e *Email) ContentBody(threadID id.EventID, options *ContentOptions) *event text = e.Text } + if options.Stripify { + text = mailstrip.Parse(text).String() + } + parsed := format.RenderMarkdown(text, true, true) parsed.RelatesTo = linkpearl.RelatesTo(threadID, !options.Threads) diff --git a/email/options.go b/email/options.go index 00e1210..47232a4 100644 --- a/email/options.go +++ b/email/options.go @@ -19,6 +19,7 @@ type ContentOptions struct { HTML bool Threads bool Threadify bool + Stripify bool // Keys MessageIDKey string diff --git a/go.mod b/go.mod index 9b9427d..fdee6f5 100644 --- a/go.mod +++ b/go.mod @@ -15,6 +15,7 @@ require ( github.com/gabriel-vasile/mimetype v1.4.3 github.com/getsentry/sentry-go v0.27.0 github.com/jhillyerd/enmime v1.2.0 + github.com/kvannotten/mailstrip v0.0.0-20200711213611-0002f5c0467e github.com/lib/pq v1.10.9 github.com/mattn/go-sqlite3 v1.14.22 github.com/mcnijman/go-emailaddress v1.1.1 @@ -25,11 +26,11 @@ require ( gitlab.com/etke.cc/go/fswatcher v1.0.0 gitlab.com/etke.cc/go/healthchecks v1.0.1 gitlab.com/etke.cc/go/mxidwc v1.0.0 - gitlab.com/etke.cc/go/psd v1.0.0 + gitlab.com/etke.cc/go/psd v1.1.1 gitlab.com/etke.cc/go/secgen v1.2.0 gitlab.com/etke.cc/go/validator v1.0.7 gitlab.com/etke.cc/linkpearl v0.0.0-20240211143445-bddf907d137a - golang.org/x/exp v0.0.0-20240213143201-ec583247a57a + golang.org/x/exp v0.0.0-20240222234643-814bf88cf225 maunium.net/go/mautrix v0.17.0 ) diff --git a/go.sum b/go.sum index 2c56060..c9859f4 100644 --- a/go.sum +++ b/go.sum @@ -41,6 +41,8 @@ github.com/jaytaylor/html2text v0.0.0-20230321000545-74c2419ad056 h1:iCHtR9CQykt github.com/jaytaylor/html2text v0.0.0-20230321000545-74c2419ad056/go.mod h1:CVKlgaMiht+LXvHG173ujK6JUhZXKb2u/BQtjPDIvyk= github.com/jhillyerd/enmime v1.2.0 h1:dIu1IPEymQgoT2dzuB//ttA/xcV40NMPpQtmd4wslHk= github.com/jhillyerd/enmime v1.2.0/go.mod h1:FRFuUPCLh8PByQv+8xRcLO9QHqaqTqreYhopv5eyk4I= +github.com/kvannotten/mailstrip v0.0.0-20200711213611-0002f5c0467e h1:GD97grZITUgTUFuCmGuOJ//4a/XW85Y4LcoSvqoeyz4= +github.com/kvannotten/mailstrip v0.0.0-20200711213611-0002f5c0467e/go.mod h1:Soh3wPUKGQ476v2WNt/mArcSE2GkSai7EK1yLoGqnhM= github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw= github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= @@ -100,8 +102,8 @@ gitlab.com/etke.cc/go/healthchecks v1.0.1 h1:IxPB+r4KtEM6wf4K7MeQoH1XnuBITMGUqFa gitlab.com/etke.cc/go/healthchecks v1.0.1/go.mod h1:EzQjwSawh8tQEX43Ls0dI9mND6iWd5NHtmapdO24fMI= gitlab.com/etke.cc/go/mxidwc v1.0.0 h1:6EAlJXvs3nU4RaMegYq6iFlyVvLw7JZYnZmNCGMYQP0= gitlab.com/etke.cc/go/mxidwc v1.0.0/go.mod h1:E/0kh45SAN9+ntTG0cwkAEKdaPxzvxVmnjwivm9nmz8= -gitlab.com/etke.cc/go/psd v1.0.0 h1:ucr0/1Qq92hFT/mjqr/k3rHaVpPHEDTRbX6koTKon7Y= -gitlab.com/etke.cc/go/psd v1.0.0/go.mod h1:6b444NOkXlZ1n7WLCNazAkOC2bHPgqgfB9earThwKPk= +gitlab.com/etke.cc/go/psd v1.1.1 h1:UIL0X+thvYaeBTX8/G6lilqAToGCypihujGu5gtK5zQ= +gitlab.com/etke.cc/go/psd v1.1.1/go.mod h1:6b444NOkXlZ1n7WLCNazAkOC2bHPgqgfB9earThwKPk= gitlab.com/etke.cc/go/secgen v1.2.0 h1:qpV7rUn5Rs6eWxAmbGG/idPCOgsN4HggGmSZ+1R/L70= gitlab.com/etke.cc/go/secgen v1.2.0/go.mod h1:v5L07AIXtNpC/miYiK0TMIn+ZKbiYrTRiXTw6qTL6pw= gitlab.com/etke.cc/go/trysmtp v1.1.3 h1:e2EHond77onMaecqCg6mWumffTSEf+ycgj88nbeefDI= @@ -114,8 +116,8 @@ go.mau.fi/util v0.4.0 h1:S2X3qU4pUcb/vxBRfAuZjbrR9xVMAXSjQojNBLPBbhs= go.mau.fi/util v0.4.0/go.mod h1:leeiHtgVBuN+W9aDii3deAXnfC563iN3WK6BF8/AjNw= golang.org/x/crypto v0.19.0 h1:ENy+Az/9Y1vSrlrvBSyna3PITt4tiZLf7sgCjZBX7Wo= golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU= -golang.org/x/exp v0.0.0-20240213143201-ec583247a57a h1:HinSgX1tJRX3KsL//Gxynpw5CTOAIPhgL4W8PNiIpVE= -golang.org/x/exp v0.0.0-20240213143201-ec583247a57a/go.mod h1:CxmFvTBINI24O/j8iY7H1xHzx2i4OsyguNBmN/uPtqc= +golang.org/x/exp v0.0.0-20240222234643-814bf88cf225 h1:LfspQV/FYTatPTr/3HzIcmiUFH7PGP+OQ6mgDYo3yuQ= +golang.org/x/exp v0.0.0-20240222234643-814bf88cf225/go.mod h1:CxmFvTBINI24O/j8iY7H1xHzx2i4OsyguNBmN/uPtqc= golang.org/x/net v0.0.0-20180911220305-26e67e76b6c3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.21.0 h1:AQyQV4dYCvJ7vGmJyKki9+PBdyvhkSd8EIx/qb0AYv4= golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44= diff --git a/vendor/github.com/kvannotten/mailstrip/.gitignore b/vendor/github.com/kvannotten/mailstrip/.gitignore new file mode 100644 index 0000000..e69de29 diff --git a/vendor/github.com/kvannotten/mailstrip/LICENSE b/vendor/github.com/kvannotten/mailstrip/LICENSE new file mode 100644 index 0000000..64e82c6 --- /dev/null +++ b/vendor/github.com/kvannotten/mailstrip/LICENSE @@ -0,0 +1,52 @@ +All original parts of this library that are not considered derivate work of +email_reply_parser: +------------------------------------------------------------------------------- +The MIT License (MIT) + +Copyright (c) 2013 Thomson Reuters Global Resources + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. +------------------------------------------------------------------------------- + +The content of the fixtures directory (as imported in 78ad5d), as well as the +comments are copied from email_reply_parser. Most of the code itself is a +line-by-line port, and is therefor likely to be considered a derivate work: +------------------------------------------------------------------------------- +The MIT License + +Copyright (c) GitHub + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. +------------------------------------------------------------------------------- diff --git a/vendor/github.com/kvannotten/mailstrip/README.md b/vendor/github.com/kvannotten/mailstrip/README.md new file mode 100644 index 0000000..9d4921c --- /dev/null +++ b/vendor/github.com/kvannotten/mailstrip/README.md @@ -0,0 +1,35 @@ +# mailstrip + +mailstrip is a [Go][2] library that parses email text and strips it of +signatures and reply quotes. It is a port of [email\_reply\_parser][1], GitHub's +library for parsing email replies. + +## Differences to email_reply_parser + +Most of mailstrip is a line-by-line port of email\_reply\_parser and it passes +all tests from the email\_reply\_parser test suite. However, it also implements +a few improvements that are not part of email\_reply\_parser: + +* Forwarded fragments are detected and considered to be visible text, see + [d321c1][3]. +* Replies from Yahoo! which lack ">" quote indicators are handled correctly, + see [e844d][4]. +* Alternative quote headers used by gmail are handled correctly, see + [7ecb6][5] +* Replies from Google inbox / gmail that has a quoute header in swedish(and possibly other languages) are handled. See [4128d][6]. + +## Documentation + +The API documentation can be found here: +http://godoc.org/github.com/kvannotten/mailstrip + +## License + +MIT License. See LICENSE file. + +[1]: https://github.com/github/email_reply_parser +[2]: http://golang.org/ +[3]: https://github.com/kvannotten/mailstrip/commit/d321c10543f77c0beaacb40b04511e619f0652c6 +[4]: https://github.com/kvannotten/mailstrip/commit/e844df52342787c3cf2e0ebb8850b16e35f7f437 +[5]: https://github.com/kvannotten/mailstrip/commit/7ecb608981016c5633575cb93abb00e4c7370bcf +[6]: https://github.com/kvannotten/mailstrip/commit/4128d1860b0b9477145ac4b4bbf14d1f072f7a4c diff --git a/vendor/github.com/kvannotten/mailstrip/mailstrip.go b/vendor/github.com/kvannotten/mailstrip/mailstrip.go new file mode 100644 index 0000000..e731a1c --- /dev/null +++ b/vendor/github.com/kvannotten/mailstrip/mailstrip.go @@ -0,0 +1,268 @@ +// mailstrip is a Go library that parses email text and strips it of +// signatures and reply quotes. It is a port of email_reply_parser, +// GitHub's library for parsing email replies. +// +// see https://github.com/github/email_reply_parser +package mailstrip + +import ( + "bufio" + "fmt" + "io" + "regexp" + "strings" + "unicode" +) + +// Parse parses a plaintext email and returns the results. +func Parse(text string) Email { + p := &parser{} + return p.Parse(text) +} + +type parser struct { + // This determines if any 'visible' Fragment has been found. Once any + // visible Fragment is found, stop looking for hidden ones. + foundVisible bool + // This instance variable points to the current Fragment. If the matched + // line fits, it should be added to this Fragment. Otherwise, finish it and + // start a new Fragment. + fragment *Fragment + // The fragments parsed so far + fragments []*Fragment +} + +// > I define UNIX as “30 definitions of regular expressions living under one +// > roof.” +// —Don Knuth +// +// Porting the Ruby regular expressions from email_reply_parser to Go required +// making the following changes: +// +// - Unlike most regexp flavors I'm familiar with, ^ and $ stand for beginning +// and end of line respectively in Ruby. Getting the same behavior in Go +// required enabling Go's multiline mode "(?m)" for these expressions. +// - Ruby's multiline mode "/m" is the same as Go's "(?s)" flag. Both are used +// to make "." match "\n" characters. +var ( + // used to join quote headers that were broken into multiple lines by the + // e-mail client. e.g. gmail does that for lines exceeding 80 chars + multiLineReplyHeaderRegexps = []*regexp.Regexp{ + // e.g. On Aug 22, 2011, at 7:37 PM, defunkt wrote: + regexp.MustCompile("(?sm)^(On\\s(?:.+)wrote:)$"), + // e.g. 2013/11/13 John Smith + regexp.MustCompile("(?sm)^(\\d{4}/\\d{1,2}/\\d{1,2} .*<.+@.+>)$"), + } + sigRegexp = regexp.MustCompile("(\\d+ swodniW rof >.*+)$") + quoteHeaderRegexp = regexp.MustCompile("(?m)^:etorw.*nO$|^.*[0-9]{4}\\s\\.\\w{2,4}\\s\\d{1,2}\\s.{3,4}$|^\\w{3,4}\\s\\d{1,2}\\s\\w{3,4}\\.\\s[0-9]{4}.*$|^>.*\\d{1,2}/\\d{1,2}/\\d{4}$|^(?m)^.*?[0-9]{4}\\s\\.\\w+\\s\\d\\s.*n\\.*$") +) + +func (p *parser) Parse(text string) Email { + // Normalize line endings. + text = strings.Replace(text, "\r\n", "\n", -1) + + // Check for multi-line reply headers. Some clients break up the "On DATE, + // NAME wrote:" line (and similar quote headers) into multiple lines. + for _, r := range multiLineReplyHeaderRegexps { + if m := r.FindStringSubmatch(text); len(m) == 2 { + // Remove all new lines from the reply header. + text = strings.Replace(text, m[1], strings.Replace(m[1], "\n", "", -1), -1) + } + } + + // The text is reversed initially due to the way we check for hidden + // fragments. + text = reverseString(text) + + // Use the Reader to pull out each line of the email content. + reader := bufio.NewReader(strings.NewReader(text)) + for { + line, e := reader.ReadBytes('\n') + p.scanLine(strings.TrimRight(string(line), "\n")) + if e == io.EOF { + break + } else if e != nil { + // Our underlaying reader is a strings.Reader, which will never return + // errors other than io.EOF, so this is merely a sanity check. + panic(fmt.Sprintf("Bug: ReadBytes returned an error other than io.EOF: %#v", e)) + } + } + + // Finish up the final fragment. Finishing a fragment will detect any + // attributes (hidden, signature, reply), and join each line into a + // string. + p.finishFragment() + + // Now that parsing is done, reverse the order. + reverseFragments(p.fragments) + return Email(p.fragments) +} + +// scaneLine scans the given line of text and figures out which fragment it +// belongs to. +func (p *parser) scanLine(line string) { + sigMatch := sigRegexp.MatchString(line) + + if !sigMatch { + line = strings.TrimLeftFunc(line, unicode.IsSpace) + } + + // We're looking for leading `>`'s to see if this line is part of a + // quoted Fragment. + isQuoted := quotedRegexp.MatchString(line) + + // Mark the current Fragment as a signature if the current line is empty + // and the Fragment starts with a common signature indicator. + if p.fragment != nil && line == "" { + // lastLine is really the first line, since the lines are still reversed + // at this point. + lastLine := p.fragment.lines[len(p.fragment.lines)-1] + if fwdRegexp.MatchString(lastLine) { + p.fragment.forwarded = true + p.finishFragment() + } else if sigRegexp.MatchString(lastLine) { + p.fragment.signature = true + p.finishFragment() + } + } + + isQuoteHeader := p.quoteHeader(line) + // Yahoo! does not use '>' quote indicator in replies, so if a quote header + // suddenly appears in an otherwise unquoted fragment, consider it quoted + // now. + if p.fragment != nil && isQuoteHeader { + p.fragment.quoted = true + } + + // If the line matches the current fragment, add it. Note that a common + // reply header also counts as part of the quoted Fragment, even though + // it doesn't start with `>`. + if p.fragment != nil && + ((p.fragment.quoted == isQuoted) || + (p.fragment.quoted && (isQuoteHeader || line == ""))) { + p.fragment.lines = append(p.fragment.lines, line) + + // Otherwise, finish the fragment and start a new one. + } else { + p.finishFragment() + p.fragment = &Fragment{quoted: isQuoted, lines: []string{line}} + } +} + +// quoteHeader detects if a given line is a header above a quoted area. It is +// only checked for lines preceding quoted regions. Returns true if the line is +// a valid header, or false. +func (p *parser) quoteHeader(line string) bool { + return quoteHeaderRegexp.MatchString(line) +} + +// finishFragment builds the fragment string and reverses it, after all lines +// have been added. It also checks to see if this Fragment is hidden. The +// hidden Fragment check reads from the bottom to the top. +// +// Any quoted Fragments or signature Fragments are marked hidden if they are +// below any visible Fragments. Visible Fragments are expected to contain +// original content by the author. If they are below a quoted Fragment, then +// the Fragment should be visible to give context to the reply. +// +// some original text (visible) +// +// > do you have any two's? (quoted, visible) +// +// Go fish! (visible) +// +// > -- > Player 1 (quoted, hidden) +// +// -- Player 2 (signature, hidden) +func (p *parser) finishFragment() { + if p.fragment != nil { + p.fragment.finish() + if !p.foundVisible { + if p.fragment.quoted || p.fragment.signature || + strings.TrimSpace(p.fragment.String()) == "" { + p.fragment.hidden = true + } else { + p.foundVisible = true + } + } + p.fragments = append(p.fragments, p.fragment) + } + p.fragment = nil +} + +func reverseString(s string) string { + runes := []rune(s) + for i, j := 0, len(runes)-1; i < j; i, j = i+1, j-1 { + runes[i], runes[j] = runes[j], runes[i] + } + return string(runes) +} + +func reverseFragments(f []*Fragment) { + for i, j := 0, len(f)-1; i < j; i, j = i+1, j-1 { + f[i], f[j] = f[j], f[i] + } +} + +// Email contains the parsed contents of an email. +type Email []*Fragment + +// String returns the non-Hidden() fragments of the Email. +func (e Email) String() string { + results := []string{} + for _, fragment := range e { + if fragment.Hidden() { + continue + } + + results = append(results, fragment.String()) + } + + result := strings.Join(results, "\n") + result = strings.TrimRightFunc(result, unicode.IsSpace) + return result +} + +// Fragment contains a parsed section of an email. +type Fragment struct { + lines []string + content string + hidden bool + signature bool + forwarded bool + quoted bool +} + +// finish builds the string content by joining the lines and reversing them. +func (f *Fragment) finish() { + f.content = strings.Join(f.lines, "\n") + f.lines = nil + f.content = reverseString(f.content) +} + +// Forwarded returns if the fragment is forwarded or not. +func (f *Fragment) Forwarded() bool { + return f.forwarded +} + +// Signature returns if the fragment is a signature or not. +func (f *Fragment) Signature() bool { + return f.signature +} + +// Signature returns if the fragment is a quote or not. +func (f *Fragment) Quoted() bool { + return f.quoted +} + +// Signature returns if the fragment is considered hidden or not. +func (f *Fragment) Hidden() bool { + return f.hidden +} + +// String returns the content of the fragment. +func (f *Fragment) String() string { + return f.content +} diff --git a/vendor/gitlab.com/etke.cc/go/psd/client.go b/vendor/gitlab.com/etke.cc/go/psd/client.go index 6828923..23f016f 100644 --- a/vendor/gitlab.com/etke.cc/go/psd/client.go +++ b/vendor/gitlab.com/etke.cc/go/psd/client.go @@ -7,9 +7,21 @@ import ( "io" "net/http" "net/url" + "runtime/debug" "time" ) +var version = func() string { + if info, ok := debug.ReadBuildInfo(); ok { + for _, setting := range info.Settings { + if setting.Key == "vcs.revision" { + return setting.Value + } + } + } + return "0.0.0-unknown" +}() + type Client struct { url *url.URL login string @@ -25,21 +37,23 @@ func NewClient(baseURL, login, password string) *Client { return &Client{url: uri, login: login, password: password} } -// Get returns the list of targets for the given identifier -func (p *Client) Get(identifier string) ([]*Target, error) { +// GetWithContext returns the list of targets for the given identifier using the given context +func (p *Client) GetWithContext(ctx context.Context, identifier string) ([]*Target, error) { if p.url == nil { return nil, nil } cloned := *p.url uri := cloned.JoinPath("/node/" + identifier) - ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + childCtx, cancel := context.WithTimeout(ctx, 10*time.Second) defer cancel() - req, err := http.NewRequestWithContext(ctx, http.MethodGet, uri.String(), http.NoBody) + + req, err := http.NewRequestWithContext(childCtx, http.MethodGet, uri.String(), http.NoBody) if err != nil { return nil, err } req.SetBasicAuth(p.login, p.password) + req.Header.Set("User-Agent", "Go-psd-client/"+version) resp, err := http.DefaultClient.Do(req) if err != nil { return nil, err @@ -61,3 +75,8 @@ func (p *Client) Get(identifier string) ([]*Target, error) { return psd, nil } + +// Get returns the list of targets for the given identifier +func (p *Client) Get(identifier string) ([]*Target, error) { + return p.GetWithContext(context.Background(), identifier) +} diff --git a/vendor/modules.txt b/vendor/modules.txt index bc59458..03ae1be 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -58,6 +58,9 @@ github.com/jhillyerd/enmime/internal/coding github.com/jhillyerd/enmime/internal/stringutil github.com/jhillyerd/enmime/internal/textproto github.com/jhillyerd/enmime/mediatype +# github.com/kvannotten/mailstrip v0.0.0-20200711213611-0002f5c0467e +## explicit; go 1.14 +github.com/kvannotten/mailstrip # github.com/lib/pq v1.10.9 ## explicit; go 1.13 github.com/lib/pq @@ -139,7 +142,7 @@ gitlab.com/etke.cc/go/healthchecks # gitlab.com/etke.cc/go/mxidwc v1.0.0 ## explicit; go 1.19 gitlab.com/etke.cc/go/mxidwc -# gitlab.com/etke.cc/go/psd v1.0.0 +# gitlab.com/etke.cc/go/psd v1.1.1 ## explicit; go 1.21.0 gitlab.com/etke.cc/go/psd # gitlab.com/etke.cc/go/secgen v1.2.0 @@ -178,7 +181,7 @@ golang.org/x/crypto/internal/poly1305 golang.org/x/crypto/pbkdf2 golang.org/x/crypto/ssh golang.org/x/crypto/ssh/internal/bcrypt_pbkdf -# golang.org/x/exp v0.0.0-20240213143201-ec583247a57a +# golang.org/x/exp v0.0.0-20240222234643-814bf88cf225 ## explicit; go 1.20 golang.org/x/exp/constraints golang.org/x/exp/maps