fix owner check

.gitlab-ci.yml

@@ -1,28 +1,24 @@
 stages:
   - test
   - release
-
-lint:
-  stage: test
+default:
   image: registry.gitlab.com/etke.cc/base/build
+variables:
+  PLATFORMS: linux/arm64/v8,linux/amd64
+
+test:
+  stage: test
   script:
     - just lint
-
-unit:
-  stage: test
-  image: registry.gitlab.com/etke.cc/base/build
-  script:
     - just test
+  cache:
+    key: ${CI_COMMIT_REF_SLUG}
+    paths:
+      - /root/cache/
 
 docker:
   stage: release
   only: ['main', 'tags']
-  services:
-    - docker:dind
-  image: ${CI_DEPENDENCY_PROXY_GROUP_IMAGE_PREFIX}/jdrouet/docker-with-buildx:latest
-  before_script:
-    - apk --no-cache add just
+  services: ['docker:dind']
   script:
     - just login docker
-  tags:
-    - docker

.golangci.yml

@@ -4,77 +4,123 @@ run:
   issues-exit-code: 1
   tests: true
   build-tags: []
-  skip-dirs: []
+  skip-dirs:
+    - mocks
   skip-dirs-use-default: true
   skip-files: []
   modules-download-mode: readonly
-  allow-parallel-runners: false
 
 output:
   format: colored-line-number
   print-issued-lines: true
   print-linter-name: true
-  uniq-by-line: true
-  path-prefix: ""
   sort-results: true
 
 linters-settings:
+  decorder:
+    dec-order:
+      - const
+      - var
+      - type
+      - func
+  dogsled:
+    max-blank-identifiers: 3
   errcheck:
+    check-type-assertions: true
     check-blank: true
+  errchkjson:
+    report-no-exported: true
+  exhaustive:
+    check:
+      - switch
+      - map
+    default-signifies-exhaustive: true
   gocognit:
     min-complexity: 15
   nestif:
-    min-complexity: 4
+    min-complexity: 5
   gocritic:
     enabled-tags:
+      - diagnostic
+      - style
       - performance
+  gofmt:
+    simplify: true
+    rewrite-rules:
+      - pattern: 'interface{}'
+        replacement: 'any'
+      - pattern: 'a[b:len(a)]'
+        replacement: 'a[b:]'
   gofumpt:
-    lang-version: "1.18"
-  gosimple:
-    go: "1.18"
-    checks: [ "all" ]
+    extra-rules: true
   govet:
     check-shadowing: true
-    enable:
-      - atomicalign
-      - shadow
+  grouper:
+    const-require-single-const: true
+    import-require-single-import: true
+    var-require-single-var: true
   misspell:
     locale: US
-  staticcheck:
-    go: "1.18"
-    checks: [ "all" ]
-  stylecheck:
-    go: "1.18"
+  usestdlibvars:
+    time-month: true
+    time-layout: true
+    crypto-hash: true
+    default-rpc-path: true
+    os-dev-null: true
+    sql-isolation-level: true
+    tls-signature-scheme: true
+    constant-kind: true
   unparam:
     check-exported: true
-  unused:
-    go: "1.18"
-  gci:
-    sections:
-      - standard
-      - default
-      - prefix(gitlab.com/etke.cc/postmoogle)
-    section-separators:
-      - newLine
 linters:
   disable-all: false
   enable:
-    - megacheck
-    - govet
+    - asasalint
+    - asciicheck
+    - bidichk
+    - bodyclose
+    - containedctx
+    - decorder
+    - dogsled
+    - dupl
+    - dupword
+    - durationcheck
     - errcheck
-    - gci
+    - errchkjson
+    - errname
+    - errorlint
+    - execinquery
+    - exhaustive
+    - exportloopref
+    - forcetypeassert
+    - gochecknoinits
     - gocognit
-    - nestif
-      # - gocritic # ref: https://github.com/golangci/golangci-lint/issues/2649#issue-1170906525
+    - gocritic
+    - gocyclo
+    - goerr113
+    - gofmt
     - gofumpt
     - goimports
+    - gosec
     - gosimple
+    - gosmopolitan
     - govet
+    - ineffassign
+    - makezero
+    - mirror
     - misspell
+    - nestif
+    - nolintlint
+    - prealloc
+    - predeclared
+    - revive
+    - sqlclosecheck
     - staticcheck
-    - stylecheck
+    - unconvert
     - unparam
     - unused
+    - usestdlibvars
+    - wastedassign
   fast: false
 
 

README.md

@@ -83,7 +83,7 @@ You can find default values in [config/defaults.go](config/defaults.go)
 
 </details>
 
-### 2. DNS (optional)
+### 2. DNS (highly recommended)
 
 Follow the [docs/dns](docs/dns.md)
 
@@ -125,6 +125,7 @@ If you want to change them - check available options in the help message (`!pm h
 
 * **`!pm autoreply`** - Get or set autoreply of the room (markdown supported) that will be sent on any new incoming email thread
 * **`!pm signature`** - Get or set signature of the room (markdown supported)
+* **`!pm threadify`** - Get or set `threadify` of the room (`true` - send incoming email body in thread; `false` - send incoming email body as part of the message)
 * **`!pm nosend`** - Get or set `nosend` of the room (`true` - disable email sending; `false` - enable email sending)
 * **`!pm noreplies`** - Get or set `noreplies` of the room (`true` - ignore matrix replies; `false` - parse matrix replies)
 * **`!pm nosender`** - Get or set `nosender` of the room (`true` - hide email sender; `false` - show email sender)

bot/access.go

@@ -22,22 +22,32 @@ func parseMXIDpatterns(patterns []string, defaultPattern string) ([]*regexp.Rege
 	return mxidwc.ParsePatterns(patterns)
 }
 
-func (b *Bot) allowUsers(actorID id.UserID) bool {
-	if len(b.allowedUsers) != 0 {
-		if !mxidwc.Match(actorID.String(), b.allowedUsers) {
-			return false
-		}
+func (b *Bot) allowUsers(actorID id.UserID, targetRoomID id.RoomID) bool {
+	// first, check if it's an allowed user
+	if mxidwc.Match(actorID.String(), b.allowedUsers) {
+		return true
 	}
 
-	return true
+	// second, check if it's an admin (admin may not fit the allowed users pattern)
+	if b.allowAdmin(actorID, targetRoomID) {
+		return true
+	}
+
+	// then, check if it's the owner (same as above)
+	cfg, err := b.cfg.GetRoom(targetRoomID)
+	if err == nil && cfg.Owner() == actorID.String() {
+		return true
+	}
+
+	return false
 }
 
-func (b *Bot) allowAnyone(actorID id.UserID, targetRoomID id.RoomID) bool {
+func (b *Bot) allowAnyone(_ id.UserID, _ id.RoomID) bool {
 	return true
 }
 
 func (b *Bot) allowOwner(actorID id.UserID, targetRoomID id.RoomID) bool {
-	if !b.allowUsers(actorID) {
+	if !b.allowUsers(actorID, targetRoomID) {
 		return false
 	}
 	cfg, err := b.cfg.GetRoom(targetRoomID)
@@ -51,15 +61,15 @@ func (b *Bot) allowOwner(actorID id.UserID, targetRoomID id.RoomID) bool {
 		return true
 	}
 
-	return owner == actorID.String()
+	return owner == actorID.String() || b.allowAdmin(actorID, targetRoomID)
 }
 
-func (b *Bot) allowAdmin(actorID id.UserID, targetRoomID id.RoomID) bool {
+func (b *Bot) allowAdmin(actorID id.UserID, _ id.RoomID) bool {
 	return mxidwc.Match(actorID.String(), b.allowedAdmins)
 }
 
 func (b *Bot) allowSend(actorID id.UserID, targetRoomID id.RoomID) bool {
-	if !b.allowUsers(actorID) {
+	if !b.allowUsers(actorID, targetRoomID) {
 		return false
 	}
 
@@ -73,7 +83,7 @@ func (b *Bot) allowSend(actorID id.UserID, targetRoomID id.RoomID) bool {
 }
 
 func (b *Bot) allowReply(actorID id.UserID, targetRoomID id.RoomID) bool {
-	if !b.allowUsers(actorID) {
+	if !b.allowUsers(actorID, targetRoomID) {
 		return false
 	}
 

bot/bot.go

@@ -91,14 +91,14 @@ func New(
 }
 
 // Error message to the log and matrix room
-func (b *Bot) Error(ctx context.Context, message string, args ...interface{}) {
+func (b *Bot) Error(ctx context.Context, message string, args ...any) {
 	evt := eventFromContext(ctx)
 	threadID := threadIDFromContext(ctx)
 	if threadID == "" {
 		threadID = linkpearl.EventParent(evt.ID, evt.Content.AsMessage())
 	}
 
-	err := fmt.Errorf(message, args...)
+	err := fmt.Errorf(message, args...) //nolint:goerr113 // we have to
 	b.log.Error().Err(err).Msg(err.Error())
 	if evt == nil {
 		return

bot/command.go

@@ -2,6 +2,7 @@ package bot
 
 import (
 	"context"
+	"errors"
 	"fmt"
 	"strings"
 	"time"
@@ -115,6 +116,15 @@ func (b *Bot) initCommands() commandList {
 			sanitizer:   func(s string) string { return s },
 			allowed:     b.allowOwner,
 		},
+		{
+			key: config.RoomThreadify,
+			description: fmt.Sprintf(
+				"Get or set `%s` of the room (`true` - send incoming email body in thread; `false` - send incoming email body as part of the message)",
+				config.RoomThreadify,
+			),
+			sanitizer: utils.SanitizeBoolString,
+			allowed:   b.allowOwner,
+		},
 		{
 			key: config.RoomNoSend,
 			description: fmt.Sprintf(
@@ -372,7 +382,7 @@ func (b *Bot) handle(ctx context.Context) {
 	if err != nil {
 		b.log.Error().Err(err).Msg("cannot send typing notification")
 	}
-	defer b.lp.GetClient().UserTyping(evt.RoomID, false, 30*time.Second) //nolint:errcheck
+	defer b.lp.GetClient().UserTyping(evt.RoomID, false, 30*time.Second) //nolint:errcheck // we don't care
 
 	if !cmd.allowed(evt.Sender, evt.RoomID) {
 		b.lp.SendNotice(evt.RoomID, "not allowed to do that, kupo")
@@ -413,9 +423,9 @@ func (b *Bot) handle(ctx context.Context) {
 	case commandBanlistTotals:
 		b.runBanlistTotals(ctx)
 	case commandBanlistAdd:
-		b.runBanlistAdd(ctx, commandSlice)
+		b.runBanlistChange(ctx, "add", commandSlice)
 	case commandBanlistRemove:
-		b.runBanlistRemove(ctx, commandSlice)
+		b.runBanlistChange(ctx, "remove", commandSlice)
 	case commandBanlistReset:
 		b.runBanlistReset(ctx)
 	case commandMailboxes:
@@ -543,7 +553,7 @@ func (b *Bot) runSend(ctx context.Context) {
 	b.runSendCommand(ctx, cfg, tos, subject, body, htmlBody)
 }
 
-func (b *Bot) getSendDetails(ctx context.Context) (string, string, string, bool) {
+func (b *Bot) getSendDetails(ctx context.Context) (to, subject, body string, ok bool) {
 	evt := eventFromContext(ctx)
 	if !b.allowSend(evt.Sender, evt.RoomID) {
 		return "", "", "", false
@@ -556,8 +566,8 @@ func (b *Bot) getSendDetails(ctx context.Context) (string, string, string, bool)
 	}
 
 	commandSlice := b.parseCommand(evt.Content.AsMessage().Body, false)
-	to, subject, body, err := utils.ParseSend(commandSlice)
-	if err == utils.ErrInvalidArgs {
+	to, subject, body, err = utils.ParseSend(commandSlice)
+	if errors.Is(err, utils.ErrInvalidArgs) {
 		b.lp.SendNotice(evt.RoomID, fmt.Sprintf(
 			"Usage:\n"+
 				"```\n"+
@@ -598,8 +608,8 @@ func (b *Bot) runSendCommand(ctx context.Context, cfg config.Room, tos []string,
 		}
 	}
 
-	b.mu.Lock(evt.RoomID.String())
-	defer b.mu.Unlock(evt.RoomID.String())
+	b.lock(evt.RoomID, evt.ID)
+	defer b.unlock(evt.RoomID, evt.ID)
 
 	domain := utils.SanitizeDomain(cfg.Domain())
 	from := cfg.Mailbox() + "@" + domain

bot/command_admin.go

@@ -21,7 +21,7 @@ func (b *Bot) sendMailboxes(ctx context.Context) {
 	evt := eventFromContext(ctx)
 	mailboxes := map[string]config.Room{}
 	slice := []string{}
-	b.rooms.Range(func(key any, value any) bool {
+	b.rooms.Range(func(key, value any) bool {
 		if key == nil {
 			return true
 		}
@@ -37,12 +37,12 @@ func (b *Bot) sendMailboxes(ctx context.Context) {
 		if !ok {
 			return true
 		}
-		config, err := b.cfg.GetRoom(roomID)
+		cfg, err := b.cfg.GetRoom(roomID)
 		if err != nil {
 			b.log.Error().Err(err).Msg("cannot retrieve settings")
 		}
 
-		mailboxes[mailbox] = config
+		mailboxes[mailbox] = cfg
 		slice = append(slice, mailbox)
 		return true
 	})
@@ -80,7 +80,10 @@ func (b *Bot) runDelete(ctx context.Context, commandSlice []string) {
 		b.lp.SendNotice(evt.RoomID, "mailbox does not exists, kupo", linkpearl.RelatesTo(evt.ID))
 		return
 	}
-	roomID := v.(id.RoomID)
+	roomID, ok := v.(id.RoomID)
+	if !ok {
+		return
+	}
 
 	b.rooms.Delete(mailbox)
 	err := b.cfg.SetRoom(roomID, config.Room{})
@@ -350,7 +353,7 @@ func (b *Bot) runBanlistTotals(ctx context.Context) {
 	b.addBanlistTimeline(ctx, true)
 }
 
-func (b *Bot) runBanlistAuth(ctx context.Context, commandSlice []string) {
+func (b *Bot) runBanlistAuth(ctx context.Context, commandSlice []string) { //nolint:dupl // not in that case
 	evt := eventFromContext(ctx)
 	cfg := b.cfg.GetBot()
 	if len(commandSlice) < 2 {
@@ -377,7 +380,7 @@ func (b *Bot) runBanlistAuth(ctx context.Context, commandSlice []string) {
 	b.lp.SendNotice(evt.RoomID, "auth banning has been updated", linkpearl.RelatesTo(evt.ID))
 }
 
-func (b *Bot) runBanlistAuto(ctx context.Context, commandSlice []string) {
+func (b *Bot) runBanlistAuto(ctx context.Context, commandSlice []string) { //nolint:dupl // not in that case
 	evt := eventFromContext(ctx)
 	cfg := b.cfg.GetBot()
 	if len(commandSlice) < 2 {
@@ -404,7 +407,7 @@ func (b *Bot) runBanlistAuto(ctx context.Context, commandSlice []string) {
 	b.lp.SendNotice(evt.RoomID, "auto banning has been updated", linkpearl.RelatesTo(evt.ID))
 }
 
-func (b *Bot) runBanlistAdd(ctx context.Context, commandSlice []string) {
+func (b *Bot) runBanlistChange(ctx context.Context, mode string, commandSlice []string) {
 	evt := eventFromContext(ctx)
 	if len(commandSlice) < 2 {
 		b.runBanlist(ctx, commandSlice)
@@ -416,37 +419,13 @@ func (b *Bot) runBanlistAdd(ctx context.Context, commandSlice []string) {
 	}
 	banlist := b.cfg.GetBanlist()
 
-	ips := commandSlice[1:]
-	for _, ip := range ips {
-		addr, err := net.ResolveIPAddr("ip", ip)
-		if err != nil {
-			b.Error(ctx, "cannot add %s to banlist: %v", ip, err)
-			return
-		}
-		banlist.Add(addr)
+	var action func(net.Addr)
+	if mode == "remove" {
+		action = banlist.Remove
+	} else {
+		action = banlist.Add
 	}
 
-	err := b.cfg.SetBanlist(banlist)
-	if err != nil {
-		b.Error(ctx, "cannot set banlist: %v", err)
-		return
-	}
-
-	b.lp.SendNotice(evt.RoomID, "banlist has been updated, kupo", linkpearl.RelatesTo(evt.ID))
-}
-
-func (b *Bot) runBanlistRemove(ctx context.Context, commandSlice []string) {
-	evt := eventFromContext(ctx)
-	if len(commandSlice) < 2 {
-		b.runBanlist(ctx, commandSlice)
-		return
-	}
-	if !b.cfg.GetBot().BanlistEnabled() {
-		b.lp.SendNotice(evt.RoomID, "banlist is disabled, you have to enable it first, kupo", linkpearl.RelatesTo(evt.ID))
-		return
-	}
-	banlist := b.cfg.GetBanlist()
-
 	ips := commandSlice[1:]
 	for _, ip := range ips {
 		addr, err := net.ResolveIPAddr("ip", ip)
@@ -454,7 +433,7 @@ func (b *Bot) runBanlistRemove(ctx context.Context, commandSlice []string) {
 			b.Error(ctx, "cannot remove %s from banlist: %v", ip, err)
 			return
 		}
-		banlist.Remove(addr)
+		action(addr)
 	}
 
 	err := b.cfg.SetBanlist(banlist)

bot/command_owner.go

@@ -81,7 +81,7 @@ func (b *Bot) getOption(ctx context.Context, name string) {
 		value = utils.EmailsList(value, cfg.Domain())
 	}
 
-	msg := fmt.Sprintf("`%s` of this room is `%s`\n"+
+	msg := fmt.Sprintf("`%s` of this room is:\n```\n%s\n```\n"+
 		"To set it to a new value, send a `%s %s VALUE` command.",
 		name, value, b.prefix, name)
 	if name == config.RoomPassword {
@@ -187,7 +187,7 @@ func (b *Bot) setOption(ctx context.Context, name, value string) {
 		return
 	}
 
-	msg := fmt.Sprintf("`%s` of this room set to `%s`", name, value)
+	msg := fmt.Sprintf("`%s` of this room set to:\n```\n%s\n```", name, value)
 	b.lp.SendNotice(evt.RoomID, msg, linkpearl.RelatesTo(evt.ID, cfg.NoThreads()))
 }
 

bot/config/manager.go

@@ -50,6 +50,9 @@ func (m *Manager) SetBot(cfg Bot) error {
 // GetRoom config
 func (m *Manager) GetRoom(roomID id.RoomID) (Room, error) {
 	config, err := m.lp.GetRoomAccountData(roomID, acRoomKey)
+	if err != nil {
+		m.log.Warn().Err(err).Str("room_id", roomID.String()).Msg("cannot get room settings")
+	}
 	if config == nil {
 		config = make(Room, 0)
 	}

bot/config/room.go

@@ -22,6 +22,7 @@ const (
 	RoomSignature = "signature"
 	RoomAutoreply = "autoreply"
 
+	RoomThreadify   = "threadify"
 	RoomNoCC        = "nocc"
 	RoomNoFiles     = "nofiles"
 	RoomNoHTML      = "nohtml"
@@ -79,6 +80,10 @@ func (s Room) Autoreply() string {
 	return s.Get(RoomAutoreply)
 }
 
+func (s Room) Threadify() bool {
+	return utils.Bool(s.Get(RoomThreadify))
+}
+
 func (s Room) NoSend() bool {
 	return utils.Bool(s.Get(RoomNoSend))
 }
@@ -193,6 +198,7 @@ func (s Room) ContentOptions() *email.ContentOptions {
 		Recipient: !s.NoRecipient(),
 		Subject:   !s.NoSubject(),
 		Threads:   !s.NoThreads(),
+		Threadify: s.Threadify(),
 
 		ToKey:         "cc.etke.postmoogle.to",
 		CcKey:         "cc.etke.postmoogle.cc",

bot/email.go

@@ -3,7 +3,9 @@ package bot
 import (
 	"context"
 	"errors"
+	"fmt"
 	"strings"
+	"time"
 
 	"gitlab.com/etke.cc/linkpearl"
 	"maunium.net/go/mautrix/event"
@@ -15,14 +17,12 @@ import (
 	"gitlab.com/etke.cc/postmoogle/utils"
 )
 
-// account data keys
 const (
+	// account data keys
 	acMessagePrefix   = "cc.etke.postmoogle.message"
 	acLastEventPrefix = "cc.etke.postmoogle.last"
-)
 
-// event keys
-const (
+	// event keys
 	eventMessageIDkey  = "cc.etke.postmoogle.messageID"
 	eventReferencesKey = "cc.etke.postmoogle.references"
 	eventInReplyToKey  = "cc.etke.postmoogle.inReplyTo"
@@ -33,6 +33,8 @@ const (
 	eventCcKey         = "cc.etke.postmoogle.cc"
 )
 
+var ErrNoRoom = errors.New("room not found")
+
 // SetSendmail sets mail sending func to the bot
 func (b *Bot) SetSendmail(sendmail func(string, string, string) error) {
 	b.sendmail = sendmail
@@ -40,31 +42,38 @@ func (b *Bot) SetSendmail(sendmail func(string, string, string) error) {
 }
 
 func (b *Bot) shouldQueue(msg string) bool {
-	errors := strings.Split(msg, ";")
-	for _, err := range errors {
-		errParts := strings.Split(strings.TrimSpace(err), ":")
-		if len(errParts) < 2 {
-			continue
-		}
-		if strings.HasPrefix(strings.TrimSpace(errParts[1]), "4") {
-			return true
-		}
+	msg = strings.TrimSpace(msg)
+	if strings.HasPrefix(msg, "4") { // any temporary issue (4xx SMTP code)
+		return true
 	}
+
+	if strings.Contains(msg, "450") || strings.Contains(msg, "451") { // greylisting
+		return true
+	}
+
+	if strings.Contains(msg, "greylisted") { // greylisting
+		return true
+	}
+
 	return false
 }
 
 // Sendmail tries to send email immediately, but if it gets 4xx error (greylisting),
 // the email will be added to the queue and retried several times after that
 func (b *Bot) Sendmail(eventID id.EventID, from, to, data string) (bool, error) {
+	log := b.log.With().Str("from", from).Str("to", to).Str("eventID", eventID.String()).Logger()
+	log.Info().Msg("attempting to deliver email")
 	err := b.sendmail(from, to, data)
 	if err != nil {
 		if b.shouldQueue(err.Error()) {
-			b.log.Info().Err(err).Str("id", eventID.String()).Str("from", from).Str("to", to).Msg("email has been added to the queue")
+			log.Info().Err(err).Msg("email has been added to the queue")
 			return true, b.q.Add(eventID.String(), from, to, data)
 		}
+		log.Warn().Err(err).Msg("email delivery failed")
 		return false, err
 	}
 
+	log.Info().Msg("email delivery succeeded")
 	return false, nil
 }
 
@@ -114,10 +123,10 @@ func (b *Bot) GetIFOptions(roomID id.RoomID) email.IncomingFilteringOptions {
 // IncomingEmail sends incoming email to matrix room
 //
 //nolint:gocognit // TODO
-func (b *Bot) IncomingEmail(ctx context.Context, email *email.Email) error {
-	roomID, ok := b.GetMapping(email.Mailbox(true))
+func (b *Bot) IncomingEmail(ctx context.Context, eml *email.Email) error {
+	roomID, ok := b.GetMapping(eml.Mailbox(true))
 	if !ok {
-		return errors.New("room not found")
+		return ErrNoRoom
 	}
 	cfg, err := b.cfg.GetRoom(roomID)
 	if err != nil {
@@ -129,18 +138,18 @@ func (b *Bot) IncomingEmail(ctx context.Context, email *email.Email) error {
 
 	var threadID id.EventID
 	newThread := true
-	if email.InReplyTo != "" || email.References != "" {
-		threadID = b.getThreadID(roomID, email.InReplyTo, email.References)
+	if eml.InReplyTo != "" || eml.References != "" {
+		threadID = b.getThreadID(roomID, eml.InReplyTo, eml.References)
 		if threadID != "" {
 			newThread = false
 			ctx = threadIDToContext(ctx, threadID)
-			b.setThreadID(roomID, email.MessageID, threadID)
+			b.setThreadID(roomID, eml.MessageID, threadID)
 		}
 	}
-	content := email.Content(threadID, cfg.ContentOptions())
+	content := eml.Content(threadID, cfg.ContentOptions())
 	eventID, serr := b.lp.Send(roomID, content)
 	if serr != nil {
-		if !strings.Contains(serr.Error(), "M_UNKNOWN") { // 	if it's not an unknown event event error
+		if !strings.Contains(serr.Error(), "M_UNKNOWN") { // if it's not an unknown event error
 			return serr
 		}
 		threadID = "" // unknown event edge case - remove existing thread ID to avoid complications
@@ -151,15 +160,22 @@ func (b *Bot) IncomingEmail(ctx context.Context, email *email.Email) error {
 		ctx = threadIDToContext(ctx, threadID)
 	}
 
-	b.setThreadID(roomID, email.MessageID, threadID)
+	b.setThreadID(roomID, eml.MessageID, threadID)
 	b.setLastEventID(roomID, threadID, eventID)
 
+	if newThread && cfg.Threadify() {
+		_, berr := b.lp.Send(roomID, eml.ContentBody(threadID, cfg.ContentOptions()))
+		if berr != nil {
+			return berr
+		}
+	}
+
 	if !cfg.NoInlines() {
-		b.sendFiles(ctx, roomID, email.InlineFiles, cfg.NoThreads(), threadID)
+		b.sendFiles(ctx, roomID, eml.InlineFiles, cfg.NoThreads(), threadID)
 	}
 
 	if !cfg.NoFiles() {
-		b.sendFiles(ctx, roomID, email.Files, cfg.NoThreads(), threadID)
+		b.sendFiles(ctx, roomID, eml.Files, cfg.NoThreads(), threadID)
 	}
 
 	if newThread && cfg.Autoreply() != "" {
@@ -188,7 +204,7 @@ func (b *Bot) sendAutoreply(roomID id.RoomID, threadID id.EventID) {
 	}
 
 	evt := &event.Event{
-		ID:     threadID + "-autoreply",
+		ID:     id.EventID(fmt.Sprintf("%s-autoreply-%s", threadID, time.Now().UTC().Format("20060102T150405Z"))),
 		RoomID: roomID,
 		Content: event.Content{
 			Parsed: &event.MessageEventContent{
@@ -242,21 +258,22 @@ func (b *Bot) sendAutoreply(roomID id.RoomID, threadID id.EventID) {
 		queued, err = b.Sendmail(evt.ID, meta.From, to, data)
 		if queued {
 			b.log.Info().Err(err).Str("from", meta.From).Str("to", to).Msg("email has been queued")
-			b.saveSentMetadata(ctx, queued, meta.ThreadID, recipients, eml, cfg, "Autoreply has been sent (queued)")
+			b.saveSentMetadata(ctx, queued, meta.ThreadID, recipients, eml, cfg, "Autoreply has been sent to "+to+" (queued)")
 			continue
 		}
 
 		if err != nil {
-			b.Error(ctx, "cannot send email: %v", err)
+			b.Error(ctx, "cannot send email to %q: %v", to, err)
 			continue
 		}
-	}
 
-	b.saveSentMetadata(ctx, queued, meta.ThreadID, recipients, eml, cfg, "Autoreply has been sent")
+		b.saveSentMetadata(ctx, queued, meta.ThreadID, recipients, eml, cfg, "Autoreply has been sent to "+to)
+	}
 }
 
-func (b *Bot) canReply(sender id.UserID, roomID id.RoomID) bool {
-	return b.allowSend(sender, roomID) && b.allowReply(sender, roomID)
+func (b *Bot) canReply(ctx context.Context) bool {
+	evt := eventFromContext(ctx)
+	return b.allowSend(evt.Sender, evt.RoomID) && b.allowReply(evt.Sender, evt.RoomID)
 }
 
 // SendEmailReply sends replies from matrix thread to email thread
@@ -264,7 +281,7 @@ func (b *Bot) canReply(sender id.UserID, roomID id.RoomID) bool {
 //nolint:gocognit // TODO
 func (b *Bot) SendEmailReply(ctx context.Context) {
 	evt := eventFromContext(ctx)
-	if !b.canReply(evt.Sender, evt.RoomID) {
+	if !b.canReply(ctx) {
 		return
 	}
 	cfg, err := b.cfg.GetRoom(evt.RoomID)
@@ -278,8 +295,8 @@ func (b *Bot) SendEmailReply(ctx context.Context) {
 		return
 	}
 
-	b.mu.Lock(evt.RoomID.String())
-	defer b.mu.Unlock(evt.RoomID.String())
+	b.lock(evt.RoomID, evt.ID)
+	defer b.unlock(evt.RoomID, evt.ID)
 
 	meta := b.getParentEmail(evt, mailbox)
 
@@ -330,12 +347,12 @@ func (b *Bot) SendEmailReply(ctx context.Context) {
 		}
 
 		if err != nil {
-			b.Error(ctx, "cannot send email: %v", err)
+			b.Error(ctx, "cannot send email to %q: %v", to, err)
 			continue
 		}
-	}
 
-	b.saveSentMetadata(ctx, queued, meta.ThreadID, recipients, eml, cfg)
+		b.saveSentMetadata(ctx, queued, meta.ThreadID, recipients, eml, cfg)
+	}
 }
 
 type parentEmail struct {
@@ -405,7 +422,7 @@ func (e *parentEmail) fixtofrom(newSenderMailbox string, domains []string) strin
 
 func (e *parentEmail) calculateRecipients(from string, forwardedFrom []string) {
 	recipients := map[string]struct{}{}
-	recipients[e.From] = struct{}{}
+	recipients[email.Address(e.From)] = struct{}{}
 
 	for _, addr := range strings.Split(email.Address(e.To), ",") {
 		recipients[addr] = struct{}{}
@@ -421,7 +438,7 @@ func (e *parentEmail) calculateRecipients(from string, forwardedFrom []string) {
 
 	rcpts := make([]string, 0, len(recipients))
 	for rcpt := range recipients {
-		rcpts = append(rcpts, rcpt)
+		rcpts = append(rcpts, email.Address(rcpt))
 	}
 
 	e.Recipients = rcpts
@@ -445,7 +462,7 @@ func (b *Bot) getParentEvent(evt *event.Event) (id.EventID, *event.Event) {
 		b.log.Error().Err(err).Msg("cannot get parent event")
 		return threadID, nil
 	}
-	linkpearl.ParseContent(parentEvt, parentEvt.Type, b.log)
+	linkpearl.ParseContent(parentEvt, b.log)
 
 	if !b.lp.GetMachine().StateStore.IsEncrypted(evt.RoomID) {
 		return threadID, parentEvt
@@ -471,10 +488,10 @@ func (b *Bot) getParentEmail(evt *event.Event, newFromMailbox string) *parentEma
 		return parent
 	}
 
-	parent.From = linkpearl.EventField[string](&parentEvt.Content, eventFromKey)
-	parent.To = linkpearl.EventField[string](&parentEvt.Content, eventToKey)
-	parent.CC = linkpearl.EventField[string](&parentEvt.Content, eventCcKey)
-	parent.RcptTo = linkpearl.EventField[string](&parentEvt.Content, eventRcptToKey)
+	parent.From = email.Address(linkpearl.EventField[string](&parentEvt.Content, eventFromKey))
+	parent.To = email.Address(linkpearl.EventField[string](&parentEvt.Content, eventToKey))
+	parent.CC = email.Address(linkpearl.EventField[string](&parentEvt.Content, eventCcKey))
+	parent.RcptTo = email.Address(linkpearl.EventField[string](&parentEvt.Content, eventRcptToKey))
 	parent.InReplyTo = linkpearl.EventField[string](&parentEvt.Content, eventMessageIDkey)
 	parent.References = linkpearl.EventField[string](&parentEvt.Content, eventReferencesKey)
 	senderEmail := parent.fixtofrom(newFromMailbox, b.domains)
@@ -543,7 +560,7 @@ func (b *Bot) sendFiles(ctx context.Context, roomID id.RoomID, files []*utils.Fi
 	}
 }
 
-func (b *Bot) getThreadID(roomID id.RoomID, messageID string, references string) id.EventID {
+func (b *Bot) getThreadID(roomID id.RoomID, messageID, references string) id.EventID {
 	refs := []string{messageID}
 	if references != "" {
 		refs = append(refs, strings.Split(references, " ")...)
@@ -592,7 +609,7 @@ func (b *Bot) getLastEventID(roomID id.RoomID, threadID id.EventID) id.EventID {
 	return threadID
 }
 
-func (b *Bot) setLastEventID(roomID id.RoomID, threadID id.EventID, eventID id.EventID) {
+func (b *Bot) setLastEventID(roomID id.RoomID, threadID, eventID id.EventID) {
 	key := acLastEventPrefix + "." + threadID.String()
 	err := b.lp.SetRoomAccountData(roomID, key, map[string]string{"eventID": eventID.String()})
 	if err != nil {

bot/mutex.go

@@ -0,0 +1,29 @@
+package bot
+
+import (
+	"maunium.net/go/mautrix/id"
+)
+
+func (b *Bot) lock(roomID id.RoomID, optionalEventID ...id.EventID) {
+	b.mu.Lock(roomID.String())
+
+	if len(optionalEventID) == 0 {
+		return
+	}
+	evtID := optionalEventID[0]
+	if _, err := b.lp.GetClient().SendReaction(roomID, evtID, "📨"); err != nil {
+		b.log.Error().Err(err).Str("roomID", roomID.String()).Str("eventID", evtID.String()).Msg("cannot send reaction on lock")
+	}
+}
+
+func (b *Bot) unlock(roomID id.RoomID, optionalEventID ...id.EventID) {
+	b.mu.Unlock(roomID.String())
+
+	if len(optionalEventID) == 0 {
+		return
+	}
+	evtID := optionalEventID[0]
+	if _, err := b.lp.GetClient().SendReaction(roomID, evtID, "✅"); err != nil {
+		b.log.Error().Err(err).Str("roomID", roomID.String()).Str("eventID", evtID.String()).Msg("cannot send reaction on unlock")
+	}
+}

bot/reaction.go

@@ -4,7 +4,6 @@ import (
 	"context"
 
 	"gitlab.com/etke.cc/linkpearl"
-	"maunium.net/go/mautrix/event"
 )
 
 var supportedReactions = map[string]string{
@@ -28,12 +27,18 @@ func (b *Bot) handleReaction(ctx context.Context) {
 		b.Error(ctx, "cannot find event %s: %v", srcID, err)
 		return
 	}
+	linkpearl.ParseContent(srcEvt, b.log)
+	if b.lp.GetMachine().StateStore.IsEncrypted(evt.RoomID) {
+		decrypted, derr := b.lp.GetClient().Crypto.Decrypt(srcEvt)
+		if derr == nil {
+			srcEvt = decrypted
+		}
+	}
 	threadID := linkpearl.EventParent(srcID, srcEvt.Content.AsMessage())
 	ctx = threadIDToContext(ctx, threadID)
-	linkpearl.ParseContent(evt, event.EventMessage, b.log)
+	linkpearl.ParseContent(evt, b.log)
 
-	switch action {
-	case commandSpamlistAdd:
+	if action == commandSpamlistAdd {
 		sender := linkpearl.EventField[string](&srcEvt.Content, eventFromKey)
 		if sender == "" {
 			b.Error(ctx, "cannot get sender of the email")

bot/sync.go

@@ -3,7 +3,6 @@ package bot
 import (
 	"context"
 
-	"gitlab.com/etke.cc/go/mxidwc"
 	"maunium.net/go/mautrix"
 	"maunium.net/go/mautrix/event"
 )
@@ -33,7 +32,7 @@ func (b *Bot) initSync() {
 
 // joinPermit is called by linkpearl when processing "invite" events and deciding if rooms should be auto-joined or not
 func (b *Bot) joinPermit(evt *event.Event) bool {
-	if !mxidwc.Match(evt.Sender.String(), b.allowedUsers) {
+	if !b.allowUsers(evt.Sender, evt.RoomID) {
 		b.log.Debug().Str("userID", evt.Sender.String()).Msg("Rejecting room invitation from unallowed user")
 		return false
 	}

cmd/cmd.go

@@ -111,7 +111,6 @@ func initMatrix(cfg *config.Config) {
 		Logger:            log,
 	})
 	if err != nil {
-		// nolint // Fatal = panic, not os.Exit()
 		log.Fatal().Err(err).Msg("cannot initialize matrix bot")
 	}
 

email/email.go

@@ -132,8 +132,15 @@ func (e *Email) contentHeader(threadID id.EventID, text *strings.Builder, option
 		text.WriteString("\n\n")
 	}
 	if options.Subject && threadID == "" {
-		text.WriteString("# ")
-		text.WriteString(e.Subject)
+		if options.Threadify {
+			text.WriteString("**")
+			text.WriteString(e.Subject)
+			text.WriteString("**")
+		} else {
+			text.WriteString("# ")
+			text.WriteString(e.Subject)
+
+		}
 		text.WriteString("\n\n")
 	}
 }
@@ -144,10 +151,12 @@ func (e *Email) Content(threadID id.EventID, options *ContentOptions) *event.Con
 
 	e.contentHeader(threadID, &text, options)
 
-	if e.HTML != "" && options.HTML {
-		text.WriteString(format.HTMLToMarkdown(e.HTML))
-	} else {
-		text.WriteString(e.Text)
+	if threadID != "" || (threadID == "" && !options.Threadify) {
+		if e.HTML != "" && options.HTML {
+			text.WriteString(format.HTMLToMarkdown(e.HTML))
+		} else {
+			text.WriteString(e.Text)
+		}
 	}
 
 	parsed := format.RenderMarkdown(text.String(), true, true)
@@ -159,7 +168,7 @@ func (e *Email) Content(threadID id.EventID, options *ContentOptions) *event.Con
 	}
 
 	content := event.Content{
-		Raw: map[string]interface{}{
+		Raw: map[string]any{
 			options.MessageIDKey:  e.MessageID,
 			options.InReplyToKey:  e.InReplyTo,
 			options.ReferencesKey: e.References,
@@ -174,6 +183,28 @@ func (e *Email) Content(threadID id.EventID, options *ContentOptions) *event.Con
 	return &content
 }
 
+// ContentBody converts the email object to a Matrix event content that contains email body only
+// NOTE: returns nil if threadify is disabled
+func (e *Email) ContentBody(threadID id.EventID, options *ContentOptions) *event.Content {
+	if !options.Threadify {
+		return nil
+	}
+	var text string
+	if e.HTML != "" && options.HTML {
+		text = format.HTMLToMarkdown(e.HTML)
+	} else {
+		text = e.Text
+	}
+
+	parsed := format.RenderMarkdown(text, true, true)
+	parsed.RelatesTo = linkpearl.RelatesTo(threadID, !options.Threads)
+
+	content := event.Content{
+		Parsed: &parsed,
+	}
+	return &content
+}
+
 // Compose converts the email object to a string (to be used for delivery via SMTP) and possibly DKIM-signs it
 func (e *Email) Compose(privkey string) string {
 	textSize := len(e.Text)
@@ -231,7 +262,10 @@ func (e *Email) sign(domain, privkey string, data strings.Builder) string {
 	if err != nil {
 		return data.String()
 	}
-	signer := parsedkey.(crypto.Signer)
+	signer, ok := parsedkey.(crypto.Signer)
+	if !ok {
+		return data.String()
+	}
 
 	options := &dkim.SignOptions{
 		Domain:   domain,

email/options.go

@@ -18,6 +18,7 @@ type ContentOptions struct {
 	Subject   bool
 	HTML      bool
 	Threads   bool
+	Threadify bool
 
 	// Keys
 	MessageIDKey  string

email/utils.go

@@ -10,7 +10,7 @@ import (
 	"maunium.net/go/mautrix/id"
 )
 
-var styleRegex = regexp.MustCompile("<style((.|\n|\r)*?)<\\/style>")
+var styleRegex = regexp.MustCompile("<style((.|\n|\r)*?)</style>")
 
 // AddressValid checks if email address is valid
 func AddressValid(email string) bool {

go.mod

@@ -14,21 +14,20 @@ require (
 	github.com/getsentry/sentry-go v0.13.0
 	github.com/jhillyerd/enmime v0.10.0
 	github.com/lib/pq v1.10.9
-	github.com/mattn/go-sqlite3 v1.14.17
+	github.com/mattn/go-sqlite3 v1.14.19
 	github.com/mcnijman/go-emailaddress v1.1.0
 	github.com/mileusna/crontab v1.2.0
 	github.com/raja/argon2pw v1.0.2-0.20210910183755-a391af63bd39
-	github.com/rs/zerolog v1.30.0
+	github.com/rs/zerolog v1.31.0
 	gitlab.com/etke.cc/go/env v1.0.0
 	gitlab.com/etke.cc/go/fswatcher v1.0.0
 	gitlab.com/etke.cc/go/healthchecks v1.0.1
 	gitlab.com/etke.cc/go/mxidwc v1.0.0
 	gitlab.com/etke.cc/go/secgen v1.1.1
-	gitlab.com/etke.cc/go/trysmtp v1.1.3
 	gitlab.com/etke.cc/go/validator v1.0.6
-	gitlab.com/etke.cc/linkpearl v0.0.0-20230928120707-1e99315dc616
-	golang.org/x/exp v0.0.0-20230905200255-921286631fa9
-	maunium.net/go/mautrix v0.16.1
+	gitlab.com/etke.cc/linkpearl v0.0.0-20231121221431-72443f33d266
+	golang.org/x/exp v0.0.0-20231110203233-9a3e6036ecaa
+	maunium.net/go/mautrix v0.16.2
 )
 
 require (
@@ -37,25 +36,26 @@ require (
 	github.com/cention-sany/utf7 v0.0.0-20170124080048-26cad61bd60a // indirect
 	github.com/gogs/chardet v0.0.0-20191104214054-4b6791f73a28 // indirect
 	github.com/google/uuid v1.3.0 // indirect
-	github.com/hashicorp/golang-lru/v2 v2.0.1 // indirect
+	github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect
 	github.com/jaytaylor/html2text v0.0.0-20200412013138-3577fbdbcff7 // indirect
 	github.com/mattn/go-colorable v0.1.13 // indirect
-	github.com/mattn/go-isatty v0.0.19 // indirect
+	github.com/mattn/go-isatty v0.0.20 // indirect
 	github.com/mattn/go-runewidth v0.0.12 // indirect
 	github.com/mikesmitty/edkey v0.0.0-20170222072505-3356ea4e686a // indirect
 	github.com/olekukonko/tablewriter v0.0.5 // indirect
 	github.com/pkg/errors v0.9.1 // indirect
 	github.com/rivo/uniseg v0.2.0 // indirect
 	github.com/ssor/bom v0.0.0-20170718123548-6386211fdfcf // indirect
-	github.com/tidwall/gjson v1.16.0 // indirect
+	github.com/tidwall/gjson v1.17.0 // indirect
 	github.com/tidwall/match v1.1.1 // indirect
 	github.com/tidwall/pretty v1.2.1 // indirect
 	github.com/tidwall/sjson v1.2.5 // indirect
-	github.com/yuin/goldmark v1.5.6 // indirect
-	go.mau.fi/util v0.1.0 // indirect
-	golang.org/x/crypto v0.13.0 // indirect
-	golang.org/x/net v0.15.0 // indirect
-	golang.org/x/sys v0.12.0 // indirect
-	golang.org/x/text v0.13.0 // indirect
+	github.com/yuin/goldmark v1.6.0 // indirect
+	gitlab.com/etke.cc/go/trysmtp v1.1.3 // indirect
+	go.mau.fi/util v0.2.1 // indirect
+	golang.org/x/crypto v0.15.0 // indirect
+	golang.org/x/net v0.18.0 // indirect
+	golang.org/x/sys v0.14.0 // indirect
+	golang.org/x/text v0.14.0 // indirect
 	maunium.net/go/maulogger/v2 v2.4.1 // indirect
 )

go.sum

@@ -37,8 +37,8 @@ github.com/gogs/chardet v0.0.0-20191104214054-4b6791f73a28/go.mod h1:Pcatq5tYkCW
 github.com/google/go-cmp v0.5.8 h1:e6P7q2lk1O+qJJb4BtCQXlK8vWEO8V1ZeuEdJNOqZyg=
 github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I=
 github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
-github.com/hashicorp/golang-lru/v2 v2.0.1 h1:5pv5N1lT1fjLg2VQ5KWc7kmucp2x/kvFOnxuVTqZ6x4=
-github.com/hashicorp/golang-lru/v2 v2.0.1/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM=
+github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k=
+github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM=
 github.com/jaytaylor/html2text v0.0.0-20200412013138-3577fbdbcff7 h1:g0fAGBisHaEQ0TRq1iBvemFRf+8AEWEmBESSiWB3Vsc=
 github.com/jaytaylor/html2text v0.0.0-20200412013138-3577fbdbcff7/go.mod h1:CVKlgaMiht+LXvHG173ujK6JUhZXKb2u/BQtjPDIvyk=
 github.com/jhillyerd/enmime v0.10.0 h1:DZEzhptPRBesvN3gf7K1BOh4rfpqdsdrEoxW1Edr/3s=
@@ -46,18 +46,19 @@ github.com/jhillyerd/enmime v0.10.0/go.mod h1:Qpe8EEemJMFAF8+NZoWdpXvK2Yb9dRF0k/
 github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw=
 github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=
 github.com/martinlindhe/base36 v1.0.0/go.mod h1:+AtEs8xrBpCeYgSLoY/aJ6Wf37jtBuR0s35750M27+8=
-github.com/mattn/go-colorable v0.1.12/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb520KVy5xxl4=
 github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA=
 github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg=
-github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94=
 github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
-github.com/mattn/go-isatty v0.0.19 h1:JITubQf0MOLdlGRuRq+jtsDlekdYPia9ZFsB8h/APPA=
 github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
+github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
+github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
 github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI=
 github.com/mattn/go-runewidth v0.0.12 h1:Y41i/hVW3Pgwr8gV+J23B9YEY0zxjptBuCWEaxmAOow=
 github.com/mattn/go-runewidth v0.0.12/go.mod h1:RAqKPSqVFrSLVXbA8x7dzmKdmGzieGRCM46jaSJTDAk=
-github.com/mattn/go-sqlite3 v1.14.17 h1:mCRHCLDUBXgpKAqIKsaAaAsrAlbkeomtRFKXh2L6YIM=
-github.com/mattn/go-sqlite3 v1.14.17/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S2DGjv9HUNg=
+github.com/mattn/go-sqlite3 v1.14.18 h1:JL0eqdCOq6DJVNPSvArO/bIV9/P7fbGrV00LZHc+5aI=
+github.com/mattn/go-sqlite3 v1.14.18/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S2DGjv9HUNg=
+github.com/mattn/go-sqlite3 v1.14.19 h1:fhGleo2h1p8tVChob4I9HpmVFIAkKGpiukdrgQbWfGI=
+github.com/mattn/go-sqlite3 v1.14.19/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S2DGjv9HUNg=
 github.com/mcnijman/go-emailaddress v1.1.0 h1:7/Uxgn9pXwXmvXsFSgORo6XoRTrttj7AGmmB2yFArAg=
 github.com/mcnijman/go-emailaddress v1.1.0/go.mod h1:m+aauxGmv31sB5zZ1I8ICcMoa9ZHOA9RiurCijfvkhI=
 github.com/mikesmitty/edkey v0.0.0-20170222072505-3356ea4e686a h1:eU8j/ClY2Ty3qdHnn0TyW3ivFoPC/0F1gQZz8yTxbbE=
@@ -77,8 +78,8 @@ github.com/rivo/uniseg v0.1.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJ
 github.com/rivo/uniseg v0.2.0 h1:S1pD9weZBuJdFmowNwbpi7BJ8TNftyUImj/0WQi72jY=
 github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
 github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg=
-github.com/rs/zerolog v1.30.0 h1:SymVODrcRsaRaSInD9yQtKbtWqwsfoPcRff/oRXLj4c=
-github.com/rs/zerolog v1.30.0/go.mod h1:/tk+P47gFdPXq4QYjvCmT5/Gsug2nagsFWBWhAiSi1w=
+github.com/rs/zerolog v1.31.0 h1:FcTR3NnLWW+NnTwwhFWiJSZr4ECLpqCm6QsEnyvbV4A=
+github.com/rs/zerolog v1.31.0/go.mod h1:/7mN4D5sKwJLZQ2b/znpjC3/GQWY/xaDXUM0kKWRHss=
 github.com/ssor/bom v0.0.0-20170718123548-6386211fdfcf h1:pvbZ0lM0XWPBqUKqFU8cmavspvIl9nulOYwdy6IFRRo=
 github.com/ssor/bom v0.0.0-20170718123548-6386211fdfcf/go.mod h1:RJID2RhlZKId02nZ62WenDCkgHFerpIOmW0iT7GKmXM=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
@@ -86,8 +87,8 @@ github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UV
 github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk=
 github.com/tidwall/gjson v1.14.2/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk=
-github.com/tidwall/gjson v1.16.0 h1:SyXa+dsSPpUlcwEDuKuEBJEz5vzTvOea+9rjyYodQFg=
-github.com/tidwall/gjson v1.16.0/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk=
+github.com/tidwall/gjson v1.17.0 h1:/Jocvlh98kcTfpN2+JzGQWQcqrPQwDrVEMApx/M5ZwM=
+github.com/tidwall/gjson v1.17.0/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk=
 github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA=
 github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM=
 github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU=
@@ -95,8 +96,8 @@ github.com/tidwall/pretty v1.2.1 h1:qjsOFOWWQl+N3RsoF5/ssm1pHmJJwhjlSbZ51I6wMl4=
 github.com/tidwall/pretty v1.2.1/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU=
 github.com/tidwall/sjson v1.2.5 h1:kLy8mja+1c9jlljvWTlSazM7cKDRfJuR/bOJhcY5NcY=
 github.com/tidwall/sjson v1.2.5/go.mod h1:Fvgq9kS/6ociJEDnK0Fk1cpYF4FIW6ZF7LAe+6jwd28=
-github.com/yuin/goldmark v1.5.6 h1:COmQAWTCcGetChm3Ig7G/t8AFAN00t+o8Mt4cf7JpwA=
-github.com/yuin/goldmark v1.5.6/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
+github.com/yuin/goldmark v1.6.0 h1:boZcn2GTjpsynOsC0iJHnBWa4Bi0qzfJjthwauItG68=
+github.com/yuin/goldmark v1.6.0/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
 gitlab.com/etke.cc/go/env v1.0.0 h1:J98BwzOuELnjsVPFvz5wa79L7IoRV9CmrS41xLYXtSw=
 gitlab.com/etke.cc/go/env v1.0.0/go.mod h1:e1l4RM5MA1sc0R1w/RBDAESWRwgo5cOG9gx8BKUn2C4=
 gitlab.com/etke.cc/go/fswatcher v1.0.0 h1:uyiVn+1NVCjOLZrXSZouIDBDZBMwVipS4oYuvAFpPzo=
@@ -111,40 +112,39 @@ gitlab.com/etke.cc/go/trysmtp v1.1.3 h1:e2EHond77onMaecqCg6mWumffTSEf+ycgj88nbee
 gitlab.com/etke.cc/go/trysmtp v1.1.3/go.mod h1:lOO7tTdAE0a3ETV3wN3GJ7I1Tqewu7YTpPWaOmTteV0=
 gitlab.com/etke.cc/go/validator v1.0.6 h1:w0Muxf9Pqw7xvF7NaaswE6d7r9U3nB2t2l5PnFMrecQ=
 gitlab.com/etke.cc/go/validator v1.0.6/go.mod h1:Id0SxRj0J3IPhiKlj0w1plxVLZfHlkwipn7HfRZsDts=
-gitlab.com/etke.cc/linkpearl v0.0.0-20230928120707-1e99315dc616 h1:Gvhmq84VmAJN1xRzRBK79XJVObAvVcx9Q3s6K+Zo644=
-gitlab.com/etke.cc/linkpearl v0.0.0-20230928120707-1e99315dc616/go.mod h1:IZ0TE+ZnIdJLb538owDMxhtpWH7blfW+oR7e5XRXxNY=
-go.mau.fi/util v0.1.0 h1:BwIFWIOEeO7lsiI2eWKFkWTfc5yQmoe+0FYyOFVyaoE=
-go.mau.fi/util v0.1.0/go.mod h1:AxuJUMCxpzgJ5eV9JbPWKRH8aAJJidxetNdUj7qcb84=
+gitlab.com/etke.cc/linkpearl v0.0.0-20231121221431-72443f33d266 h1:mGbLQkdE35WeyinqP38HC0dqUOJ7FItEAumVIOz7Gg8=
+gitlab.com/etke.cc/linkpearl v0.0.0-20231121221431-72443f33d266/go.mod h1:wFEvngglb6ZTlE58/2a9gwYYs6V3FTYclYn5Pf5EGyQ=
+go.mau.fi/util v0.2.1 h1:eazulhFE/UmjOFtPrGg6zkF5YfAyiDzQb8ihLMbsPWw=
+go.mau.fi/util v0.2.1/go.mod h1:MjlzCQEMzJ+G8RsPawHzpLB8rwTo3aPIjG5FzBvQT/c=
 golang.org/x/crypto v0.0.0-20220518034528-6f7dac969898/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
-golang.org/x/crypto v0.13.0 h1:mvySKfSWJ+UKUii46M40LOvyWfN0s2U+46/jDd0e6Ck=
-golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc=
-golang.org/x/exp v0.0.0-20230905200255-921286631fa9 h1:GoHiUyI/Tp2nVkLI2mCxVkOjsbSXD66ic0XW0js0R9g=
-golang.org/x/exp v0.0.0-20230905200255-921286631fa9/go.mod h1:S2oDrQGGwySpoQPVqRShND87VCbxmc6bL1Yd2oYrm6k=
+golang.org/x/crypto v0.15.0 h1:frVn1TEaCEaZcn3Tmd7Y2b5KKPaZ+I32Q2OA3kYp5TA=
+golang.org/x/crypto v0.15.0/go.mod h1:4ChreQoLWfG3xLDer1WdlH5NdlQ3+mwnQq1YTKY+72g=
+golang.org/x/exp v0.0.0-20231110203233-9a3e6036ecaa h1:FRnLl4eNAQl8hwxVVC17teOw8kdjVDVAiFMtgUdTSRQ=
+golang.org/x/exp v0.0.0-20231110203233-9a3e6036ecaa/go.mod h1:zk2irFbV9DP96SEBUUAy67IdHUaZuSnrz1n472HUCLE=
 golang.org/x/net v0.0.0-20180911220305-26e67e76b6c3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20210501142056-aec3718b3fa0/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk=
 golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/net v0.0.0-20220624214902-1bab6f366d9e/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
-golang.org/x/net v0.15.0 h1:ugBLEUaxABaB5AJqW9enI0ACdci2RUd4eP51NTBvuJ8=
-golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk=
+golang.org/x/net v0.18.0 h1:mIYleuAkSbHh0tCv7RvjL3F6ZVbLjq4+R7zbOn3Kokg=
+golang.org/x/net v0.18.0/go.mod h1:/czyP5RqHAH4odGYxBJ1qz0+CE5WZ+2j1YgoEo8F2jQ=
 golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.12.0 h1:CM0HF96J0hcLAwsHPJZjfdNzs0gftsLfgKt57wWHJ0o=
 golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.14.0 h1:Vz7Qs629MkJkGyHxUlRHizWJRG2j8fbQKjELVSNhy7Q=
+golang.org/x/sys v0.14.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
-golang.org/x/term v0.12.0 h1:/ZfYdc3zq+q02Rv9vGqTeSItdzZTSNDmfTi0mBAuidU=
+golang.org/x/term v0.14.0 h1:LGK9IlZ8T9jvdy6cTdfKUCltatMFOehAQo9SRC46UQ8=
 golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
 golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
-golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k=
-golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
+golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ=
+golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
@@ -152,5 +152,5 @@ gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
 gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 maunium.net/go/maulogger/v2 v2.4.1 h1:N7zSdd0mZkB2m2JtFUsiGTQQAdP0YeFWT7YMc80yAL8=
 maunium.net/go/maulogger/v2 v2.4.1/go.mod h1:omPuYwYBILeVQobz8uO3XC8DIRuEb5rXYlQSuqrbCho=
-maunium.net/go/mautrix v0.16.1 h1:Wb3CvOCe8A/NLsFeZYxKrgXKiqeZUQEBD1zqm7n/kWk=
-maunium.net/go/mautrix v0.16.1/go.mod h1:2Jf15tulVtr6LxoiRL4smRXwpkGWUNfBFhwh/aXDBuk=
+maunium.net/go/mautrix v0.16.2 h1:a6GUJXNWsTEOO8VE4dROBfCIfPp50mqaqzv7KPzChvg=
+maunium.net/go/mautrix v0.16.2/go.mod h1:YL4l4rZB46/vj/ifRMEjcibbvHjgxHftOF1SgmruLu4=

justfile

@@ -1,14 +1,17 @@
-CI_REGISTRY_IMAGE := env_var_or_default("CI_REGISTRY_IMAGE", "registry.gitlab.com/etke.cc/postmoogle")
-REGISTRY_IMAGE := env_var_or_default("REGISTRY_IMAGE", "registry.etke.cc/etke.cc/postmoogle")
-CI_COMMIT_TAG := if env_var_or_default("CI_COMMIT_TAG", "main") == "main" { "latest" } else { env_var_or_default("CI_COMMIT_TAG", "latest") }
+platforms := env_var_or_default("PLATFORMS", "linux/amd64")
+tag := if env_var_or_default("CI_COMMIT_TAG", "main") == "main" { "latest" } else { env_var_or_default("CI_COMMIT_TAG", "latest") }
+repo := trim_end_match(replace(replace_regex(env_var_or_default("CI_REPOSITORY_URL", `git remote get-url origin`), ".*@|", ""), ":", "/"), ".git")
+project := file_name(repo)
+gitlab_image := "registry." + repo + ":" + tag
+etke_image := replace(gitlab_image, "gitlab.com", "etke.cc")
 
 # show help by default
 default:
     @just --list --justfile {{ justfile() }}
 
 # update go deps
-update:
-    go get ./cmd
+update *flags:
+    go get {{flags}} ./cmd
     go get gitlab.com/etke.cc/linkpearl@latest
     go mod tidy
     go mod vendor
@@ -21,9 +24,17 @@ lint:
 lintfix:
     golangci-lint run --fix ./...
 
+# generate mocks
+mocks:
+    @mockery --all --inpackage --testonly --exclude vendor
+
+# run cpu or mem profiler UI
+profile type:
+    go tool pprof -http 127.0.0.1:8000 .pprof/{{ type }}.prof
+
 # run unit tests
 test:
-    @go test -coverprofile=cover.out ./...
+    @go test -cover -coverprofile=cover.out -coverpkg=./... -covermode=set ./...
     @go tool cover -func=cover.out
     -@rm -f cover.out
 
@@ -33,7 +44,7 @@ run:
 
 # build app
 build:
-    go build -v -o postmoogle ./cmd
+    go build -v -o {{ project }} ./cmd
 
 # docker login
 login:
@@ -42,4 +53,4 @@ login:
 # docker build
 docker:
     docker buildx create --use
-    docker buildx build --platform linux/arm64/v8,linux/amd64 --push -t {{ CI_REGISTRY_IMAGE }}:{{ CI_COMMIT_TAG }} -t {{ REGISTRY_IMAGE }}:{{ CI_COMMIT_TAG }} .
+    docker buildx build --pull --provenance=false --platform {{ platforms }} --push -t {{ gitlab_image }} -t {{ etke_image }} .

smtp/client.go

@@ -2,16 +2,17 @@ package smtp
 
 import (
 	"crypto/tls"
+	"fmt"
 	"io"
+	"net"
 	"net/smtp"
 	"strings"
 
 	"github.com/rs/zerolog"
-	"gitlab.com/etke.cc/go/trysmtp"
 )
 
 type MailSender interface {
-	Send(from string, to string, data string) error
+	Send(from, to, data string) error
 }
 
 // SMTP client
@@ -20,6 +21,8 @@ type Client struct {
 	log    *zerolog.Logger
 }
 
+var errNoSMTP = fmt.Errorf("cannot connect to any SMTP server")
+
 func newClient(cfg *RelayConfig, log *zerolog.Logger) *Client {
 	return &Client{
 		config: cfg,
@@ -28,46 +31,49 @@ func newClient(cfg *RelayConfig, log *zerolog.Logger) *Client {
 }
 
 // Send email
-func (c Client) Send(from string, to string, data string) error {
-	c.log.Debug().Str("from", from).Str("to", to).Msg("sending email")
+func (c Client) Send(from, to, data string) error {
+	log := c.log.With().Str("from", from).Str("to", to).Logger()
+	log.Debug().Msg("sending email")
 
 	var conn *smtp.Client
 	var err error
 	if c.config.Host != "" {
-		conn, err = c.createDirectClient(from, to)
+		log.Debug().Msg("creating relay client...")
+		conn, err = c.createRelayClient(from, to)
 	} else {
-		conn, err = trysmtp.Connect(from, to)
+		log.Debug().Msg("trying direct SMTP connection...")
+		conn, err = c.createDirectClient(from, to)
 	}
 
 	if conn == nil {
-		c.log.Error().Err(err).Str("server_of", to).Msg("cannot connect to SMTP server")
+		log.Error().Err(err).Str("server_of", to).Msg("cannot connect to SMTP server")
 		return err
 	}
 	if err != nil {
-		c.log.Warn().Err(err).Str("server_of", to).Msg("connection to the SMTP server returned non-fatal error(-s)")
+		log.Warn().Err(err).Str("server_of", to).Msg("connection to the SMTP server returned non-fatal error(-s)")
 	}
 	defer conn.Close()
 
 	var w io.WriteCloser
 	w, err = conn.Data()
 	if err != nil {
-		c.log.Error().Err(err).Msg("cannot send DATA command")
+		log.Error().Err(err).Msg("cannot send DATA command")
 		return err
 	}
 	defer w.Close()
-	c.log.Debug().Str("DATA", data).Msg("sending command")
+	log.Debug().Str("DATA", data).Msg("sending command")
 	_, err = strings.NewReader(data).WriteTo(w)
 	if err != nil {
-		c.log.Error().Err(err).Msg("cannot write DATA")
+		log.Error().Err(err).Msg("cannot write DATA")
 		return err
 	}
 
-	c.log.Debug().Msg("email has been sent")
+	log.Debug().Msg("email has been sent")
 	return nil
 }
 
-// createDirectClient connects directly to the provided smtp host
-func (c *Client) createDirectClient(from string, to string) (*smtp.Client, error) {
+// createRelayClientconnects directly to the provided smtp host
+func (c *Client) createRelayClient(from, to string) (*smtp.Client, error) {
 	localname := strings.SplitN(from, "@", 2)[1]
 	target := c.config.Host + ":" + c.config.Port
 	conn, err := smtp.Dial(target)
@@ -81,8 +87,8 @@ func (c *Client) createDirectClient(from string, to string) (*smtp.Client, error
 	}
 
 	if ok, _ := conn.Extension("STARTTLS"); ok {
-		config := &tls.Config{ServerName: c.config.Host}
-		conn.StartTLS(config) //nolint:errcheck // if it doesn't work - we can't do anything anyway
+		config := &tls.Config{ServerName: c.config.Host} //nolint:gosec // it's smtp, even that is too strict sometimes
+		conn.StartTLS(config)                            //nolint:errcheck // if it doesn't work - we can't do anything anyway
 	}
 
 	if c.config.Usename != "" {
@@ -107,3 +113,76 @@ func (c *Client) createDirectClient(from string, to string) (*smtp.Client, error
 
 	return conn, nil
 }
+
+func (c *Client) createDirectClient(from, to string) (*smtp.Client, error) {
+	localname := strings.SplitN(from, "@", 2)[1]
+	hostname := strings.SplitN(to, "@", 2)[1]
+	client, cerr := c.trySMTP(localname, hostname)
+	if client == nil {
+		c.log.Warn().Err(cerr).Str("from", from).Str("to", to).Msg("cannot create direct SMTP client")
+		return nil, cerr
+	}
+
+	err := client.Mail(from)
+	if err != nil {
+		c.log.Warn().Err(err).Str("from", from).Str("to", to).Msg("cannot send MAIL command")
+		client.Close()
+		return nil, err
+	}
+
+	err = client.Rcpt(to)
+	if err != nil {
+		c.log.Warn().Err(err).Str("from", from).Str("to", to).Msg("cannot send RCPT command")
+		client.Close()
+		return nil, err
+	}
+
+	return client, cerr
+}
+
+func (c *Client) trySMTP(localname, hostname string) (*smtp.Client, error) {
+	mxs, err := net.LookupMX(hostname)
+	if err != nil {
+		return nil, err
+	}
+
+	var client *smtp.Client
+	for _, mx := range mxs {
+		if mx.Host == "." {
+			continue // no records case
+		}
+		client = c.connect(localname, hostname, strings.TrimSuffix(mx.Host, "."))
+		if client != nil {
+			return client, nil
+		}
+	}
+
+	// If there are no MX records, according to https://datatracker.ietf.org/doc/html/rfc5321#section-5.1,
+	// we're supposed to try talking directly to the host.
+	client = c.connect(localname, hostname, hostname)
+	if client != nil {
+		return client, nil
+	}
+
+	return nil, errNoSMTP
+}
+
+func (c *Client) connect(localname, serverOf, mxhost string) *smtp.Client {
+	target := mxhost + ":25"
+	conn, err := smtp.Dial(target)
+	if err != nil {
+		c.log.Warn().Err(err).Str("target", serverOf).Str("host", mxhost).Msg("cannot dial SMTP server")
+		return nil
+	}
+	err = conn.Hello(localname)
+	if err != nil {
+		c.log.Warn().Err(err).Str("target", serverOf).Str("host", mxhost).Msg("cannot hello SMTP server")
+		return nil
+	}
+	if ok, _ := conn.Extension("STARTTLS"); ok {
+		config := &tls.Config{ServerName: mxhost} //nolint:gosec // it's smtp, even that is too strict sometimes
+		conn.StartTLS(config)                     //nolint:errcheck // if it doesn't work - we can't do anything anyway
+	}
+
+	return conn
+}

smtp/logger.go

@@ -11,24 +11,24 @@ type validatorLoggerWrapper struct {
 	log *zerolog.Logger
 }
 
-func (l validatorLoggerWrapper) Info(msg string, args ...interface{}) {
+func (l validatorLoggerWrapper) Info(msg string, args ...any) {
 	l.log.Info().Msgf(msg, args...)
 }
 
-func (l validatorLoggerWrapper) Error(msg string, args ...interface{}) {
+func (l validatorLoggerWrapper) Error(msg string, args ...any) {
 	l.log.Error().Msgf(msg, args...)
 }
 
 // loggerWrapper is a wrapper around any logger to implement smtp.Logger interface
 type loggerWrapper struct {
-	log func(string, ...interface{})
+	log func(string, ...any)
 }
 
-func (l loggerWrapper) Printf(format string, v ...interface{}) {
+func (l loggerWrapper) Printf(format string, v ...any) {
 	l.log(format, v...)
 }
 
-func (l loggerWrapper) Println(v ...interface{}) {
+func (l loggerWrapper) Println(v ...any) {
 	msg := strings.Repeat("%v ", len(v))
 	l.log(msg, v...)
 }

smtp/manager.go

@@ -90,7 +90,7 @@ func NewManager(cfg *Config) *Manager {
 	}
 
 	s := smtp.NewServer(mailsrv)
-	s.ErrorLog = loggerWrapper{func(s string, i ...interface{}) { cfg.Logger.Error().Msgf(s, i...) }}
+	s.ErrorLog = loggerWrapper{func(s string, i ...any) { cfg.Logger.Error().Msgf(s, i...) }}
 	s.ReadTimeout = 10 * time.Second
 	s.WriteTimeout = 10 * time.Second
 	s.MaxMessageBytes = cfg.MaxSize * 1024 * 1024
@@ -209,7 +209,7 @@ func (m *Manager) loadTLSConfig() bool {
 		return false
 	}
 
-	m.tls.Config = &tls.Config{Certificates: certificates}
+	m.tls.Config = &tls.Config{Certificates: certificates} //nolint:gosec // it's email, even that config is too strict sometimes
 	m.smtp.TLSConfig = m.tls.Config
 	return true
 }

smtp/server.go

@@ -10,17 +10,28 @@ import (
 	"gitlab.com/etke.cc/postmoogle/email"
 )
 
+const (
+	// NoUserCode SMTP code
+	NoUserCode = 550
+	// BannedCode SMTP code
+	BannedCode = 554
+)
+
 var (
+	// NoUserEnhancedCode enhanced SMTP code
+	NoUserEnhancedCode = smtp.EnhancedCode{5, 5, 0}
+	// BannedEnhancedCode enhanced SMTP code
+	BannedEnhancedCode = smtp.EnhancedCode{5, 5, 4}
 	// ErrBanned returned to banned hosts
 	ErrBanned = &smtp.SMTPError{
-		Code:         554,
-		EnhancedCode: smtp.EnhancedCode{5, 5, 4},
+		Code:         BannedCode,
+		EnhancedCode: BannedEnhancedCode,
 		Message:      "please, don't bother me anymore, kupo.",
 	}
 	// ErrNoUser returned when no such mailbox found
 	ErrNoUser = &smtp.SMTPError{
-		Code:         550,
-		EnhancedCode: smtp.EnhancedCode{5, 5, 0},
+		Code:         NoUserCode,
+		EnhancedCode: NoUserEnhancedCode,
 		Message:      "no such user here, kupo.",
 	}
 )

smtp/session.go

@@ -20,6 +20,16 @@ import (
 	"gitlab.com/etke.cc/postmoogle/utils"
 )
 
+// GraylistCode SMTP code
+const GraylistCode = 451
+
+var (
+	// ErrInvalidEmail for invalid emails :)
+	ErrInvalidEmail = errors.New("please, provide valid email address")
+	// GraylistEnhancedCode is GraylistCode in enhanced code notation
+	GraylistEnhancedCode = smtp.EnhancedCode{4, 5, 1}
+)
+
 // incomingSession represents an SMTP-submission session receiving emails from remote servers
 type incomingSession struct {
 	log          *zerolog.Logger
@@ -32,7 +42,7 @@ type incomingSession struct {
 	domains      []string
 	roomID       id.RoomID
 
-	ctx  context.Context
+	ctx  context.Context //nolint:containedctx // that's session
 	addr net.Addr
 	tos  []string
 	from string
@@ -45,7 +55,7 @@ func (s *incomingSession) Mail(from string, opts smtp.MailOptions) error {
 		s.ban(s.addr)
 		return ErrBanned
 	}
-	s.from = from
+	s.from = email.Address(from)
 	s.log.Debug().Str("from", from).Any("options", opts).Msg("incoming mail")
 	return nil
 }
@@ -89,13 +99,13 @@ func (s *incomingSession) getAddr(envelope *enmime.Envelope) net.Addr {
 		return s.addr
 	}
 
-	host, portString, _ := net.SplitHostPort(addrHeader) //nolint:errcheck
+	host, portString, _ := net.SplitHostPort(addrHeader) //nolint:errcheck // it is real addr
 	if host == "" {
 		return s.addr
 	}
 
 	var port int
-	port, _ = strconv.Atoi(portString) //nolint:errcheck
+	port, _ = strconv.Atoi(portString) //nolint:errcheck // it's a real addr
 
 	realAddr := &net.TCPAddr{IP: net.ParseIP(host), Port: port}
 	s.log.Info().Str("addr", realAddr.String()).Msg("real address")
@@ -115,7 +125,7 @@ func (s *incomingSession) Data(r io.Reader) error {
 		return err
 	}
 	addr := s.getAddr(envelope)
-	reader.Seek(0, io.SeekStart) //nolint:errcheck
+	reader.Seek(0, io.SeekStart) //nolint:errcheck // becase we're sure that's ok
 	validations := s.getFilters(s.roomID)
 	if !validateIncoming(s.from, s.tos[0], addr, s.log, validations) {
 		s.ban(addr)
@@ -123,8 +133,8 @@ func (s *incomingSession) Data(r io.Reader) error {
 	}
 	if s.greylisted(addr) {
 		return &smtp.SMTPError{
-			Code:         451,
-			EnhancedCode: smtp.EnhancedCode{4, 5, 1},
+			Code:         GraylistCode,
+			EnhancedCode: GraylistEnhancedCode,
 			Message:      "You have been greylisted, try again a bit later.",
 		}
 	}
@@ -164,16 +174,16 @@ type outgoingSession struct {
 	domains   []string
 	getRoomID func(string) (id.RoomID, bool)
 
-	ctx      context.Context
+	ctx      context.Context //nolint:containedctx // that's session
 	tos      []string
 	from     string
 	fromRoom id.RoomID
 }
 
-func (s *outgoingSession) Mail(from string, opts smtp.MailOptions) error {
+func (s *outgoingSession) Mail(from string, _ smtp.MailOptions) error {
 	sentry.GetHubFromContext(s.ctx).Scope().SetTag("from", from)
 	if !email.AddressValid(from) {
-		return errors.New("please, provide email address")
+		return ErrInvalidEmail
 	}
 	hostname := utils.Hostname(from)
 	var domainok bool
@@ -234,7 +244,7 @@ func validateIncoming(from, to string, senderAddr net.Addr, log *zerolog.Logger,
 	case *net.TCPAddr:
 		sender = netaddr.IP
 	default:
-		host, _, _ := net.SplitHostPort(senderAddr.String()) // nolint:errcheck
+		host, _, _ := net.SplitHostPort(senderAddr.String()) //nolint:errcheck // interface constraints
 		sender = net.ParseIP(host)
 	}
 

utils/command.go

@@ -5,21 +5,24 @@ import (
 	"strings"
 )
 
+// MinSendCommandParts is minimal count of space-separated parts for !pm send command
+const MinSendCommandParts = 3
+
 // ErrInvalidArgs returned when a command's arguments are invalid
 var ErrInvalidArgs = fmt.Errorf("invalid arguments")
 
 // ParseSend parses "!pm send" command, returns to, subject, body, err
-func ParseSend(commandSlice []string) (string, string, string, error) {
+func ParseSend(commandSlice []string) (to, subject, body string, err error) {
 	message := strings.Join(commandSlice, " ")
 	lines := strings.Split(message, "\n")
-	if len(lines) < 3 {
+	if len(lines) < MinSendCommandParts {
 		return "", "", "", ErrInvalidArgs
 	}
 
 	commandSlice = strings.Split(lines[0], " ")
-	to := commandSlice[1]
-	subject := lines[1]
-	body := strings.Join(lines[2:], "\n")
+	to = commandSlice[1]
+	subject = lines[1]
+	body = strings.Join(lines[2:], "\n")
 
 	return to, subject, body, nil
 }

utils/mail.go

@@ -25,8 +25,7 @@ func Hostname(email string) string {
 }
 
 // EmailParts parses email address into mailbox, subaddress, and hostname
-func EmailParts(email string) (string, string, string) {
-	var mailbox, hostname string
+func EmailParts(email string) (mailbox, sub, hostname string) {
 	address, err := emailaddress.Parse(email)
 	if err == nil {
 		mailbox = address.LocalPart
@@ -44,7 +43,6 @@ func EmailParts(email string) (string, string, string) {
 		}
 	}
 
-	var sub string
 	idx := strings.Index(mailbox, "+")
 	if idx != -1 {
 		sub = strings.ReplaceAll(mailbox[idx:], "+", "")
@@ -54,7 +52,7 @@ func EmailParts(email string) (string, string, string) {
 }
 
 // EmailsList returns human-readable list of mailbox's emails for all available domains
-func EmailsList(mailbox string, domain string) string {
+func EmailsList(mailbox, domain string) string {
 	var msg strings.Builder
 	domain = SanitizeDomain(domain)
 	msg.WriteString(mailbox)

vendor/github.com/hashicorp/golang-lru/v2/.golangci.yml

@@ -1,10 +1,14 @@
+# Copyright (c) HashiCorp, Inc.
+# SPDX-License-Identifier: MPL-2.0
+
 linters:
+  fast: false
+  disable-all: true
   enable:
-    - megacheck
     - revive
+    - megacheck
     - govet
     - unconvert
-    - megacheck
     - gas
     - gocyclo
     - dupl
@@ -13,18 +17,30 @@ linters:
     - unused
     - typecheck
     - ineffassign
-    - stylecheck
+    # - stylecheck
     - exportloopref
     - gocritic
     - nakedret
     - gosimple
     - prealloc
-  fast: false
-  disable-all: true
+
+# golangci-lint configuration file
+linters-settings:
+  revive:
+    ignore-generated-header: true
+    severity: warning
+    rules:
+      - name: package-comments
+        severity: warning
+        disabled: true
+      - name: exported
+        severity: warning
+        disabled: false
+        arguments: ["checkPrivateReceivers", "disableStutteringCheck"]
 
 issues:
+  exclude-use-default: false
   exclude-rules:
     - path: _test\.go
       linters:
         - dupl
-  exclude-use-default: false

vendor/github.com/hashicorp/golang-lru/v2/2q.go

@@ -1,7 +1,10 @@
+// Copyright (c) HashiCorp, Inc.
+// SPDX-License-Identifier: MPL-2.0
+
 package lru
 
 import (
-	"fmt"
+	"errors"
 	"sync"
 
 	"github.com/hashicorp/golang-lru/v2/simplelru"
@@ -27,12 +30,14 @@ const (
 // head. The ARCCache is similar, but does not require setting any
 // parameters.
 type TwoQueueCache[K comparable, V any] struct {
-	size       int
-	recentSize int
+	size        int
+	recentSize  int
+	recentRatio float64
+	ghostRatio  float64
 
 	recent      simplelru.LRUCache[K, V]
 	frequent    simplelru.LRUCache[K, V]
-	recentEvict simplelru.LRUCache[K, V]
+	recentEvict simplelru.LRUCache[K, struct{}]
 	lock        sync.RWMutex
 }
 
@@ -46,13 +51,13 @@ func New2Q[K comparable, V any](size int) (*TwoQueueCache[K, V], error) {
 // parameter values.
 func New2QParams[K comparable, V any](size int, recentRatio, ghostRatio float64) (*TwoQueueCache[K, V], error) {
 	if size <= 0 {
-		return nil, fmt.Errorf("invalid size")
+		return nil, errors.New("invalid size")
 	}
 	if recentRatio < 0.0 || recentRatio > 1.0 {
-		return nil, fmt.Errorf("invalid recent ratio")
+		return nil, errors.New("invalid recent ratio")
 	}
 	if ghostRatio < 0.0 || ghostRatio > 1.0 {
-		return nil, fmt.Errorf("invalid ghost ratio")
+		return nil, errors.New("invalid ghost ratio")
 	}
 
 	// Determine the sub-sizes
@@ -68,7 +73,7 @@ func New2QParams[K comparable, V any](size int, recentRatio, ghostRatio float64)
 	if err != nil {
 		return nil, err
 	}
-	recentEvict, err := simplelru.NewLRU[K, V](evictSize, nil)
+	recentEvict, err := simplelru.NewLRU[K, struct{}](evictSize, nil)
 	if err != nil {
 		return nil, err
 	}
@@ -77,6 +82,8 @@ func New2QParams[K comparable, V any](size int, recentRatio, ghostRatio float64)
 	c := &TwoQueueCache[K, V]{
 		size:        size,
 		recentSize:  recentSize,
+		recentRatio: recentRatio,
+		ghostRatio:  ghostRatio,
 		recent:      recent,
 		frequent:    frequent,
 		recentEvict: recentEvict,
@@ -153,8 +160,7 @@ func (c *TwoQueueCache[K, V]) ensureSpace(recentEvict bool) {
 	// the target, evict from there
 	if recentLen > 0 && (recentLen > c.recentSize || (recentLen == c.recentSize && !recentEvict)) {
 		k, _, _ := c.recent.RemoveOldest()
-		var empty V
-		c.recentEvict.Add(k, empty)
+		c.recentEvict.Add(k, struct{}{})
 		return
 	}
 
@@ -169,6 +175,34 @@ func (c *TwoQueueCache[K, V]) Len() int {
 	return c.recent.Len() + c.frequent.Len()
 }
 
+// Resize changes the cache size.
+func (c *TwoQueueCache[K, V]) Resize(size int) (evicted int) {
+	c.lock.Lock()
+	defer c.lock.Unlock()
+
+	// Recalculate the sub-sizes
+	recentSize := int(float64(size) * c.recentRatio)
+	evictSize := int(float64(size) * c.ghostRatio)
+	c.size = size
+	c.recentSize = recentSize
+
+	// ensureSpace
+	diff := c.recent.Len() + c.frequent.Len() - size
+	if diff < 0 {
+		diff = 0
+	}
+	for i := 0; i < diff; i++ {
+		c.ensureSpace(true)
+	}
+
+	// Reallocate the LRUs
+	c.recent.Resize(size)
+	c.frequent.Resize(size)
+	c.recentEvict.Resize(evictSize)
+
+	return diff
+}
+
 // Keys returns a slice of the keys in the cache.
 // The frequently used keys are first in the returned slice.
 func (c *TwoQueueCache[K, V]) Keys() []K {
@@ -179,6 +213,16 @@ func (c *TwoQueueCache[K, V]) Keys() []K {
 	return append(k1, k2...)
 }
 
+// Values returns a slice of the values in the cache.
+// The frequently used values are first in the returned slice.
+func (c *TwoQueueCache[K, V]) Values() []V {
+	c.lock.RLock()
+	defer c.lock.RUnlock()
+	v1 := c.frequent.Values()
+	v2 := c.recent.Values()
+	return append(v1, v2...)
+}
+
 // Remove removes the provided key from the cache.
 func (c *TwoQueueCache[K, V]) Remove(key K) {
 	c.lock.Lock()

vendor/github.com/hashicorp/golang-lru/v2/README.md

@@ -9,17 +9,71 @@ Documentation
 
 Full docs are available on [Go Packages](https://pkg.go.dev/github.com/hashicorp/golang-lru/v2)
 
-Example
-=======
-
-Using the LRU is very simple:
+LRU cache example
+=================
 
 ```go
-l, _ := New(128)
-for i := 0; i < 256; i++ {
-    l.Add(i, nil)
-}
-if l.Len() != 128 {
-    panic(fmt.Sprintf("bad len: %v", l.Len()))
+package main
+
+import (
+	"fmt"
+	"github.com/hashicorp/golang-lru/v2"
+)
+
+func main() {
+	l, _ := lru.New[int, any](128)
+	for i := 0; i < 256; i++ {
+		l.Add(i, nil)
+	}
+	if l.Len() != 128 {
+		panic(fmt.Sprintf("bad len: %v", l.Len()))
+	}
+}
+```
+
+Expirable LRU cache example
+===========================
+
+```go
+package main
+
+import (
+	"fmt"
+	"time"
+
+	"github.com/hashicorp/golang-lru/v2/expirable"
+)
+
+func main() {
+	// make cache with 10ms TTL and 5 max keys
+	cache := expirable.NewLRU[string, string](5, nil, time.Millisecond*10)
+
+
+	// set value under key1.
+	cache.Add("key1", "val1")
+
+	// get value under key1
+	r, ok := cache.Get("key1")
+
+	// check for OK value
+	if ok {
+		fmt.Printf("value before expiration is found: %v, value: %q\n", ok, r)
+	}
+
+	// wait for cache to expire
+	time.Sleep(time.Millisecond * 12)
+
+	// get value under key1 after key expiration
+	r, ok = cache.Get("key1")
+	fmt.Printf("value after expiration is found: %v, value: %q\n", ok, r)
+
+	// set value under key2, would evict old entry because it is already expired.
+	cache.Add("key2", "val2")
+
+	fmt.Printf("Cache len: %d\n", cache.Len())
+	// Output:
+	// value before expiration is found: true, value: "val1"
+	// value after expiration is found: false, value: ""
+	// Cache len: 1
 }
 ```

vendor/github.com/hashicorp/golang-lru/v2/arc.go

@@ -1,258 +0,0 @@
-package lru
-
-import (
-	"sync"
-
-	"github.com/hashicorp/golang-lru/v2/simplelru"
-)
-
-// ARCCache is a thread-safe fixed size Adaptive Replacement Cache (ARC).
-// ARC is an enhancement over the standard LRU cache in that tracks both
-// frequency and recency of use. This avoids a burst in access to new
-// entries from evicting the frequently used older entries. It adds some
-// additional tracking overhead to a standard LRU cache, computationally
-// it is roughly 2x the cost, and the extra memory overhead is linear
-// with the size of the cache. ARC has been patented by IBM, but is
-// similar to the TwoQueueCache (2Q) which requires setting parameters.
-type ARCCache[K comparable, V any] struct {
-	size int // Size is the total capacity of the cache
-	p    int // P is the dynamic preference towards T1 or T2
-
-	t1 simplelru.LRUCache[K, V] // T1 is the LRU for recently accessed items
-	b1 simplelru.LRUCache[K, V] // B1 is the LRU for evictions from t1
-
-	t2 simplelru.LRUCache[K, V] // T2 is the LRU for frequently accessed items
-	b2 simplelru.LRUCache[K, V] // B2 is the LRU for evictions from t2
-
-	lock sync.RWMutex
-}
-
-// NewARC creates an ARC of the given size
-func NewARC[K comparable, V any](size int) (*ARCCache[K, V], error) {
-	// Create the sub LRUs
-	b1, err := simplelru.NewLRU[K, V](size, nil)
-	if err != nil {
-		return nil, err
-	}
-	b2, err := simplelru.NewLRU[K, V](size, nil)
-	if err != nil {
-		return nil, err
-	}
-	t1, err := simplelru.NewLRU[K, V](size, nil)
-	if err != nil {
-		return nil, err
-	}
-	t2, err := simplelru.NewLRU[K, V](size, nil)
-	if err != nil {
-		return nil, err
-	}
-
-	// Initialize the ARC
-	c := &ARCCache[K, V]{
-		size: size,
-		p:    0,
-		t1:   t1,
-		b1:   b1,
-		t2:   t2,
-		b2:   b2,
-	}
-	return c, nil
-}
-
-// Get looks up a key's value from the cache.
-func (c *ARCCache[K, V]) Get(key K) (value V, ok bool) {
-	c.lock.Lock()
-	defer c.lock.Unlock()
-
-	// If the value is contained in T1 (recent), then
-	// promote it to T2 (frequent)
-	if val, ok := c.t1.Peek(key); ok {
-		c.t1.Remove(key)
-		c.t2.Add(key, val)
-		return val, ok
-	}
-
-	// Check if the value is contained in T2 (frequent)
-	if val, ok := c.t2.Get(key); ok {
-		return val, ok
-	}
-
-	// No hit
-	return
-}
-
-// Add adds a value to the cache.
-func (c *ARCCache[K, V]) Add(key K, value V) {
-	c.lock.Lock()
-	defer c.lock.Unlock()
-
-	// Check if the value is contained in T1 (recent), and potentially
-	// promote it to frequent T2
-	if c.t1.Contains(key) {
-		c.t1.Remove(key)
-		c.t2.Add(key, value)
-		return
-	}
-
-	// Check if the value is already in T2 (frequent) and update it
-	if c.t2.Contains(key) {
-		c.t2.Add(key, value)
-		return
-	}
-
-	// Check if this value was recently evicted as part of the
-	// recently used list
-	if c.b1.Contains(key) {
-		// T1 set is too small, increase P appropriately
-		delta := 1
-		b1Len := c.b1.Len()
-		b2Len := c.b2.Len()
-		if b2Len > b1Len {
-			delta = b2Len / b1Len
-		}
-		if c.p+delta >= c.size {
-			c.p = c.size
-		} else {
-			c.p += delta
-		}
-
-		// Potentially need to make room in the cache
-		if c.t1.Len()+c.t2.Len() >= c.size {
-			c.replace(false)
-		}
-
-		// Remove from B1
-		c.b1.Remove(key)
-
-		// Add the key to the frequently used list
-		c.t2.Add(key, value)
-		return
-	}
-
-	// Check if this value was recently evicted as part of the
-	// frequently used list
-	if c.b2.Contains(key) {
-		// T2 set is too small, decrease P appropriately
-		delta := 1
-		b1Len := c.b1.Len()
-		b2Len := c.b2.Len()
-		if b1Len > b2Len {
-			delta = b1Len / b2Len
-		}
-		if delta >= c.p {
-			c.p = 0
-		} else {
-			c.p -= delta
-		}
-
-		// Potentially need to make room in the cache
-		if c.t1.Len()+c.t2.Len() >= c.size {
-			c.replace(true)
-		}
-
-		// Remove from B2
-		c.b2.Remove(key)
-
-		// Add the key to the frequently used list
-		c.t2.Add(key, value)
-		return
-	}
-
-	// Potentially need to make room in the cache
-	if c.t1.Len()+c.t2.Len() >= c.size {
-		c.replace(false)
-	}
-
-	// Keep the size of the ghost buffers trim
-	if c.b1.Len() > c.size-c.p {
-		c.b1.RemoveOldest()
-	}
-	if c.b2.Len() > c.p {
-		c.b2.RemoveOldest()
-	}
-
-	// Add to the recently seen list
-	c.t1.Add(key, value)
-}
-
-// replace is used to adaptively evict from either T1 or T2
-// based on the current learned value of P
-func (c *ARCCache[K, V]) replace(b2ContainsKey bool) {
-	t1Len := c.t1.Len()
-	if t1Len > 0 && (t1Len > c.p || (t1Len == c.p && b2ContainsKey)) {
-		k, _, ok := c.t1.RemoveOldest()
-		if ok {
-			var empty V
-			c.b1.Add(k, empty)
-		}
-	} else {
-		k, _, ok := c.t2.RemoveOldest()
-		if ok {
-			var empty V
-			c.b2.Add(k, empty)
-		}
-	}
-}
-
-// Len returns the number of cached entries
-func (c *ARCCache[K, V]) Len() int {
-	c.lock.RLock()
-	defer c.lock.RUnlock()
-	return c.t1.Len() + c.t2.Len()
-}
-
-// Keys returns all the cached keys
-func (c *ARCCache[K, V]) Keys() []K {
-	c.lock.RLock()
-	defer c.lock.RUnlock()
-	k1 := c.t1.Keys()
-	k2 := c.t2.Keys()
-	return append(k1, k2...)
-}
-
-// Remove is used to purge a key from the cache
-func (c *ARCCache[K, V]) Remove(key K) {
-	c.lock.Lock()
-	defer c.lock.Unlock()
-	if c.t1.Remove(key) {
-		return
-	}
-	if c.t2.Remove(key) {
-		return
-	}
-	if c.b1.Remove(key) {
-		return
-	}
-	if c.b2.Remove(key) {
-		return
-	}
-}
-
-// Purge is used to clear the cache
-func (c *ARCCache[K, V]) Purge() {
-	c.lock.Lock()
-	defer c.lock.Unlock()
-	c.t1.Purge()
-	c.t2.Purge()
-	c.b1.Purge()
-	c.b2.Purge()
-}
-
-// Contains is used to check if the cache contains a key
-// without updating recency or frequency.
-func (c *ARCCache[K, V]) Contains(key K) bool {
-	c.lock.RLock()
-	defer c.lock.RUnlock()
-	return c.t1.Contains(key) || c.t2.Contains(key)
-}
-
-// Peek is used to inspect the cache value of a key
-// without updating recency or frequency.
-func (c *ARCCache[K, V]) Peek(key K) (value V, ok bool) {
-	c.lock.RLock()
-	defer c.lock.RUnlock()
-	if val, ok := c.t1.Peek(key); ok {
-		return val, ok
-	}
-	return c.t2.Peek(key)
-}

vendor/github.com/hashicorp/golang-lru/v2/doc.go

@@ -1,20 +1,23 @@
+// Copyright (c) HashiCorp, Inc.
+// SPDX-License-Identifier: MPL-2.0
+
 // Package lru provides three different LRU caches of varying sophistication.
 //
-// Cache is a simple LRU cache. It is based on the
-// LRU implementation in groupcache:
-// https://github.com/golang/groupcache/tree/master/lru
+// Cache is a simple LRU cache. It is based on the LRU implementation in
+// groupcache: https://github.com/golang/groupcache/tree/master/lru
 //
 // TwoQueueCache tracks frequently used and recently used entries separately.
-// This avoids a burst of accesses from taking out frequently used entries,
-// at the cost of about 2x computational overhead and some extra bookkeeping.
+// This avoids a burst of accesses from taking out frequently used entries, at
+// the cost of about 2x computational overhead and some extra bookkeeping.
 //
-// ARCCache is an adaptive replacement cache. It tracks recent evictions as
-// well as recent usage in both the frequent and recent caches. Its
-// computational overhead is comparable to TwoQueueCache, but the memory
-// overhead is linear with the size of the cache.
+// ARCCache is an adaptive replacement cache. It tracks recent evictions as well
+// as recent usage in both the frequent and recent caches. Its computational
+// overhead is comparable to TwoQueueCache, but the memory overhead is linear
+// with the size of the cache.
 //
 // ARC has been patented by IBM, so do not use it if that is problematic for
-// your program.
+// your program. For this reason, it is in a separate go module contained within
+// this repository.
 //
 // All caches in this package take locks while operating, and are therefore
 // thread-safe for consumers.

vendor/github.com/hashicorp/golang-lru/v2/internal/list.go

@@ -0,0 +1,142 @@
+// Copyright 2009 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE_list file.
+
+package internal
+
+import "time"
+
+// Entry is an LRU Entry
+type Entry[K comparable, V any] struct {
+	// Next and previous pointers in the doubly-linked list of elements.
+	// To simplify the implementation, internally a list l is implemented
+	// as a ring, such that &l.root is both the next element of the last
+	// list element (l.Back()) and the previous element of the first list
+	// element (l.Front()).
+	next, prev *Entry[K, V]
+
+	// The list to which this element belongs.
+	list *LruList[K, V]
+
+	// The LRU Key of this element.
+	Key K
+
+	// The Value stored with this element.
+	Value V
+
+	// The time this element would be cleaned up, optional
+	ExpiresAt time.Time
+
+	// The expiry bucket item was put in, optional
+	ExpireBucket uint8
+}
+
+// PrevEntry returns the previous list element or nil.
+func (e *Entry[K, V]) PrevEntry() *Entry[K, V] {
+	if p := e.prev; e.list != nil && p != &e.list.root {
+		return p
+	}
+	return nil
+}
+
+// LruList represents a doubly linked list.
+// The zero Value for LruList is an empty list ready to use.
+type LruList[K comparable, V any] struct {
+	root Entry[K, V] // sentinel list element, only &root, root.prev, and root.next are used
+	len  int         // current list Length excluding (this) sentinel element
+}
+
+// Init initializes or clears list l.
+func (l *LruList[K, V]) Init() *LruList[K, V] {
+	l.root.next = &l.root
+	l.root.prev = &l.root
+	l.len = 0
+	return l
+}
+
+// NewList returns an initialized list.
+func NewList[K comparable, V any]() *LruList[K, V] { return new(LruList[K, V]).Init() }
+
+// Length returns the number of elements of list l.
+// The complexity is O(1).
+func (l *LruList[K, V]) Length() int { return l.len }
+
+// Back returns the last element of list l or nil if the list is empty.
+func (l *LruList[K, V]) Back() *Entry[K, V] {
+	if l.len == 0 {
+		return nil
+	}
+	return l.root.prev
+}
+
+// lazyInit lazily initializes a zero List Value.
+func (l *LruList[K, V]) lazyInit() {
+	if l.root.next == nil {
+		l.Init()
+	}
+}
+
+// insert inserts e after at, increments l.len, and returns e.
+func (l *LruList[K, V]) insert(e, at *Entry[K, V]) *Entry[K, V] {
+	e.prev = at
+	e.next = at.next
+	e.prev.next = e
+	e.next.prev = e
+	e.list = l
+	l.len++
+	return e
+}
+
+// insertValue is a convenience wrapper for insert(&Entry{Value: v, ExpiresAt: ExpiresAt}, at).
+func (l *LruList[K, V]) insertValue(k K, v V, expiresAt time.Time, at *Entry[K, V]) *Entry[K, V] {
+	return l.insert(&Entry[K, V]{Value: v, Key: k, ExpiresAt: expiresAt}, at)
+}
+
+// Remove removes e from its list, decrements l.len
+func (l *LruList[K, V]) Remove(e *Entry[K, V]) V {
+	e.prev.next = e.next
+	e.next.prev = e.prev
+	e.next = nil // avoid memory leaks
+	e.prev = nil // avoid memory leaks
+	e.list = nil
+	l.len--
+
+	return e.Value
+}
+
+// move moves e to next to at.
+func (l *LruList[K, V]) move(e, at *Entry[K, V]) {
+	if e == at {
+		return
+	}
+	e.prev.next = e.next
+	e.next.prev = e.prev
+
+	e.prev = at
+	e.next = at.next
+	e.prev.next = e
+	e.next.prev = e
+}
+
+// PushFront inserts a new element e with value v at the front of list l and returns e.
+func (l *LruList[K, V]) PushFront(k K, v V) *Entry[K, V] {
+	l.lazyInit()
+	return l.insertValue(k, v, time.Time{}, &l.root)
+}
+
+// PushFrontExpirable inserts a new expirable element e with Value v at the front of list l and returns e.
+func (l *LruList[K, V]) PushFrontExpirable(k K, v V, expiresAt time.Time) *Entry[K, V] {
+	l.lazyInit()
+	return l.insertValue(k, v, expiresAt, &l.root)
+}
+
+// MoveToFront moves element e to the front of list l.
+// If e is not an element of l, the list is not modified.
+// The element must not be nil.
+func (l *LruList[K, V]) MoveToFront(e *Entry[K, V]) {
+	if e.list != l || l.root.next == e {
+		return
+	}
+	// see comment in List.Remove about initialization of l
+	l.move(e, &l.root)
+}

vendor/github.com/hashicorp/golang-lru/v2/lru.go

@@ -1,3 +1,6 @@
+// Copyright (c) HashiCorp, Inc.
+// SPDX-License-Identifier: MPL-2.0
+
 package lru
 
 import (
@@ -230,6 +233,14 @@ func (c *Cache[K, V]) Keys() []K {
 	return keys
 }
 
+// Values returns a slice of the values in the cache, from oldest to newest.
+func (c *Cache[K, V]) Values() []V {
+	c.lock.RLock()
+	values := c.lru.Values()
+	c.lock.RUnlock()
+	return values
+}
+
 // Len returns the number of items in the cache.
 func (c *Cache[K, V]) Len() int {
 	c.lock.RLock()

vendor/github.com/hashicorp/golang-lru/v2/simplelru/list.go

@@ -1,128 +0,0 @@
-// Copyright 2009 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE_list file.
-
-package simplelru
-
-// entry is an LRU entry
-type entry[K comparable, V any] struct {
-	// Next and previous pointers in the doubly-linked list of elements.
-	// To simplify the implementation, internally a list l is implemented
-	// as a ring, such that &l.root is both the next element of the last
-	// list element (l.Back()) and the previous element of the first list
-	// element (l.Front()).
-	next, prev *entry[K, V]
-
-	// The list to which this element belongs.
-	list *lruList[K, V]
-
-	// The LRU key of this element.
-	key K
-
-	// The value stored with this element.
-	value V
-}
-
-// prevEntry returns the previous list element or nil.
-func (e *entry[K, V]) prevEntry() *entry[K, V] {
-	if p := e.prev; e.list != nil && p != &e.list.root {
-		return p
-	}
-	return nil
-}
-
-// lruList represents a doubly linked list.
-// The zero value for lruList is an empty list ready to use.
-type lruList[K comparable, V any] struct {
-	root entry[K, V] // sentinel list element, only &root, root.prev, and root.next are used
-	len  int         // current list length excluding (this) sentinel element
-}
-
-// init initializes or clears list l.
-func (l *lruList[K, V]) init() *lruList[K, V] {
-	l.root.next = &l.root
-	l.root.prev = &l.root
-	l.len = 0
-	return l
-}
-
-// newList returns an initialized list.
-func newList[K comparable, V any]() *lruList[K, V] { return new(lruList[K, V]).init() }
-
-// length returns the number of elements of list l.
-// The complexity is O(1).
-func (l *lruList[K, V]) length() int { return l.len }
-
-// back returns the last element of list l or nil if the list is empty.
-func (l *lruList[K, V]) back() *entry[K, V] {
-	if l.len == 0 {
-		return nil
-	}
-	return l.root.prev
-}
-
-// lazyInit lazily initializes a zero List value.
-func (l *lruList[K, V]) lazyInit() {
-	if l.root.next == nil {
-		l.init()
-	}
-}
-
-// insert inserts e after at, increments l.len, and returns e.
-func (l *lruList[K, V]) insert(e, at *entry[K, V]) *entry[K, V] {
-	e.prev = at
-	e.next = at.next
-	e.prev.next = e
-	e.next.prev = e
-	e.list = l
-	l.len++
-	return e
-}
-
-// insertValue is a convenience wrapper for insert(&Element{Value: v}, at).
-func (l *lruList[K, V]) insertValue(k K, v V, at *entry[K, V]) *entry[K, V] {
-	return l.insert(&entry[K, V]{value: v, key: k}, at)
-}
-
-// remove removes e from its list, decrements l.len
-func (l *lruList[K, V]) remove(e *entry[K, V]) V {
-	e.prev.next = e.next
-	e.next.prev = e.prev
-	e.next = nil // avoid memory leaks
-	e.prev = nil // avoid memory leaks
-	e.list = nil
-	l.len--
-
-	return e.value
-}
-
-// move moves e to next to at.
-func (l *lruList[K, V]) move(e, at *entry[K, V]) {
-	if e == at {
-		return
-	}
-	e.prev.next = e.next
-	e.next.prev = e.prev
-
-	e.prev = at
-	e.next = at.next
-	e.prev.next = e
-	e.next.prev = e
-}
-
-// pushFront inserts a new element e with value v at the front of list l and returns e.
-func (l *lruList[K, V]) pushFront(k K, v V) *entry[K, V] {
-	l.lazyInit()
-	return l.insertValue(k, v, &l.root)
-}
-
-// moveToFront moves element e to the front of list l.
-// If e is not an element of l, the list is not modified.
-// The element must not be nil.
-func (l *lruList[K, V]) moveToFront(e *entry[K, V]) {
-	if e.list != l || l.root.next == e {
-		return
-	}
-	// see comment in List.Remove about initialization of l
-	l.move(e, &l.root)
-}

vendor/github.com/hashicorp/golang-lru/v2/simplelru/lru.go

@@ -1,7 +1,12 @@
+// Copyright (c) HashiCorp, Inc.
+// SPDX-License-Identifier: MPL-2.0
+
 package simplelru
 
 import (
 	"errors"
+
+	"github.com/hashicorp/golang-lru/v2/internal"
 )
 
 // EvictCallback is used to get a callback when a cache entry is evicted
@@ -10,8 +15,8 @@ type EvictCallback[K comparable, V any] func(key K, value V)
 // LRU implements a non-thread safe fixed size LRU cache
 type LRU[K comparable, V any] struct {
 	size      int
-	evictList *lruList[K, V]
-	items     map[K]*entry[K, V]
+	evictList *internal.LruList[K, V]
+	items     map[K]*internal.Entry[K, V]
 	onEvict   EvictCallback[K, V]
 }
 
@@ -23,8 +28,8 @@ func NewLRU[K comparable, V any](size int, onEvict EvictCallback[K, V]) (*LRU[K,
 
 	c := &LRU[K, V]{
 		size:      size,
-		evictList: newList[K, V](),
-		items:     make(map[K]*entry[K, V]),
+		evictList: internal.NewList[K, V](),
+		items:     make(map[K]*internal.Entry[K, V]),
 		onEvict:   onEvict,
 	}
 	return c, nil
@@ -34,27 +39,27 @@ func NewLRU[K comparable, V any](size int, onEvict EvictCallback[K, V]) (*LRU[K,
 func (c *LRU[K, V]) Purge() {
 	for k, v := range c.items {
 		if c.onEvict != nil {
-			c.onEvict(k, v.value)
+			c.onEvict(k, v.Value)
 		}
 		delete(c.items, k)
 	}
-	c.evictList.init()
+	c.evictList.Init()
 }
 
 // Add adds a value to the cache.  Returns true if an eviction occurred.
 func (c *LRU[K, V]) Add(key K, value V) (evicted bool) {
 	// Check for existing item
 	if ent, ok := c.items[key]; ok {
-		c.evictList.moveToFront(ent)
-		ent.value = value
+		c.evictList.MoveToFront(ent)
+		ent.Value = value
 		return false
 	}
 
 	// Add new item
-	ent := c.evictList.pushFront(key, value)
+	ent := c.evictList.PushFront(key, value)
 	c.items[key] = ent
 
-	evict := c.evictList.length() > c.size
+	evict := c.evictList.Length() > c.size
 	// Verify size not exceeded
 	if evict {
 		c.removeOldest()
@@ -65,8 +70,8 @@ func (c *LRU[K, V]) Add(key K, value V) (evicted bool) {
 // Get looks up a key's value from the cache.
 func (c *LRU[K, V]) Get(key K) (value V, ok bool) {
 	if ent, ok := c.items[key]; ok {
-		c.evictList.moveToFront(ent)
-		return ent.value, true
+		c.evictList.MoveToFront(ent)
+		return ent.Value, true
 	}
 	return
 }
@@ -81,9 +86,9 @@ func (c *LRU[K, V]) Contains(key K) (ok bool) {
 // Peek returns the key value (or undefined if not found) without updating
 // the "recently used"-ness of the key.
 func (c *LRU[K, V]) Peek(key K) (value V, ok bool) {
-	var ent *entry[K, V]
+	var ent *internal.Entry[K, V]
 	if ent, ok = c.items[key]; ok {
-		return ent.value, true
+		return ent.Value, true
 	}
 	return
 }
@@ -100,37 +105,46 @@ func (c *LRU[K, V]) Remove(key K) (present bool) {
 
 // RemoveOldest removes the oldest item from the cache.
 func (c *LRU[K, V]) RemoveOldest() (key K, value V, ok bool) {
-	ent := c.evictList.back()
-	if ent != nil {
+	if ent := c.evictList.Back(); ent != nil {
 		c.removeElement(ent)
-		return ent.key, ent.value, true
+		return ent.Key, ent.Value, true
 	}
 	return
 }
 
 // GetOldest returns the oldest entry
 func (c *LRU[K, V]) GetOldest() (key K, value V, ok bool) {
-	ent := c.evictList.back()
-	if ent != nil {
-		return ent.key, ent.value, true
+	if ent := c.evictList.Back(); ent != nil {
+		return ent.Key, ent.Value, true
 	}
 	return
 }
 
 // Keys returns a slice of the keys in the cache, from oldest to newest.
 func (c *LRU[K, V]) Keys() []K {
-	keys := make([]K, c.evictList.length())
+	keys := make([]K, c.evictList.Length())
 	i := 0
-	for ent := c.evictList.back(); ent != nil; ent = ent.prevEntry() {
-		keys[i] = ent.key
+	for ent := c.evictList.Back(); ent != nil; ent = ent.PrevEntry() {
+		keys[i] = ent.Key
 		i++
 	}
 	return keys
 }
 
+// Values returns a slice of the values in the cache, from oldest to newest.
+func (c *LRU[K, V]) Values() []V {
+	values := make([]V, len(c.items))
+	i := 0
+	for ent := c.evictList.Back(); ent != nil; ent = ent.PrevEntry() {
+		values[i] = ent.Value
+		i++
+	}
+	return values
+}
+
 // Len returns the number of items in the cache.
 func (c *LRU[K, V]) Len() int {
-	return c.evictList.length()
+	return c.evictList.Length()
 }
 
 // Resize changes the cache size.
@@ -148,17 +162,16 @@ func (c *LRU[K, V]) Resize(size int) (evicted int) {
 
 // removeOldest removes the oldest item from the cache.
 func (c *LRU[K, V]) removeOldest() {
-	ent := c.evictList.back()
-	if ent != nil {
+	if ent := c.evictList.Back(); ent != nil {
 		c.removeElement(ent)
 	}
 }
 
 // removeElement is used to remove a given list element from the cache
-func (c *LRU[K, V]) removeElement(e *entry[K, V]) {
-	c.evictList.remove(e)
-	delete(c.items, e.key)
+func (c *LRU[K, V]) removeElement(e *internal.Entry[K, V]) {
+	c.evictList.Remove(e)
+	delete(c.items, e.Key)
 	if c.onEvict != nil {
-		c.onEvict(e.key, e.value)
+		c.onEvict(e.Key, e.Value)
 	}
 }

vendor/github.com/hashicorp/golang-lru/v2/simplelru/lru_interface.go

@@ -1,3 +1,6 @@
+// Copyright (c) HashiCorp, Inc.
+// SPDX-License-Identifier: MPL-2.0
+
 // Package simplelru provides simple LRU implementation based on build-in container/list.
 package simplelru
 
@@ -29,6 +32,9 @@ type LRUCache[K comparable, V any] interface {
 	// Returns a slice of the keys in the cache, from oldest to newest.
 	Keys() []K
 
+	// Values returns a slice of the values in the cache, from oldest to newest.
+	Values() []V
+
 	// Returns the number of items in the cache.
 	Len() int
 

vendor/github.com/hashicorp/golang-lru/v2/testing.go

@@ -1,16 +0,0 @@
-package lru
-
-import (
-	"crypto/rand"
-	"math"
-	"math/big"
-	"testing"
-)
-
-func getRand(tb testing.TB) int64 {
-	out, err := rand.Int(rand.Reader, big.NewInt(math.MaxInt64))
-	if err != nil {
-		tb.Fatal(err)
-	}
-	return out.Int64()
-}

vendor/github.com/mattn/go-isatty/isatty_bsd.go

@@ -1,6 +1,7 @@
-//go:build (darwin || freebsd || openbsd || netbsd || dragonfly || hurd) && !appengine
+//go:build (darwin || freebsd || openbsd || netbsd || dragonfly || hurd) && !appengine && !tinygo
 // +build darwin freebsd openbsd netbsd dragonfly hurd
 // +build !appengine
+// +build !tinygo
 
 package isatty
 

vendor/github.com/mattn/go-isatty/isatty_others.go

@@ -1,5 +1,6 @@
-//go:build appengine || js || nacl || wasm
-// +build appengine js nacl wasm
+//go:build (appengine || js || nacl || tinygo || wasm) && !windows
+// +build appengine js nacl tinygo wasm
+// +build !windows
 
 package isatty
 

vendor/github.com/mattn/go-isatty/isatty_tcgets.go

@@ -1,6 +1,7 @@
-//go:build (linux || aix || zos) && !appengine
+//go:build (linux || aix || zos) && !appengine && !tinygo
 // +build linux aix zos
 // +build !appengine
+// +build !tinygo
 
 package isatty
 

vendor/github.com/mattn/go-sqlite3/sqlite3-binding.h

@@ -147,9 +147,9 @@ extern "C" {
 ** [sqlite3_libversion_number()], [sqlite3_sourceid()],
 ** [sqlite_version()] and [sqlite_source_id()].
 */
-#define SQLITE_VERSION        "3.42.0"
-#define SQLITE_VERSION_NUMBER 3042000
-#define SQLITE_SOURCE_ID      "2023-05-16 12:36:15 831d0fb2836b71c9bc51067c49fee4b8f18047814f2ff22d817d25195cf350b0"
+#define SQLITE_VERSION        "3.44.0"
+#define SQLITE_VERSION_NUMBER 3044000
+#define SQLITE_SOURCE_ID      "2023-11-01 11:23:50 17129ba1ff7f0daf37100ee82d507aef7827cf38de1866e2633096ae6ad81301"
 
 /*
 ** CAPI3REF: Run-Time Library Version Numbers
@@ -529,6 +529,7 @@ SQLITE_API int sqlite3_exec(
 #define SQLITE_IOERR_ROLLBACK_ATOMIC   (SQLITE_IOERR | (31<<8))
 #define SQLITE_IOERR_DATA              (SQLITE_IOERR | (32<<8))
 #define SQLITE_IOERR_CORRUPTFS         (SQLITE_IOERR | (33<<8))
+#define SQLITE_IOERR_IN_PAGE           (SQLITE_IOERR | (34<<8))
 #define SQLITE_LOCKED_SHAREDCACHE      (SQLITE_LOCKED |  (1<<8))
 #define SQLITE_LOCKED_VTAB             (SQLITE_LOCKED |  (2<<8))
 #define SQLITE_BUSY_RECOVERY           (SQLITE_BUSY   |  (1<<8))
@@ -1191,7 +1192,7 @@ struct sqlite3_io_methods {
 ** by clients within the current process, only within other processes.
 **
 ** <li>[[SQLITE_FCNTL_CKSM_FILE]]
-** The [SQLITE_FCNTL_CKSM_FILE] opcode is for use interally by the
+** The [SQLITE_FCNTL_CKSM_FILE] opcode is for use internally by the
 ** [checksum VFS shim] only.
 **
 ** <li>[[SQLITE_FCNTL_RESET_CACHE]]
@@ -2127,7 +2128,7 @@ struct sqlite3_mem_methods {
 ** is stored in each sorted record and the required column values loaded
 ** from the database as records are returned in sorted order. The default
 ** value for this option is to never use this optimization. Specifying a
-** negative value for this option restores the default behaviour.
+** negative value for this option restores the default behavior.
 ** This option is only available if SQLite is compiled with the
 ** [SQLITE_ENABLE_SORTER_REFERENCES] compile-time option.
 **
@@ -2302,7 +2303,7 @@ struct sqlite3_mem_methods {
 ** database handle, SQLite checks if this will mean that there are now no
 ** connections at all to the database. If so, it performs a checkpoint
 ** operation before closing the connection. This option may be used to
-** override this behaviour. The first parameter passed to this operation
+** override this behavior. The first parameter passed to this operation
 ** is an integer - positive to disable checkpoints-on-close, or zero (the
 ** default) to enable them, and negative to leave the setting unchanged.
 ** The second parameter is a pointer to an integer
@@ -2455,7 +2456,7 @@ struct sqlite3_mem_methods {
 ** the [VACUUM] command will fail with an obscure error when attempting to
 ** process a table with generated columns and a descending index.  This is
 ** not considered a bug since SQLite versions 3.3.0 and earlier do not support
-** either generated columns or decending indexes.
+** either generated columns or descending indexes.
 ** </dd>
 **
 ** [[SQLITE_DBCONFIG_STMT_SCANSTATUS]]
@@ -2736,6 +2737,7 @@ SQLITE_API sqlite3_int64 sqlite3_total_changes64(sqlite3*);
 **
 ** ^The [sqlite3_is_interrupted(D)] interface can be used to determine whether
 ** or not an interrupt is currently in effect for [database connection] D.
+** It returns 1 if an interrupt is currently in effect, or 0 otherwise.
 */
 SQLITE_API void sqlite3_interrupt(sqlite3*);
 SQLITE_API int sqlite3_is_interrupted(sqlite3*);
@@ -3389,8 +3391,10 @@ SQLITE_API SQLITE_DEPRECATED void *sqlite3_profile(sqlite3*,
 ** M argument should be the bitwise OR-ed combination of
 ** zero or more [SQLITE_TRACE] constants.
 **
-** ^Each call to either sqlite3_trace() or sqlite3_trace_v2() overrides
-** (cancels) any prior calls to sqlite3_trace() or sqlite3_trace_v2().
+** ^Each call to either sqlite3_trace(D,X,P) or sqlite3_trace_v2(D,M,X,P)
+** overrides (cancels) all prior calls to sqlite3_trace(D,X,P) or
+** sqlite3_trace_v2(D,M,X,P) for the [database connection] D.  Each
+** database connection may have at most one trace callback.
 **
 ** ^The X callback is invoked whenever any of the events identified by
 ** mask M occur.  ^The integer return value from the callback is currently
@@ -3759,7 +3763,7 @@ SQLITE_API int sqlite3_open_v2(
 ** as F) must be one of:
 ** <ul>
 ** <li> A database filename pointer created by the SQLite core and
-** passed into the xOpen() method of a VFS implemention, or
+** passed into the xOpen() method of a VFS implementation, or
 ** <li> A filename obtained from [sqlite3_db_filename()], or
 ** <li> A new filename constructed using [sqlite3_create_filename()].
 ** </ul>
@@ -3872,7 +3876,7 @@ SQLITE_API sqlite3_file *sqlite3_database_file_object(const char*);
 /*
 ** CAPI3REF: Create and Destroy VFS Filenames
 **
-** These interfces are provided for use by [VFS shim] implementations and
+** These interfaces are provided for use by [VFS shim] implementations and
 ** are not useful outside of that context.
 **
 ** The sqlite3_create_filename(D,J,W,N,P) allocates memory to hold a version of
@@ -3952,6 +3956,7 @@ SQLITE_API void sqlite3_free_filename(sqlite3_filename);
 **
 ** ^The sqlite3_errmsg() and sqlite3_errmsg16() return English-language
 ** text that describes the error, as either UTF-8 or UTF-16 respectively.
+** (See how SQLite handles [invalid UTF] for exceptions to this rule.)
 ** ^(Memory to hold the error message string is managed internally.
 ** The application does not need to worry about freeing the result.
 ** However, the error string might be overwritten or deallocated by
@@ -4419,6 +4424,41 @@ SQLITE_API int sqlite3_stmt_readonly(sqlite3_stmt *pStmt);
 */
 SQLITE_API int sqlite3_stmt_isexplain(sqlite3_stmt *pStmt);
 
+/*
+** CAPI3REF: Change The EXPLAIN Setting For A Prepared Statement
+** METHOD: sqlite3_stmt
+**
+** The sqlite3_stmt_explain(S,E) interface changes the EXPLAIN
+** setting for [prepared statement] S.  If E is zero, then S becomes
+** a normal prepared statement.  If E is 1, then S behaves as if
+** its SQL text began with "[EXPLAIN]".  If E is 2, then S behaves as if
+** its SQL text began with "[EXPLAIN QUERY PLAN]".
+**
+** Calling sqlite3_stmt_explain(S,E) might cause S to be reprepared.
+** SQLite tries to avoid a reprepare, but a reprepare might be necessary
+** on the first transition into EXPLAIN or EXPLAIN QUERY PLAN mode.
+**
+** Because of the potential need to reprepare, a call to
+** sqlite3_stmt_explain(S,E) will fail with SQLITE_ERROR if S cannot be
+** reprepared because it was created using [sqlite3_prepare()] instead of
+** the newer [sqlite3_prepare_v2()] or [sqlite3_prepare_v3()] interfaces and
+** hence has no saved SQL text with which to reprepare.
+**
+** Changing the explain setting for a prepared statement does not change
+** the original SQL text for the statement.  Hence, if the SQL text originally
+** began with EXPLAIN or EXPLAIN QUERY PLAN, but sqlite3_stmt_explain(S,0)
+** is called to convert the statement into an ordinary statement, the EXPLAIN
+** or EXPLAIN QUERY PLAN keywords will still appear in the sqlite3_sql(S)
+** output, even though the statement now acts like a normal SQL statement.
+**
+** This routine returns SQLITE_OK if the explain mode is successfully
+** changed, or an error code if the explain mode could not be changed.
+** The explain mode cannot be changed while a statement is active.
+** Hence, it is good practice to call [sqlite3_reset(S)]
+** immediately prior to calling sqlite3_stmt_explain(S,E).
+*/
+SQLITE_API int sqlite3_stmt_explain(sqlite3_stmt *pStmt, int eMode);
+
 /*
 ** CAPI3REF: Determine If A Prepared Statement Has Been Reset
 ** METHOD: sqlite3_stmt
@@ -4582,7 +4622,7 @@ typedef struct sqlite3_context sqlite3_context;
 ** with it may be passed. ^It is called to dispose of the BLOB or string even
 ** if the call to the bind API fails, except the destructor is not called if
 ** the third parameter is a NULL pointer or the fourth parameter is negative.
-** ^ (2) The special constant, [SQLITE_STATIC], may be passsed to indicate that
+** ^ (2) The special constant, [SQLITE_STATIC], may be passed to indicate that
 ** the application remains responsible for disposing of the object. ^In this
 ** case, the object and the provided pointer to it must remain valid until
 ** either the prepared statement is finalized or the same SQL parameter is
@@ -5261,20 +5301,33 @@ SQLITE_API int sqlite3_finalize(sqlite3_stmt *pStmt);
 ** ^The [sqlite3_reset(S)] interface resets the [prepared statement] S
 ** back to the beginning of its program.
 **
-** ^If the most recent call to [sqlite3_step(S)] for the
-** [prepared statement] S returned [SQLITE_ROW] or [SQLITE_DONE],
-** or if [sqlite3_step(S)] has never before been called on S,
-** then [sqlite3_reset(S)] returns [SQLITE_OK].
+** ^The return code from [sqlite3_reset(S)] indicates whether or not
+** the previous evaluation of prepared statement S completed successfully.
+** ^If [sqlite3_step(S)] has never before been called on S or if
+** [sqlite3_step(S)] has not been called since the previous call
+** to [sqlite3_reset(S)], then [sqlite3_reset(S)] will return
+** [SQLITE_OK].
 **
 ** ^If the most recent call to [sqlite3_step(S)] for the
 ** [prepared statement] S indicated an error, then
 ** [sqlite3_reset(S)] returns an appropriate [error code].
+** ^The [sqlite3_reset(S)] interface might also return an [error code]
+** if there were no prior errors but the process of resetting
+** the prepared statement caused a new error. ^For example, if an
+** [INSERT] statement with a [RETURNING] clause is only stepped one time,
+** that one call to [sqlite3_step(S)] might return SQLITE_ROW but
+** the overall statement might still fail and the [sqlite3_reset(S)] call
+** might return SQLITE_BUSY if locking constraints prevent the
+** database change from committing.  Therefore, it is important that
+** applications check the return code from [sqlite3_reset(S)] even if
+** no prior call to [sqlite3_step(S)] indicated a problem.
 **
 ** ^The [sqlite3_reset(S)] interface does not change the values
 ** of any [sqlite3_bind_blob|bindings] on the [prepared statement] S.
 */
 SQLITE_API int sqlite3_reset(sqlite3_stmt *pStmt);
 
+
 /*
 ** CAPI3REF: Create Or Redefine SQL Functions
 ** KEYWORDS: {function creation routines}
@@ -5485,7 +5538,7 @@ SQLITE_API int sqlite3_create_window_function(
 ** [application-defined SQL function]
 ** that has side-effects or that could potentially leak sensitive information.
 ** This will prevent attacks in which an application is tricked
-** into using a database file that has had its schema surreptiously
+** into using a database file that has had its schema surreptitiously
 ** modified to invoke the application-defined function in ways that are
 ** harmful.
 ** <p>
@@ -5829,32 +5882,32 @@ SQLITE_API sqlite3 *sqlite3_context_db_handle(sqlite3_context*);
 ** METHOD: sqlite3_context
 **
 ** These functions may be used by (non-aggregate) SQL functions to
-** associate metadata with argument values. If the same value is passed to
-** multiple invocations of the same SQL function during query execution, under
-** some circumstances the associated metadata may be preserved.  An example
-** of where this might be useful is in a regular-expression matching
-** function. The compiled version of the regular expression can be stored as
-** metadata associated with the pattern string.
+** associate auxiliary data with argument values. If the same argument
+** value is passed to multiple invocations of the same SQL function during
+** query execution, under some circumstances the associated auxiliary data
+** might be preserved.  An example of where this might be useful is in a
+** regular-expression matching function. The compiled version of the regular
+** expression can be stored as auxiliary data associated with the pattern string.
 ** Then as long as the pattern string remains the same,
 ** the compiled regular expression can be reused on multiple
 ** invocations of the same function.
 **
-** ^The sqlite3_get_auxdata(C,N) interface returns a pointer to the metadata
+** ^The sqlite3_get_auxdata(C,N) interface returns a pointer to the auxiliary data
 ** associated by the sqlite3_set_auxdata(C,N,P,X) function with the Nth argument
 ** value to the application-defined function.  ^N is zero for the left-most
-** function argument.  ^If there is no metadata
+** function argument.  ^If there is no auxiliary data
 ** associated with the function argument, the sqlite3_get_auxdata(C,N) interface
 ** returns a NULL pointer.
 **
-** ^The sqlite3_set_auxdata(C,N,P,X) interface saves P as metadata for the N-th
-** argument of the application-defined function.  ^Subsequent
+** ^The sqlite3_set_auxdata(C,N,P,X) interface saves P as auxiliary data for the
+** N-th argument of the application-defined function.  ^Subsequent
 ** calls to sqlite3_get_auxdata(C,N) return P from the most recent
-** sqlite3_set_auxdata(C,N,P,X) call if the metadata is still valid or
-** NULL if the metadata has been discarded.
+** sqlite3_set_auxdata(C,N,P,X) call if the auxiliary data is still valid or
+** NULL if the auxiliary data has been discarded.
 ** ^After each call to sqlite3_set_auxdata(C,N,P,X) where X is not NULL,
 ** SQLite will invoke the destructor function X with parameter P exactly
-** once, when the metadata is discarded.
-** SQLite is free to discard the metadata at any time, including: <ul>
+** once, when the auxiliary data is discarded.
+** SQLite is free to discard the auxiliary data at any time, including: <ul>
 ** <li> ^(when the corresponding function parameter changes)^, or
 ** <li> ^(when [sqlite3_reset()] or [sqlite3_finalize()] is called for the
 **      SQL statement)^, or
@@ -5870,7 +5923,7 @@ SQLITE_API sqlite3 *sqlite3_context_db_handle(sqlite3_context*);
 ** function implementation should not make any use of P after
 ** sqlite3_set_auxdata() has been called.
 **
-** ^(In practice, metadata is preserved between function calls for
+** ^(In practice, auxiliary data is preserved between function calls for
 ** function parameters that are compile-time constants, including literal
 ** values and [parameters] and expressions composed from the same.)^
 **
@@ -5880,10 +5933,67 @@ SQLITE_API sqlite3 *sqlite3_context_db_handle(sqlite3_context*);
 **
 ** These routines must be called from the same thread in which
 ** the SQL function is running.
+**
+** See also: [sqlite3_get_clientdata()] and [sqlite3_set_clientdata()].
 */
 SQLITE_API void *sqlite3_get_auxdata(sqlite3_context*, int N);
 SQLITE_API void sqlite3_set_auxdata(sqlite3_context*, int N, void*, void (*)(void*));
 
+/*
+** CAPI3REF: Database Connection Client Data
+** METHOD: sqlite3
+**
+** These functions are used to associate one or more named pointers
+** with a [database connection].
+** A call to sqlite3_set_clientdata(D,N,P,X) causes the pointer P
+** to be attached to [database connection] D using name N.  Subsequent
+** calls to sqlite3_get_clientdata(D,N) will return a copy of pointer P
+** or a NULL pointer if there were no prior calls to
+** sqlite3_set_clientdata() with the same values of D and N.
+** Names are compared using strcmp() and are thus case sensitive.
+**
+** If P and X are both non-NULL, then the destructor X is invoked with
+** argument P on the first of the following occurrences:
+** <ul>
+** <li> An out-of-memory error occurs during the call to
+**      sqlite3_set_clientdata() which attempts to register pointer P.
+** <li> A subsequent call to sqlite3_set_clientdata(D,N,P,X) is made
+**      with the same D and N parameters.
+** <li> The database connection closes.  SQLite does not make any guarantees
+**      about the order in which destructors are called, only that all
+**      destructors will be called exactly once at some point during the
+**      database connection closing process.
+** </ul>
+**
+** SQLite does not do anything with client data other than invoke
+** destructors on the client data at the appropriate time.  The intended
+** use for client data is to provide a mechanism for wrapper libraries
+** to store additional information about an SQLite database connection.
+**
+** There is no limit (other than available memory) on the number of different
+** client data pointers (with different names) that can be attached to a
+** single database connection.  However, the implementation is optimized
+** for the case of having only one or two different client data names.
+** Applications and wrapper libraries are discouraged from using more than
+** one client data name each.
+**
+** There is no way to enumerate the client data pointers
+** associated with a database connection.  The N parameter can be thought
+** of as a secret key such that only code that knows the secret key is able
+** to access the associated data.
+**
+** Security Warning:  These interfaces should not be exposed in scripting
+** languages or in other circumstances where it might be possible for an
+** an attacker to invoke them.  Any agent that can invoke these interfaces
+** can probably also take control of the process.
+**
+** Database connection client data is only available for SQLite
+** version 3.44.0 ([dateof:3.44.0]) and later.
+**
+** See also: [sqlite3_set_auxdata()] and [sqlite3_get_auxdata()].
+*/
+SQLITE_API void *sqlite3_get_clientdata(sqlite3*,const char*);
+SQLITE_API int sqlite3_set_clientdata(sqlite3*, const char*, void*, void(*)(void*));
 
 /*
 ** CAPI3REF: Constants Defining Special Destructor Behavior
@@ -6516,7 +6626,7 @@ SQLITE_API int sqlite3_db_readonly(sqlite3 *db, const char *zDbName);
 SQLITE_API int sqlite3_txn_state(sqlite3*,const char *zSchema);
 
 /*
-** CAPI3REF: Allowed return values from [sqlite3_txn_state()]
+** CAPI3REF: Allowed return values from sqlite3_txn_state()
 ** KEYWORDS: {transaction state}
 **
 ** These constants define the current transaction state of a database file.
@@ -6648,7 +6758,7 @@ SQLITE_API void *sqlite3_rollback_hook(sqlite3*, void(*)(void *), void*);
 ** ^Each call to the sqlite3_autovacuum_pages() interface overrides all
 ** previous invocations for that database connection.  ^If the callback
 ** argument (C) to sqlite3_autovacuum_pages(D,C,P,X) is a NULL pointer,
-** then the autovacuum steps callback is cancelled.  The return value
+** then the autovacuum steps callback is canceled.  The return value
 ** from sqlite3_autovacuum_pages() is normally SQLITE_OK, but might
 ** be some other error code if something goes wrong.  The current
 ** implementation will only return SQLITE_OK or SQLITE_MISUSE, but other
@@ -7167,6 +7277,10 @@ struct sqlite3_module {
   /* The methods above are in versions 1 and 2 of the sqlite_module object.
   ** Those below are for version 3 and greater. */
   int (*xShadowName)(const char*);
+  /* The methods above are in versions 1 through 3 of the sqlite_module object.
+  ** Those below are for version 4 and greater. */
+  int (*xIntegrity)(sqlite3_vtab *pVTab, const char *zSchema,
+                    const char *zTabName, int mFlags, char **pzErr);
 };
 
 /*
@@ -7654,7 +7768,7 @@ SQLITE_API int sqlite3_blob_reopen(sqlite3_blob *, sqlite3_int64);
 ** code is returned and the transaction rolled back.
 **
 ** Calling this function with an argument that is not a NULL pointer or an
-** open blob handle results in undefined behaviour. ^Calling this routine
+** open blob handle results in undefined behavior. ^Calling this routine
 ** with a null pointer (such as would be returned by a failed call to
 ** [sqlite3_blob_open()]) is a harmless no-op. ^Otherwise, if this function
 ** is passed a valid open blob handle, the values returned by the
@@ -8134,6 +8248,7 @@ SQLITE_API int sqlite3_test_control(int op, ...);
 #define SQLITE_TESTCTRL_PRNG_SAVE                5
 #define SQLITE_TESTCTRL_PRNG_RESTORE             6
 #define SQLITE_TESTCTRL_PRNG_RESET               7  /* NOT USED */
+#define SQLITE_TESTCTRL_FK_NO_ACTION             7
 #define SQLITE_TESTCTRL_BITVEC_TEST              8
 #define SQLITE_TESTCTRL_FAULT_INSTALL            9
 #define SQLITE_TESTCTRL_BENIGN_MALLOC_HOOKS     10
@@ -8162,7 +8277,8 @@ SQLITE_API int sqlite3_test_control(int op, ...);
 #define SQLITE_TESTCTRL_TRACEFLAGS              31
 #define SQLITE_TESTCTRL_TUNE                    32
 #define SQLITE_TESTCTRL_LOGEST                  33
-#define SQLITE_TESTCTRL_LAST                    33  /* Largest TESTCTRL */
+#define SQLITE_TESTCTRL_USELONGDOUBLE           34
+#define SQLITE_TESTCTRL_LAST                    34  /* Largest TESTCTRL */
 
 /*
 ** CAPI3REF: SQL Keyword Checking
@@ -9618,7 +9734,7 @@ SQLITE_API int sqlite3_vtab_config(sqlite3*, int op, ...);
 ** [[SQLITE_VTAB_DIRECTONLY]]<dt>SQLITE_VTAB_DIRECTONLY</dt>
 ** <dd>Calls of the form
 ** [sqlite3_vtab_config](db,SQLITE_VTAB_DIRECTONLY) from within the
-** the [xConnect] or [xCreate] methods of a [virtual table] implmentation
+** the [xConnect] or [xCreate] methods of a [virtual table] implementation
 ** prohibits that virtual table from being used from within triggers and
 ** views.
 ** </dd>
@@ -9808,7 +9924,7 @@ SQLITE_API int sqlite3_vtab_distinct(sqlite3_index_info*);
 ** communicated to the xBestIndex method as a
 ** [SQLITE_INDEX_CONSTRAINT_EQ] constraint.)^  If xBestIndex wants to use
 ** this constraint, it must set the corresponding
-** aConstraintUsage[].argvIndex to a postive integer.  ^(Then, under
+** aConstraintUsage[].argvIndex to a positive integer.  ^(Then, under
 ** the usual mode of handling IN operators, SQLite generates [bytecode]
 ** that invokes the [xFilter|xFilter() method] once for each value
 ** on the right-hand side of the IN operator.)^  Thus the virtual table
@@ -10237,7 +10353,7 @@ SQLITE_API int sqlite3_db_cacheflush(sqlite3*);
 ** When the [sqlite3_blob_write()] API is used to update a blob column,
 ** the pre-update hook is invoked with SQLITE_DELETE. This is because the
 ** in this case the new values are not available. In this case, when a
-** callback made with op==SQLITE_DELETE is actuall a write using the
+** callback made with op==SQLITE_DELETE is actually a write using the
 ** sqlite3_blob_write() API, the [sqlite3_preupdate_blobwrite()] returns
 ** the index of the column being written. In other cases, where the
 ** pre-update hook is being invoked for some other reason, including a
@@ -10498,6 +10614,13 @@ SQLITE_API SQLITE_EXPERIMENTAL int sqlite3_snapshot_recover(sqlite3 *db, const c
 ** SQLITE_SERIALIZE_NOCOPY bit is set but no contiguous copy
 ** of the database exists.
 **
+** After the call, if the SQLITE_SERIALIZE_NOCOPY bit had been set,
+** the returned buffer content will remain accessible and unchanged
+** until either the next write operation on the connection or when
+** the connection is closed, and applications must not modify the
+** buffer. If the bit had been clear, the returned buffer will not
+** be accessed by SQLite after the call.
+**
 ** A call to sqlite3_serialize(D,S,P,F) might return NULL even if the
 ** SQLITE_SERIALIZE_NOCOPY bit is omitted from argument F if a memory
 ** allocation error occurs.
@@ -10546,6 +10669,9 @@ SQLITE_API unsigned char *sqlite3_serialize(
 ** SQLite will try to increase the buffer size using sqlite3_realloc64()
 ** if writes on the database cause it to grow larger than M bytes.
 **
+** Applications must not modify the buffer P or invalidate it before
+** the database connection D is closed.
+**
 ** The sqlite3_deserialize() interface will fail with SQLITE_BUSY if the
 ** database is currently in a read transaction or is involved in a backup
 ** operation.
@@ -10554,6 +10680,13 @@ SQLITE_API unsigned char *sqlite3_serialize(
 ** S argument to sqlite3_deserialize(D,S,P,N,M,F) is "temp" then the
 ** function returns SQLITE_ERROR.
 **
+** The deserialized database should not be in [WAL mode].  If the database
+** is in WAL mode, then any attempt to use the database file will result
+** in an [SQLITE_CANTOPEN] error.  The application can set the
+** [file format version numbers] (bytes 18 and 19) of the input database P
+** to 0x01 prior to invoking sqlite3_deserialize(D,S,P,N,M,F) to force the
+** database file into rollback mode and work around this limitation.
+**
 ** If sqlite3_deserialize(D,S,P,N,M,F) fails for any reason and if the
 ** SQLITE_DESERIALIZE_FREEONCLOSE bit is set in argument F, then
 ** [sqlite3_free()] is invoked on argument P prior to returning.
@@ -11626,6 +11759,18 @@ SQLITE_API int sqlite3changeset_concat(
 );
 
 
+/*
+** CAPI3REF: Upgrade the Schema of a Changeset/Patchset
+*/
+SQLITE_API int sqlite3changeset_upgrade(
+  sqlite3 *db,
+  const char *zDb,
+  int nIn, const void *pIn,       /* Input changeset */
+  int *pnOut, void **ppOut        /* OUT: Inverse of input */
+);
+
+
+
 /*
 ** CAPI3REF: Changegroup Handle
 **
@@ -11672,6 +11817,38 @@ typedef struct sqlite3_changegroup sqlite3_changegroup;
 */
 SQLITE_API int sqlite3changegroup_new(sqlite3_changegroup **pp);
 
+/*
+** CAPI3REF: Add a Schema to a Changegroup
+** METHOD: sqlite3_changegroup_schema
+**
+** This method may be used to optionally enforce the rule that the changesets
+** added to the changegroup handle must match the schema of database zDb
+** ("main", "temp", or the name of an attached database). If
+** sqlite3changegroup_add() is called to add a changeset that is not compatible
+** with the configured schema, SQLITE_SCHEMA is returned and the changegroup
+** object is left in an undefined state.
+**
+** A changeset schema is considered compatible with the database schema in
+** the same way as for sqlite3changeset_apply(). Specifically, for each
+** table in the changeset, there exists a database table with:
+**
+** <ul>
+**   <li> The name identified by the changeset, and
+**   <li> at least as many columns as recorded in the changeset, and
+**   <li> the primary key columns in the same position as recorded in
+**        the changeset.
+** </ul>
+**
+** The output of the changegroup object always has the same schema as the
+** database nominated using this function. In cases where changesets passed
+** to sqlite3changegroup_add() have fewer columns than the corresponding table
+** in the database schema, these are filled in using the default column
+** values from the database schema. This makes it possible to combined
+** changesets that have different numbers of columns for a single table
+** within a changegroup, provided that they are otherwise compatible.
+*/
+SQLITE_API int sqlite3changegroup_schema(sqlite3_changegroup*, sqlite3*, const char *zDb);
+
 /*
 ** CAPI3REF: Add A Changeset To A Changegroup
 ** METHOD: sqlite3_changegroup
@@ -11740,13 +11917,18 @@ SQLITE_API int sqlite3changegroup_new(sqlite3_changegroup **pp);
 ** If the new changeset contains changes to a table that is already present
 ** in the changegroup, then the number of columns and the position of the
 ** primary key columns for the table must be consistent. If this is not the
-** case, this function fails with SQLITE_SCHEMA. If the input changeset
-** appears to be corrupt and the corruption is detected, SQLITE_CORRUPT is
-** returned. Or, if an out-of-memory condition occurs during processing, this
-** function returns SQLITE_NOMEM. In all cases, if an error occurs the state
-** of the final contents of the changegroup is undefined.
+** case, this function fails with SQLITE_SCHEMA. Except, if the changegroup
+** object has been configured with a database schema using the
+** sqlite3changegroup_schema() API, then it is possible to combine changesets
+** with different numbers of columns for a single table, provided that
+** they are otherwise compatible.
 **
-** If no error occurs, SQLITE_OK is returned.
+** If the input changeset appears to be corrupt and the corruption is
+** detected, SQLITE_CORRUPT is returned. Or, if an out-of-memory condition
+** occurs during processing, this function returns SQLITE_NOMEM.
+**
+** In all cases, if an error occurs the state of the final contents of the
+** changegroup is undefined. If no error occurs, SQLITE_OK is returned.
 */
 SQLITE_API int sqlite3changegroup_add(sqlite3_changegroup*, int nData, void *pData);
 
@@ -12011,10 +12193,17 @@ SQLITE_API int sqlite3changeset_apply_v2(
 **    <li>an insert change if all fields of the conflicting row match
 **        the row being inserted.
 **    </ul>
+**
+** <dt>SQLITE_CHANGESETAPPLY_FKNOACTION <dd>
+**   If this flag it set, then all foreign key constraints in the target
+**   database behave as if they were declared with "ON UPDATE NO ACTION ON
+**   DELETE NO ACTION", even if they are actually CASCADE, RESTRICT, SET NULL
+**   or SET DEFAULT.
 */
 #define SQLITE_CHANGESETAPPLY_NOSAVEPOINT   0x0001
 #define SQLITE_CHANGESETAPPLY_INVERT        0x0002
 #define SQLITE_CHANGESETAPPLY_IGNORENOOP    0x0004
+#define SQLITE_CHANGESETAPPLY_FKNOACTION    0x0008
 
 /*
 ** CAPI3REF: Constants Passed To The Conflict Handler
@@ -12755,7 +12944,7 @@ struct Fts5PhraseIter {
 **   See xPhraseFirstColumn above.
 */
 struct Fts5ExtensionApi {
-  int iVersion;                   /* Currently always set to 3 */
+  int iVersion;                   /* Currently always set to 2 */
 
   void *(*xUserData)(Fts5Context*);
 
@@ -12984,8 +13173,8 @@ struct Fts5ExtensionApi {
 **   as separate queries of the FTS index are required for each synonym.
 **
 **   When using methods (2) or (3), it is important that the tokenizer only
-**   provide synonyms when tokenizing document text (method (2)) or query
-**   text (method (3)), not both. Doing so will not cause any errors, but is
+**   provide synonyms when tokenizing document text (method (3)) or query
+**   text (method (2)), not both. Doing so will not cause any errors, but is
 **   inefficient.
 */
 typedef struct Fts5Tokenizer Fts5Tokenizer;
@@ -13033,7 +13222,7 @@ struct fts5_api {
   int (*xCreateTokenizer)(
     fts5_api *pApi,
     const char *zName,
-    void *pContext,
+    void *pUserData,
     fts5_tokenizer *pTokenizer,
     void (*xDestroy)(void*)
   );
@@ -13042,7 +13231,7 @@ struct fts5_api {
   int (*xFindTokenizer)(
     fts5_api *pApi,
     const char *zName,
-    void **ppContext,
+    void **ppUserData,
     fts5_tokenizer *pTokenizer
   );
 
@@ -13050,7 +13239,7 @@ struct fts5_api {
   int (*xCreateFunction)(
     fts5_api *pApi,
     const char *zName,
-    void *pContext,
+    void *pUserData,
     fts5_extension_function xFunction,
     void (*xDestroy)(void*)
   );

vendor/github.com/mattn/go-sqlite3/sqlite3.go

@@ -21,7 +21,6 @@ package sqlite3
 #cgo CFLAGS: -DSQLITE_DEFAULT_WAL_SYNCHRONOUS=1
 #cgo CFLAGS: -DSQLITE_ENABLE_UPDATE_DELETE_LIMIT
 #cgo CFLAGS: -Wno-deprecated-declarations
-#cgo linux,!android CFLAGS: -DHAVE_PREAD64=1 -DHAVE_PWRITE64=1
 #cgo openbsd CFLAGS: -I/usr/local/include
 #cgo openbsd LDFLAGS: -L/usr/local/lib
 #ifndef USE_LIBSQLITE3
@@ -48,6 +47,18 @@ package sqlite3
 # define SQLITE_DETERMINISTIC 0
 #endif
 
+#if defined(HAVE_PREAD64) && defined(HAVE_PWRITE64)
+# undef USE_PREAD
+# undef USE_PWRITE
+# define USE_PREAD64 1
+# define USE_PWRITE64 1
+#elif defined(HAVE_PREAD) && defined(HAVE_PWRITE)
+# undef USE_PREAD
+# undef USE_PWRITE
+# define USE_PREAD64 1
+# define USE_PWRITE64 1
+#endif
+
 static int
 _sqlite3_open_v2(const char *filename, sqlite3 **ppDb, int flags, const char *zVfs) {
 #ifdef SQLITE_OPEN_URI

vendor/github.com/mattn/go-sqlite3/sqlite3ext.h

@@ -366,6 +366,11 @@ struct sqlite3_api_routines {
   int (*value_encoding)(sqlite3_value*);
   /* Version 3.41.0 and later */
   int (*is_interrupted)(sqlite3*);
+  /* Version 3.43.0 and later */
+  int (*stmt_explain)(sqlite3_stmt*,int);
+  /* Version 3.44.0 and later */
+  void *(*get_clientdata)(sqlite3*,const char*);
+  int (*set_clientdata)(sqlite3*, const char*, void*, void(*)(void*));
 };
 
 /*
@@ -694,6 +699,11 @@ typedef int (*sqlite3_loadext_entry)(
 #define sqlite3_value_encoding         sqlite3_api->value_encoding
 /* Version 3.41.0 and later */
 #define sqlite3_is_interrupted         sqlite3_api->is_interrupted
+/* Version 3.43.0 and later */
+#define sqlite3_stmt_explain           sqlite3_api->stmt_explain
+/* Version 3.44.0 and later */
+#define sqlite3_get_clientdata         sqlite3_api->get_clientdata
+#define sqlite3_set_clientdata         sqlite3_api->set_clientdata
 #endif /* !defined(SQLITE_CORE) && !defined(SQLITE_OMIT_LOAD_EXTENSION) */
 
 #if !defined(SQLITE_CORE) && !defined(SQLITE_OMIT_LOAD_EXTENSION)

vendor/github.com/rs/zerolog/README.md

@@ -1,6 +1,6 @@
 # Zero Allocation JSON Logger
 
-[![godoc](http://img.shields.io/badge/godoc-reference-blue.svg?style=flat)](https://godoc.org/github.com/rs/zerolog) [![license](http://img.shields.io/badge/license-MIT-red.svg?style=flat)](https://raw.githubusercontent.com/rs/zerolog/master/LICENSE) [![Build Status](https://travis-ci.org/rs/zerolog.svg?branch=master)](https://travis-ci.org/rs/zerolog) [![Coverage](http://gocover.io/_badge/github.com/rs/zerolog)](http://gocover.io/github.com/rs/zerolog)
+[![godoc](http://img.shields.io/badge/godoc-reference-blue.svg?style=flat)](https://godoc.org/github.com/rs/zerolog) [![license](http://img.shields.io/badge/license-MIT-red.svg?style=flat)](https://raw.githubusercontent.com/rs/zerolog/master/LICENSE) [![Build Status](https://github.com/rs/zerolog/actions/workflows/test.yml/badge.svg)](https://github.com/rs/zerolog/actions/workflows/test.yml) [![Go Coverage](https://github.com/rs/zerolog/wiki/coverage.svg)](https://raw.githack.com/wiki/rs/zerolog/coverage.html)
 
 The zerolog package provides a fast and simple logger dedicated to JSON output.
 
@@ -499,7 +499,7 @@ log.Ctx(ctx).Info().Msg("hello world")
 ### Set as standard logger output
 
 ```go
-stdlog := zerolog.New(os.Stdout).With().
+log := zerolog.New(os.Stdout).With().
     Str("foo", "bar").
     Logger()
 
@@ -694,7 +694,7 @@ with zerolog library is [CSD](https://github.com/toravir/csd/).
 
 ## Benchmarks
 
-See [logbench](http://hackemist.com/logbench/) for more comprehensive and up-to-date benchmarks.
+See [logbench](http://bench.zerolog.io/) for more comprehensive and up-to-date benchmarks.
 
 All operations are allocation free (those numbers *include* JSON encoding):
 

vendor/github.com/rs/zerolog/console.go

@@ -312,6 +312,11 @@ func needsQuote(s string) bool {
 
 // colorize returns the string s wrapped in ANSI code c, unless disabled is true.
 func colorize(s interface{}, c int, disabled bool) string {
+	e := os.Getenv("NO_COLOR")
+	if e != "" {
+		disabled = true
+	}
+
 	if disabled {
 		return fmt.Sprintf("%s", s)
 	}

vendor/github.com/rs/zerolog/context.go

@@ -57,7 +57,7 @@ func (c Context) Array(key string, arr LogArrayMarshaler) Context {
 
 // Object marshals an object that implement the LogObjectMarshaler interface.
 func (c Context) Object(key string, obj LogObjectMarshaler) Context {
-	e := newEvent(levelWriterAdapter{ioutil.Discard}, 0)
+	e := newEvent(LevelWriterAdapter{ioutil.Discard}, 0)
 	e.Object(key, obj)
 	c.l.context = enc.AppendObjectData(c.l.context, e.buf)
 	putEvent(e)
@@ -66,7 +66,7 @@ func (c Context) Object(key string, obj LogObjectMarshaler) Context {
 
 // EmbedObject marshals and Embeds an object that implement the LogObjectMarshaler interface.
 func (c Context) EmbedObject(obj LogObjectMarshaler) Context {
-	e := newEvent(levelWriterAdapter{ioutil.Discard}, 0)
+	e := newEvent(LevelWriterAdapter{ioutil.Discard}, 0)
 	e.EmbedObject(obj)
 	c.l.context = enc.AppendObjectData(c.l.context, e.buf)
 	putEvent(e)
@@ -379,6 +379,11 @@ func (c Context) Interface(key string, i interface{}) Context {
 	return c
 }
 
+// Any is a wrapper around Context.Interface.
+func (c Context) Any(key string, i interface{}) Context {
+	return c.Interface(key, i)
+}
+
 type callerHook struct {
 	callerSkipFrameCount int
 }

vendor/github.com/rs/zerolog/log.go

@@ -250,7 +250,7 @@ func New(w io.Writer) Logger {
 	}
 	lw, ok := w.(LevelWriter)
 	if !ok {
-		lw = levelWriterAdapter{w}
+		lw = LevelWriterAdapter{w}
 	}
 	return Logger{w: lw, level: TraceLevel}
 }

vendor/github.com/rs/zerolog/writer.go

@@ -17,11 +17,13 @@ type LevelWriter interface {
 	WriteLevel(level Level, p []byte) (n int, err error)
 }
 
-type levelWriterAdapter struct {
+// LevelWriterAdapter adapts an io.Writer to support the LevelWriter interface.
+type LevelWriterAdapter struct {
 	io.Writer
 }
 
-func (lw levelWriterAdapter) WriteLevel(l Level, p []byte) (n int, err error) {
+// WriteLevel simply writes everything to the adapted writer, ignoring the level.
+func (lw LevelWriterAdapter) WriteLevel(l Level, p []byte) (n int, err error) {
 	return lw.Write(p)
 }
 
@@ -38,7 +40,7 @@ func SyncWriter(w io.Writer) io.Writer {
 	if lw, ok := w.(LevelWriter); ok {
 		return &syncWriter{lw: lw}
 	}
-	return &syncWriter{lw: levelWriterAdapter{w}}
+	return &syncWriter{lw: LevelWriterAdapter{w}}
 }
 
 // Write implements the io.Writer interface.
@@ -96,7 +98,7 @@ func MultiLevelWriter(writers ...io.Writer) LevelWriter {
 		if lw, ok := w.(LevelWriter); ok {
 			lwriters = append(lwriters, lw)
 		} else {
-			lwriters = append(lwriters, levelWriterAdapter{w})
+			lwriters = append(lwriters, LevelWriterAdapter{w})
 		}
 	}
 	return multiLevelWriter{lwriters}
@@ -152,3 +154,29 @@ func ConsoleTestWriter(t TestingLog) func(w *ConsoleWriter) {
 		w.Out = TestWriter{T: t, Frame: 6}
 	}
 }
+
+// FilteredLevelWriter writes only logs at Level or above to Writer.
+//
+// It should be used only in combination with MultiLevelWriter when you
+// want to write to multiple destinations at different levels. Otherwise
+// you should just set the level on the logger and filter events early.
+// When using MultiLevelWriter then you set the level on the logger to
+// the lowest of the levels you use for writers.
+type FilteredLevelWriter struct {
+	Writer LevelWriter
+	Level  Level
+}
+
+// Write writes to the underlying Writer.
+func (w *FilteredLevelWriter) Write(p []byte) (int, error) {
+	return w.Writer.Write(p)
+}
+
+// WriteLevel calls WriteLevel of the underlying Writer only if the level is equal
+// or above the Level.
+func (w *FilteredLevelWriter) WriteLevel(level Level, p []byte) (int, error) {
+	if level >= w.Level {
+		return w.Writer.WriteLevel(level, p)
+	}
+	return len(p), nil
+}

vendor/github.com/tidwall/gjson/README.md

@@ -427,16 +427,6 @@ if result.Index > 0 {
 
 This is a best-effort no allocation sub slice of the original json. This method utilizes the `result.Index` field, which is the position of the raw data in the original json. It's possible that the value of `result.Index` equals zero, in which case the `result.Raw` is converted to a `[]byte`.
 
-## Get multiple values at once
-
-The `GetMany` function can be used to get multiple values at the same time.
-
-```go
-results := gjson.GetMany(json, "name.first", "name.last", "age")
-```
-
-The return value is a `[]Result`, which will always contain exactly the same number of items as the input paths.
-
 ## Performance
 
 Benchmarks of GJSON alongside [encoding/json](https://golang.org/pkg/encoding/json/), 

vendor/github.com/tidwall/gjson/gjson.go

@@ -3410,7 +3410,7 @@ func (t Result) Path(json string) string {
 		if !rcomp.Exists() {
 			goto fail
 		}
-		comp := escapeComp(rcomp.String())
+		comp := Escape(rcomp.String())
 		path = append(path, '.')
 		path = append(path, comp...)
 	}
@@ -3425,17 +3425,31 @@ fail:
 // isSafePathKeyChar returns true if the input character is safe for not
 // needing escaping.
 func isSafePathKeyChar(c byte) bool {
-	return c <= ' ' || c > '~' || c == '_' || c == '-' || c == ':' ||
-		(c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') ||
-		(c >= '0' && c <= '9')
+	return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') ||
+		(c >= '0' && c <= '9') || c <= ' ' || c > '~' || c == '_' ||
+		c == '-' || c == ':'
 }
 
-// escapeComp escaped a path compontent, making it safe for generating a
-// path for later use.
-func escapeComp(comp string) string {
+// Escape returns an escaped path component.
+//
+//	json := `{
+//	  "user":{
+//	     "first.name": "Janet",
+//	     "last.name": "Prichard"
+//	   }
+//	}`
+//	user := gjson.Get(json, "user")
+//	println(user.Get(gjson.Escape("first.name"))
+//	println(user.Get(gjson.Escape("last.name"))
+//	// Output:
+//	// Janet
+//	// Prichard
+func Escape(comp string) string {
 	for i := 0; i < len(comp); i++ {
 		if !isSafePathKeyChar(comp[i]) {
-			ncomp := []byte(comp[:i])
+			ncomp := make([]byte, len(comp)+1)
+			copy(ncomp, comp[:i])
+			ncomp = ncomp[:i]
 			for ; i < len(comp); i++ {
 				if !isSafePathKeyChar(comp[i]) {
 					ncomp = append(ncomp, '\\')

vendor/github.com/yuin/goldmark/.golangci.yml

@@ -45,7 +45,7 @@ linters-settings:
         disabled: false
       - name: dot-imports
         severity: warning
-        disabled: false
+        disabled: true
       - name: error-return
         severity: warning
         disabled: false

vendor/github.com/yuin/goldmark/README.md

@@ -379,9 +379,47 @@ This extension provides additional options for CJK users.
 
 | Functional option | Type | Description |
 | ----------------- | ---- | ----------- |
-| `extension.WithEastAsianLineBreaks` | `-` | Soft line breaks are rendered as a newline. Some asian users will see it as an unnecessary space. With this option, soft line breaks between east asian wide characters will be ignored. |
+| `extension.WithEastAsianLineBreaks` | `...extension.EastAsianLineBreaksStyle` | Soft line breaks are rendered as a newline. Some asian users will see it as an unnecessary space. With this option, soft line breaks between east asian wide characters will be ignored. |
 | `extension.WithEscapedSpace` | `-` | Without spaces around an emphasis started with east asian punctuations, it is not interpreted as an emphasis(as defined in CommonMark spec). With this option, you can avoid this inconvenient behavior by putting 'not rendered' spaces around an emphasis like `太郎は\ **「こんにちわ」**\ といった`. |
 
+#### Styles of Line Breaking
+
+| Style | Description |
+| ----- | ----------- |
+| `EastAsianLineBreaksStyleSimple` | Soft line breaks are ignored if both sides of the break are east asian wide character. This behavior is the same as [`east_asian_line_breaks`](https://pandoc.org/MANUAL.html#extension-east_asian_line_breaks) in Pandoc. |
+| `EastAsianLineBreaksCSS3Draft` | This option implements CSS text level3 [Segment Break Transformation Rules](https://drafts.csswg.org/css-text-3/#line-break-transform) with [some enhancements](https://github.com/w3c/csswg-drafts/issues/5086). |
+
+#### Example of `EastAsianLineBreaksStyleSimple`
+
+Input Markdown:
+
+```md
+私はプログラマーです。
+東京の会社に勤めています。
+GoでWebアプリケーションを開発しています。
+```
+
+Output:
+
+```html
+<p>私はプログラマーです。東京の会社に勤めています。\nGoでWebアプリケーションを開発しています。</p>
+```
+
+#### Example of `EastAsianLineBreaksCSS3Draft`
+
+Input Markdown:
+
+```md
+私はプログラマーです。
+東京の会社に勤めています。
+GoでWebアプリケーションを開発しています。
+```
+
+Output:
+
+```html
+<p>私はプログラマーです。東京の会社に勤めています。GoでWebアプリケーションを開発しています。</p>
+```
 
 Security
 --------------------

vendor/github.com/yuin/goldmark/extension/cjk.go

@@ -9,11 +9,30 @@ import (
 // A CJKOption sets options for CJK support mostly for HTML based renderers.
 type CJKOption func(*cjk)
 
+// A EastAsianLineBreaks is a style of east asian line breaks.
+type EastAsianLineBreaks int
+
+const (
+	//EastAsianLineBreaksNone renders line breaks as it is.
+	EastAsianLineBreaksNone EastAsianLineBreaks = iota
+	// EastAsianLineBreaksSimple is a style where soft line breaks are ignored
+	// if both sides of the break are east asian wide characters.
+	EastAsianLineBreaksSimple
+	// EastAsianLineBreaksCSS3Draft is a style where soft line breaks are ignored
+	// even if only one side of the break is an east asian wide character.
+	EastAsianLineBreaksCSS3Draft
+)
+
 // WithEastAsianLineBreaks is a functional option that indicates whether softline breaks
 // between east asian wide characters should be ignored.
-func WithEastAsianLineBreaks() CJKOption {
+// style defauts to [EastAsianLineBreaksSimple] .
+func WithEastAsianLineBreaks(style ...EastAsianLineBreaks) CJKOption {
 	return func(c *cjk) {
-		c.EastAsianLineBreaks = true
+		if len(style) == 0 {
+			c.EastAsianLineBreaks = EastAsianLineBreaksSimple
+			return
+		}
+		c.EastAsianLineBreaks = style[0]
 	}
 }
 
@@ -25,7 +44,7 @@ func WithEscapedSpace() CJKOption {
 }
 
 type cjk struct {
-	EastAsianLineBreaks bool
+	EastAsianLineBreaks EastAsianLineBreaks
 	EscapedSpace        bool
 }
 
@@ -34,7 +53,9 @@ var CJK = NewCJK(WithEastAsianLineBreaks(), WithEscapedSpace())
 
 // NewCJK returns a new extension with given options.
 func NewCJK(opts ...CJKOption) goldmark.Extender {
-	e := &cjk{}
+	e := &cjk{
+		EastAsianLineBreaks: EastAsianLineBreaksNone,
+	}
 	for _, opt := range opts {
 		opt(e)
 	}
@@ -42,9 +63,8 @@ func NewCJK(opts ...CJKOption) goldmark.Extender {
 }
 
 func (e *cjk) Extend(m goldmark.Markdown) {
-	if e.EastAsianLineBreaks {
-		m.Renderer().AddOptions(html.WithEastAsianLineBreaks())
-	}
+	m.Renderer().AddOptions(html.WithEastAsianLineBreaks(
+		html.EastAsianLineBreaks(e.EastAsianLineBreaks)))
 	if e.EscapedSpace {
 		m.Renderer().AddOptions(html.WithWriter(html.NewWriter(html.WithEscapedSpace())))
 		m.Parser().AddOptions(parser.WithEscapedSpace())

vendor/github.com/yuin/goldmark/extension/tasklist.go

@@ -41,6 +41,9 @@ func (s *taskCheckBoxParser) Parse(parent gast.Node, block text.Reader, pc parse
 		return nil
 	}
 
+	if parent.HasChildren() {
+		return nil
+	}
 	if _, ok := parent.Parent().(*gast.ListItem); !ok {
 		return nil
 	}

vendor/github.com/yuin/goldmark/renderer/html/html.go

@@ -5,6 +5,7 @@ import (
 	"bytes"
 	"fmt"
 	"strconv"
+	"unicode"
 	"unicode/utf8"
 
 	"github.com/yuin/goldmark/ast"
@@ -16,7 +17,7 @@ import (
 type Config struct {
 	Writer              Writer
 	HardWraps           bool
-	EastAsianLineBreaks bool
+	EastAsianLineBreaks EastAsianLineBreaks
 	XHTML               bool
 	Unsafe              bool
 }
@@ -26,7 +27,7 @@ func NewConfig() Config {
 	return Config{
 		Writer:              DefaultWriter,
 		HardWraps:           false,
-		EastAsianLineBreaks: false,
+		EastAsianLineBreaks: EastAsianLineBreaksNone,
 		XHTML:               false,
 		Unsafe:              false,
 	}
@@ -38,7 +39,7 @@ func (c *Config) SetOption(name renderer.OptionName, value interface{}) {
 	case optHardWraps:
 		c.HardWraps = value.(bool)
 	case optEastAsianLineBreaks:
-		c.EastAsianLineBreaks = value.(bool)
+		c.EastAsianLineBreaks = value.(EastAsianLineBreaks)
 	case optXHTML:
 		c.XHTML = value.(bool)
 	case optUnsafe:
@@ -103,24 +104,94 @@ func WithHardWraps() interface {
 // EastAsianLineBreaks is an option name used in WithEastAsianLineBreaks.
 const optEastAsianLineBreaks renderer.OptionName = "EastAsianLineBreaks"
 
+// A EastAsianLineBreaks is a style of east asian line breaks.
+type EastAsianLineBreaks int
+
+const (
+	//EastAsianLineBreaksNone renders line breaks as it is.
+	EastAsianLineBreaksNone EastAsianLineBreaks = iota
+	// EastAsianLineBreaksSimple follows east_asian_line_breaks in Pandoc.
+	EastAsianLineBreaksSimple
+	// EastAsianLineBreaksCSS3Draft follows CSS text level3 "Segment Break Transformation Rules" with some enhancements.
+	EastAsianLineBreaksCSS3Draft
+)
+
+func (b EastAsianLineBreaks) softLineBreak(thisLastRune rune, siblingFirstRune rune) bool {
+	switch b {
+	case EastAsianLineBreaksNone:
+		return false
+	case EastAsianLineBreaksSimple:
+		return !(util.IsEastAsianWideRune(thisLastRune) && util.IsEastAsianWideRune(siblingFirstRune))
+	case EastAsianLineBreaksCSS3Draft:
+		return eastAsianLineBreaksCSS3DraftSoftLineBreak(thisLastRune, siblingFirstRune)
+	}
+	return false
+}
+
+func eastAsianLineBreaksCSS3DraftSoftLineBreak(thisLastRune rune, siblingFirstRune rune) bool {
+	// Implements CSS text level3 Segment Break Transformation Rules with some enhancements.
+	// References:
+	//   - https://www.w3.org/TR/2020/WD-css-text-3-20200429/#line-break-transform
+	//   - https://github.com/w3c/csswg-drafts/issues/5086
+
+	// Rule1:
+	//   If the character immediately before or immediately after the segment break is
+	//   the zero-width space character (U+200B), then the break is removed, leaving behind the zero-width space.
+	if thisLastRune == '\u200B' || siblingFirstRune == '\u200B' {
+		return false
+	}
+
+	// Rule2:
+	//   Otherwise, if the East Asian Width property of both the character before and after the segment break is
+	//   F, W, or H (not A), and neither side is Hangul, then the segment break is removed.
+	thisLastRuneEastAsianWidth := util.EastAsianWidth(thisLastRune)
+	siblingFirstRuneEastAsianWidth := util.EastAsianWidth(siblingFirstRune)
+	if (thisLastRuneEastAsianWidth == "F" ||
+		thisLastRuneEastAsianWidth == "W" ||
+		thisLastRuneEastAsianWidth == "H") &&
+		(siblingFirstRuneEastAsianWidth == "F" ||
+			siblingFirstRuneEastAsianWidth == "W" ||
+			siblingFirstRuneEastAsianWidth == "H") {
+		return unicode.Is(unicode.Hangul, thisLastRune) || unicode.Is(unicode.Hangul, siblingFirstRune)
+	}
+
+	// Rule3:
+	//   Otherwise, if either the character before or after the segment break belongs to
+	//   the space-discarding character set and it is a Unicode Punctuation (P*) or U+3000,
+	//   then the segment break is removed.
+	if util.IsSpaceDiscardingUnicodeRune(thisLastRune) ||
+		unicode.IsPunct(thisLastRune) ||
+		thisLastRune == '\u3000' ||
+		util.IsSpaceDiscardingUnicodeRune(siblingFirstRune) ||
+		unicode.IsPunct(siblingFirstRune) ||
+		siblingFirstRune == '\u3000' {
+		return false
+	}
+
+	// Rule4:
+	//   Otherwise, the segment break is converted to a space (U+0020).
+	return true
+}
+
 type withEastAsianLineBreaks struct {
+	eastAsianLineBreaksStyle EastAsianLineBreaks
 }
 
 func (o *withEastAsianLineBreaks) SetConfig(c *renderer.Config) {
-	c.Options[optEastAsianLineBreaks] = true
+	c.Options[optEastAsianLineBreaks] = o.eastAsianLineBreaksStyle
 }
 
 func (o *withEastAsianLineBreaks) SetHTMLOption(c *Config) {
-	c.EastAsianLineBreaks = true
+	c.EastAsianLineBreaks = o.eastAsianLineBreaksStyle
 }
 
 // WithEastAsianLineBreaks is a functional option that indicates whether softline breaks
 // between east asian wide characters should be ignored.
-func WithEastAsianLineBreaks() interface {
+func WithEastAsianLineBreaks(e EastAsianLineBreaks) interface {
 	renderer.Option
 	Option
 } {
-	return &withEastAsianLineBreaks{}
+	return &withEastAsianLineBreaks{e}
 }
 
 // XHTML is an option name used in WithXHTML.
@@ -663,14 +734,13 @@ func (r *Renderer) renderText(w util.BufWriter, source []byte, node ast.Node, en
 				_, _ = w.WriteString("<br>\n")
 			}
 		} else if n.SoftLineBreak() {
-			if r.EastAsianLineBreaks && len(value) != 0 {
+			if r.EastAsianLineBreaks != EastAsianLineBreaksNone && len(value) != 0 {
 				sibling := node.NextSibling()
 				if sibling != nil && sibling.Kind() == ast.KindText {
 					if siblingText := sibling.(*ast.Text).Text(source); len(siblingText) != 0 {
 						thisLastRune := util.ToRune(value, len(value)-1)
 						siblingFirstRune, _ := utf8.DecodeRune(siblingText)
-						if !(util.IsEastAsianWideRune(thisLastRune) &&
-							util.IsEastAsianWideRune(siblingFirstRune)) {
+						if r.EastAsianLineBreaks.softLineBreak(thisLastRune, siblingFirstRune) {
 							_ = w.WriteByte('\n')
 						}
 					}
@@ -938,6 +1008,9 @@ func nodeToHTMLText(n ast.Node, source []byte) []byte {
 			buf.Write(s.Text(source))
 		} else if !c.HasChildren() {
 			buf.Write(util.EscapeHTML(c.Text(source)))
+			if t, ok := c.(*ast.Text); ok && t.SoftLineBreak() {
+				buf.WriteByte('\n')
+			}
 		} else {
 			buf.Write(nodeToHTMLText(c, source))
 		}

vendor/github.com/yuin/goldmark/util/util.go

@@ -836,22 +836,6 @@ func IsAlphaNumeric(c byte) bool {
 	return c >= 'a' && c <= 'z' || c >= 'A' && c <= 'Z' || c >= '0' && c <= '9'
 }
 
-// IsEastAsianWideRune returns trhe if the given rune is an east asian wide character, otherwise false.
-func IsEastAsianWideRune(r rune) bool {
-	// https://en.wikipedia.org/wiki/CJK_Symbols_and_Punctuation
-	var CJKSymbolsAndPunctuation = &unicode.RangeTable{
-		R16: []unicode.Range16{
-			{0x3000, 0x303F, 1},
-		},
-	}
-	return unicode.Is(unicode.Hiragana, r) ||
-		unicode.Is(unicode.Katakana, r) ||
-		unicode.Is(unicode.Han, r) ||
-		unicode.Is(unicode.Lm, r) ||
-		unicode.Is(unicode.Hangul, r) ||
-		unicode.Is(CJKSymbolsAndPunctuation, r)
-}
-
 // A BufWriter is a subset of the bufio.Writer .
 type BufWriter interface {
 	io.Writer

vendor/github.com/yuin/goldmark/util/util_cjk.go

@@ -0,0 +1,469 @@
+package util
+
+import "unicode"
+
+var cjkRadicalsSupplement = &unicode.RangeTable{
+	R16: []unicode.Range16{
+		{0x2E80, 0x2EFF, 1},
+	},
+}
+
+var kangxiRadicals = &unicode.RangeTable{
+	R16: []unicode.Range16{
+		{0x2F00, 0x2FDF, 1},
+	},
+}
+
+var ideographicDescriptionCharacters = &unicode.RangeTable{
+	R16: []unicode.Range16{
+		{0x2FF0, 0x2FFF, 1},
+	},
+}
+
+var cjkSymbolsAndPunctuation = &unicode.RangeTable{
+	R16: []unicode.Range16{
+		{0x3000, 0x303F, 1},
+	},
+}
+
+var hiragana = &unicode.RangeTable{
+	R16: []unicode.Range16{
+		{0x3040, 0x309F, 1},
+	},
+}
+
+var katakana = &unicode.RangeTable{
+	R16: []unicode.Range16{
+		{0x30A0, 0x30FF, 1},
+	},
+}
+
+var kanbun = &unicode.RangeTable{
+	R16: []unicode.Range16{
+		{0x3130, 0x318F, 1},
+		{0x3190, 0x319F, 1},
+	},
+}
+
+var cjkStrokes = &unicode.RangeTable{
+	R16: []unicode.Range16{
+		{0x31C0, 0x31EF, 1},
+	},
+}
+
+var katakanaPhoneticExtensions = &unicode.RangeTable{
+	R16: []unicode.Range16{
+		{0x31F0, 0x31FF, 1},
+	},
+}
+
+var cjkCompatibility = &unicode.RangeTable{
+	R16: []unicode.Range16{
+		{0x3300, 0x33FF, 1},
+	},
+}
+
+var cjkUnifiedIdeographsExtensionA = &unicode.RangeTable{
+	R16: []unicode.Range16{
+		{0x3400, 0x4DBF, 1},
+	},
+}
+
+var cjkUnifiedIdeographs = &unicode.RangeTable{
+	R16: []unicode.Range16{
+		{0x4E00, 0x9FFF, 1},
+	},
+}
+
+var yiSyllables = &unicode.RangeTable{
+	R16: []unicode.Range16{
+		{0xA000, 0xA48F, 1},
+	},
+}
+
+var yiRadicals = &unicode.RangeTable{
+	R16: []unicode.Range16{
+		{0xA490, 0xA4CF, 1},
+	},
+}
+
+var cjkCompatibilityIdeographs = &unicode.RangeTable{
+	R16: []unicode.Range16{
+		{0xF900, 0xFAFF, 1},
+	},
+}
+
+var verticalForms = &unicode.RangeTable{
+	R16: []unicode.Range16{
+		{0xFE10, 0xFE1F, 1},
+	},
+}
+
+var cjkCompatibilityForms = &unicode.RangeTable{
+	R16: []unicode.Range16{
+		{0xFE30, 0xFE4F, 1},
+	},
+}
+
+var smallFormVariants = &unicode.RangeTable{
+	R16: []unicode.Range16{
+		{0xFE50, 0xFE6F, 1},
+	},
+}
+
+var halfwidthAndFullwidthForms = &unicode.RangeTable{
+	R16: []unicode.Range16{
+		{0xFF00, 0xFFEF, 1},
+	},
+}
+
+var kanaSupplement = &unicode.RangeTable{
+	R32: []unicode.Range32{
+		{0x1B000, 0x1B0FF, 1},
+	},
+}
+
+var kanaExtendedA = &unicode.RangeTable{
+	R32: []unicode.Range32{
+		{0x1B100, 0x1B12F, 1},
+	},
+}
+
+var smallKanaExtension = &unicode.RangeTable{
+	R32: []unicode.Range32{
+		{0x1B130, 0x1B16F, 1},
+	},
+}
+
+var cjkUnifiedIdeographsExtensionB = &unicode.RangeTable{
+	R32: []unicode.Range32{
+		{0x20000, 0x2A6DF, 1},
+	},
+}
+
+var cjkUnifiedIdeographsExtensionC = &unicode.RangeTable{
+	R32: []unicode.Range32{
+		{0x2A700, 0x2B73F, 1},
+	},
+}
+
+var cjkUnifiedIdeographsExtensionD = &unicode.RangeTable{
+	R32: []unicode.Range32{
+		{0x2B740, 0x2B81F, 1},
+	},
+}
+
+var cjkUnifiedIdeographsExtensionE = &unicode.RangeTable{
+	R32: []unicode.Range32{
+		{0x2B820, 0x2CEAF, 1},
+	},
+}
+
+var cjkUnifiedIdeographsExtensionF = &unicode.RangeTable{
+	R32: []unicode.Range32{
+		{0x2CEB0, 0x2EBEF, 1},
+	},
+}
+
+var cjkCompatibilityIdeographsSupplement = &unicode.RangeTable{
+	R32: []unicode.Range32{
+		{0x2F800, 0x2FA1F, 1},
+	},
+}
+
+var cjkUnifiedIdeographsExtensionG = &unicode.RangeTable{
+	R32: []unicode.Range32{
+		{0x30000, 0x3134F, 1},
+	},
+}
+
+// IsEastAsianWideRune returns trhe if the given rune is an east asian wide character, otherwise false.
+func IsEastAsianWideRune(r rune) bool {
+	return unicode.Is(unicode.Hiragana, r) ||
+		unicode.Is(unicode.Katakana, r) ||
+		unicode.Is(unicode.Han, r) ||
+		unicode.Is(unicode.Lm, r) ||
+		unicode.Is(unicode.Hangul, r) ||
+		unicode.Is(cjkSymbolsAndPunctuation, r)
+}
+
+// IsSpaceDiscardingUnicodeRune returns true if the given rune is space-discarding unicode character, otherwise false.
+// See https://www.w3.org/TR/2020/WD-css-text-3-20200429/#space-discard-set
+func IsSpaceDiscardingUnicodeRune(r rune) bool {
+	return unicode.Is(cjkRadicalsSupplement, r) ||
+		unicode.Is(kangxiRadicals, r) ||
+		unicode.Is(ideographicDescriptionCharacters, r) ||
+		unicode.Is(cjkSymbolsAndPunctuation, r) ||
+		unicode.Is(hiragana, r) ||
+		unicode.Is(katakana, r) ||
+		unicode.Is(kanbun, r) ||
+		unicode.Is(cjkStrokes, r) ||
+		unicode.Is(katakanaPhoneticExtensions, r) ||
+		unicode.Is(cjkCompatibility, r) ||
+		unicode.Is(cjkUnifiedIdeographsExtensionA, r) ||
+		unicode.Is(cjkUnifiedIdeographs, r) ||
+		unicode.Is(yiSyllables, r) ||
+		unicode.Is(yiRadicals, r) ||
+		unicode.Is(cjkCompatibilityIdeographs, r) ||
+		unicode.Is(verticalForms, r) ||
+		unicode.Is(cjkCompatibilityForms, r) ||
+		unicode.Is(smallFormVariants, r) ||
+		unicode.Is(halfwidthAndFullwidthForms, r) ||
+		unicode.Is(kanaSupplement, r) ||
+		unicode.Is(kanaExtendedA, r) ||
+		unicode.Is(smallKanaExtension, r) ||
+		unicode.Is(cjkUnifiedIdeographsExtensionB, r) ||
+		unicode.Is(cjkUnifiedIdeographsExtensionC, r) ||
+		unicode.Is(cjkUnifiedIdeographsExtensionD, r) ||
+		unicode.Is(cjkUnifiedIdeographsExtensionE, r) ||
+		unicode.Is(cjkUnifiedIdeographsExtensionF, r) ||
+		unicode.Is(cjkCompatibilityIdeographsSupplement, r) ||
+		unicode.Is(cjkUnifiedIdeographsExtensionG, r)
+}
+
+// EastAsianWidth returns the east asian width of the given rune.
+// See https://www.unicode.org/reports/tr11/tr11-36.html
+func EastAsianWidth(r rune) string {
+	switch {
+	case r == 0x3000,
+		(0xFF01 <= r && r <= 0xFF60),
+		(0xFFE0 <= r && r <= 0xFFE6):
+		return "F"
+
+	case r == 0x20A9,
+		(0xFF61 <= r && r <= 0xFFBE),
+		(0xFFC2 <= r && r <= 0xFFC7),
+		(0xFFCA <= r && r <= 0xFFCF),
+		(0xFFD2 <= r && r <= 0xFFD7),
+		(0xFFDA <= r && r <= 0xFFDC),
+		(0xFFE8 <= r && r <= 0xFFEE):
+		return "H"
+
+	case (0x1100 <= r && r <= 0x115F),
+		(0x11A3 <= r && r <= 0x11A7),
+		(0x11FA <= r && r <= 0x11FF),
+		(0x2329 <= r && r <= 0x232A),
+		(0x2E80 <= r && r <= 0x2E99),
+		(0x2E9B <= r && r <= 0x2EF3),
+		(0x2F00 <= r && r <= 0x2FD5),
+		(0x2FF0 <= r && r <= 0x2FFB),
+		(0x3001 <= r && r <= 0x303E),
+		(0x3041 <= r && r <= 0x3096),
+		(0x3099 <= r && r <= 0x30FF),
+		(0x3105 <= r && r <= 0x312D),
+		(0x3131 <= r && r <= 0x318E),
+		(0x3190 <= r && r <= 0x31BA),
+		(0x31C0 <= r && r <= 0x31E3),
+		(0x31F0 <= r && r <= 0x321E),
+		(0x3220 <= r && r <= 0x3247),
+		(0x3250 <= r && r <= 0x32FE),
+		(0x3300 <= r && r <= 0x4DBF),
+		(0x4E00 <= r && r <= 0xA48C),
+		(0xA490 <= r && r <= 0xA4C6),
+		(0xA960 <= r && r <= 0xA97C),
+		(0xAC00 <= r && r <= 0xD7A3),
+		(0xD7B0 <= r && r <= 0xD7C6),
+		(0xD7CB <= r && r <= 0xD7FB),
+		(0xF900 <= r && r <= 0xFAFF),
+		(0xFE10 <= r && r <= 0xFE19),
+		(0xFE30 <= r && r <= 0xFE52),
+		(0xFE54 <= r && r <= 0xFE66),
+		(0xFE68 <= r && r <= 0xFE6B),
+		(0x1B000 <= r && r <= 0x1B001),
+		(0x1F200 <= r && r <= 0x1F202),
+		(0x1F210 <= r && r <= 0x1F23A),
+		(0x1F240 <= r && r <= 0x1F248),
+		(0x1F250 <= r && r <= 0x1F251),
+		(0x20000 <= r && r <= 0x2F73F),
+		(0x2B740 <= r && r <= 0x2FFFD),
+		(0x30000 <= r && r <= 0x3FFFD):
+		return "W"
+
+	case (0x0020 <= r && r <= 0x007E),
+		(0x00A2 <= r && r <= 0x00A3),
+		(0x00A5 <= r && r <= 0x00A6),
+		r == 0x00AC,
+		r == 0x00AF,
+		(0x27E6 <= r && r <= 0x27ED),
+		(0x2985 <= r && r <= 0x2986):
+		return "Na"
+
+	case (0x00A1 == r),
+		(0x00A4 == r),
+		(0x00A7 <= r && r <= 0x00A8),
+		(0x00AA == r),
+		(0x00AD <= r && r <= 0x00AE),
+		(0x00B0 <= r && r <= 0x00B4),
+		(0x00B6 <= r && r <= 0x00BA),
+		(0x00BC <= r && r <= 0x00BF),
+		(0x00C6 == r),
+		(0x00D0 == r),
+		(0x00D7 <= r && r <= 0x00D8),
+		(0x00DE <= r && r <= 0x00E1),
+		(0x00E6 == r),
+		(0x00E8 <= r && r <= 0x00EA),
+		(0x00EC <= r && r <= 0x00ED),
+		(0x00F0 == r),
+		(0x00F2 <= r && r <= 0x00F3),
+		(0x00F7 <= r && r <= 0x00FA),
+		(0x00FC == r),
+		(0x00FE == r),
+		(0x0101 == r),
+		(0x0111 == r),
+		(0x0113 == r),
+		(0x011B == r),
+		(0x0126 <= r && r <= 0x0127),
+		(0x012B == r),
+		(0x0131 <= r && r <= 0x0133),
+		(0x0138 == r),
+		(0x013F <= r && r <= 0x0142),
+		(0x0144 == r),
+		(0x0148 <= r && r <= 0x014B),
+		(0x014D == r),
+		(0x0152 <= r && r <= 0x0153),
+		(0x0166 <= r && r <= 0x0167),
+		(0x016B == r),
+		(0x01CE == r),
+		(0x01D0 == r),
+		(0x01D2 == r),
+		(0x01D4 == r),
+		(0x01D6 == r),
+		(0x01D8 == r),
+		(0x01DA == r),
+		(0x01DC == r),
+		(0x0251 == r),
+		(0x0261 == r),
+		(0x02C4 == r),
+		(0x02C7 == r),
+		(0x02C9 <= r && r <= 0x02CB),
+		(0x02CD == r),
+		(0x02D0 == r),
+		(0x02D8 <= r && r <= 0x02DB),
+		(0x02DD == r),
+		(0x02DF == r),
+		(0x0300 <= r && r <= 0x036F),
+		(0x0391 <= r && r <= 0x03A1),
+		(0x03A3 <= r && r <= 0x03A9),
+		(0x03B1 <= r && r <= 0x03C1),
+		(0x03C3 <= r && r <= 0x03C9),
+		(0x0401 == r),
+		(0x0410 <= r && r <= 0x044F),
+		(0x0451 == r),
+		(0x2010 == r),
+		(0x2013 <= r && r <= 0x2016),
+		(0x2018 <= r && r <= 0x2019),
+		(0x201C <= r && r <= 0x201D),
+		(0x2020 <= r && r <= 0x2022),
+		(0x2024 <= r && r <= 0x2027),
+		(0x2030 == r),
+		(0x2032 <= r && r <= 0x2033),
+		(0x2035 == r),
+		(0x203B == r),
+		(0x203E == r),
+		(0x2074 == r),
+		(0x207F == r),
+		(0x2081 <= r && r <= 0x2084),
+		(0x20AC == r),
+		(0x2103 == r),
+		(0x2105 == r),
+		(0x2109 == r),
+		(0x2113 == r),
+		(0x2116 == r),
+		(0x2121 <= r && r <= 0x2122),
+		(0x2126 == r),
+		(0x212B == r),
+		(0x2153 <= r && r <= 0x2154),
+		(0x215B <= r && r <= 0x215E),
+		(0x2160 <= r && r <= 0x216B),
+		(0x2170 <= r && r <= 0x2179),
+		(0x2189 == r),
+		(0x2190 <= r && r <= 0x2199),
+		(0x21B8 <= r && r <= 0x21B9),
+		(0x21D2 == r),
+		(0x21D4 == r),
+		(0x21E7 == r),
+		(0x2200 == r),
+		(0x2202 <= r && r <= 0x2203),
+		(0x2207 <= r && r <= 0x2208),
+		(0x220B == r),
+		(0x220F == r),
+		(0x2211 == r),
+		(0x2215 == r),
+		(0x221A == r),
+		(0x221D <= r && r <= 0x2220),
+		(0x2223 == r),
+		(0x2225 == r),
+		(0x2227 <= r && r <= 0x222C),
+		(0x222E == r),
+		(0x2234 <= r && r <= 0x2237),
+		(0x223C <= r && r <= 0x223D),
+		(0x2248 == r),
+		(0x224C == r),
+		(0x2252 == r),
+		(0x2260 <= r && r <= 0x2261),
+		(0x2264 <= r && r <= 0x2267),
+		(0x226A <= r && r <= 0x226B),
+		(0x226E <= r && r <= 0x226F),
+		(0x2282 <= r && r <= 0x2283),
+		(0x2286 <= r && r <= 0x2287),
+		(0x2295 == r),
+		(0x2299 == r),
+		(0x22A5 == r),
+		(0x22BF == r),
+		(0x2312 == r),
+		(0x2460 <= r && r <= 0x24E9),
+		(0x24EB <= r && r <= 0x254B),
+		(0x2550 <= r && r <= 0x2573),
+		(0x2580 <= r && r <= 0x258F),
+		(0x2592 <= r && r <= 0x2595),
+		(0x25A0 <= r && r <= 0x25A1),
+		(0x25A3 <= r && r <= 0x25A9),
+		(0x25B2 <= r && r <= 0x25B3),
+		(0x25B6 <= r && r <= 0x25B7),
+		(0x25BC <= r && r <= 0x25BD),
+		(0x25C0 <= r && r <= 0x25C1),
+		(0x25C6 <= r && r <= 0x25C8),
+		(0x25CB == r),
+		(0x25CE <= r && r <= 0x25D1),
+		(0x25E2 <= r && r <= 0x25E5),
+		(0x25EF == r),
+		(0x2605 <= r && r <= 0x2606),
+		(0x2609 == r),
+		(0x260E <= r && r <= 0x260F),
+		(0x2614 <= r && r <= 0x2615),
+		(0x261C == r),
+		(0x261E == r),
+		(0x2640 == r),
+		(0x2642 == r),
+		(0x2660 <= r && r <= 0x2661),
+		(0x2663 <= r && r <= 0x2665),
+		(0x2667 <= r && r <= 0x266A),
+		(0x266C <= r && r <= 0x266D),
+		(0x266F == r),
+		(0x269E <= r && r <= 0x269F),
+		(0x26BE <= r && r <= 0x26BF),
+		(0x26C4 <= r && r <= 0x26CD),
+		(0x26CF <= r && r <= 0x26E1),
+		(0x26E3 == r),
+		(0x26E8 <= r && r <= 0x26FF),
+		(0x273D == r),
+		(0x2757 == r),
+		(0x2776 <= r && r <= 0x277F),
+		(0x2B55 <= r && r <= 0x2B59),
+		(0x3248 <= r && r <= 0x324F),
+		(0xE000 <= r && r <= 0xF8FF),
+		(0xFE00 <= r && r <= 0xFE0F),
+		(0xFFFD == r),
+		(0x1F100 <= r && r <= 0x1F10A),
+		(0x1F110 <= r && r <= 0x1F12D),
+		(0x1F130 <= r && r <= 0x1F169),
+		(0x1F170 <= r && r <= 0x1F19A),
+		(0xE0100 <= r && r <= 0xE01EF),
+		(0xF0000 <= r && r <= 0xFFFFD),
+		(0x100000 <= r && r <= 0x10FFFD):
+		return "A"
+
+	default:
+		return "N"
+	}
+}

vendor/gitlab.com/etke.cc/linkpearl/README.md

@@ -32,4 +32,5 @@ go lp.Start()
 * Zero configuration session restores
 * Zero configuration room and user account data encryption with AES GCM (both keys and values)
 * Zero configuration room and user account data caching
+* Threads support
 * All wrapped components exported

vendor/gitlab.com/etke.cc/linkpearl/config.go

@@ -39,6 +39,9 @@ type Config struct {
 	// MaxRetries for operations like auto join
 	MaxRetries int
 
+	// EventsLimit for methods like lp.Threads() or lp.FindEventBy()
+	EventsLimit int
+
 	// Logger
 	Logger zerolog.Logger
 

vendor/gitlab.com/etke.cc/linkpearl/events.go

@@ -0,0 +1,101 @@
+package linkpearl
+
+import (
+	"strconv"
+
+	"maunium.net/go/mautrix"
+	"maunium.net/go/mautrix/event"
+	"maunium.net/go/mautrix/id"
+)
+
+// RespThreads is response of https://spec.matrix.org/v1.8/client-server-api/#get_matrixclientv1roomsroomidthreads
+type RespThreads struct {
+	Chunk     []*event.Event `json:"chunk"`
+	NextBatch string         `json:"next_batch"`
+}
+
+// Threads endpoint, ref: https://spec.matrix.org/v1.8/client-server-api/#get_matrixclientv1roomsroomidthreads
+func (l *Linkpearl) Threads(roomID id.RoomID, fromToken ...string) (*RespThreads, error) {
+	var from string
+	if len(fromToken) > 0 {
+		from = fromToken[0]
+	}
+
+	query := map[string]string{
+		"from":  from,
+		"limit": strconv.Itoa(l.eventsLimit),
+	}
+
+	var resp *RespThreads
+	urlPath := l.GetClient().BuildURLWithQuery(mautrix.ClientURLPath{"v1", "rooms", roomID, "threads"}, query)
+	_, err := l.GetClient().MakeRequest("GET", urlPath, nil, &resp)
+	return resp, UnwrapError(err)
+}
+
+// FindThreadBy tries to find thread message event by field and value
+func (l *Linkpearl) FindThreadBy(roomID id.RoomID, field, value string, fromToken ...string) *event.Event {
+	var from string
+	if len(fromToken) > 0 {
+		from = fromToken[0]
+	}
+
+	resp, err := l.Threads(roomID, from)
+	err = UnwrapError(err)
+	if err != nil {
+		l.log.Warn().Err(err).Str("roomID", roomID.String()).Msg("cannot get room threads")
+		return nil
+	}
+
+	for _, msg := range resp.Chunk {
+		evt, contains := l.eventContains(msg, field, value)
+		if contains {
+			return evt
+		}
+	}
+
+	if resp.NextBatch == "" { // nothing more
+		return nil
+	}
+
+	return l.FindThreadBy(roomID, field, value, resp.NextBatch)
+}
+
+// FindEventBy tries to find message event by field and value
+func (l *Linkpearl) FindEventBy(roomID id.RoomID, field, value string, fromToken ...string) *event.Event {
+	var from string
+	if len(fromToken) > 0 {
+		from = fromToken[0]
+	}
+
+	resp, err := l.GetClient().Messages(roomID, from, "", mautrix.DirectionBackward, nil, l.eventsLimit)
+	err = UnwrapError(err)
+	if err != nil {
+		l.log.Warn().Err(err).Str("roomID", roomID.String()).Msg("cannot get room events")
+		return nil
+	}
+
+	for _, msg := range resp.Chunk {
+		evt, contains := l.eventContains(msg, field, value)
+		if contains {
+			return evt
+		}
+	}
+
+	if resp.End == "" { // nothing more
+		return nil
+	}
+
+	return l.FindEventBy(roomID, field, value, resp.End)
+}
+
+func (l *Linkpearl) eventContains(evt *event.Event, field, value string) (*event.Event, bool) {
+	if evt.Type == event.EventEncrypted {
+		ParseContent(evt, &l.log)
+		decrypted, err := l.GetClient().Crypto.Decrypt(evt)
+		if err == nil {
+			evt = decrypted
+		}
+	}
+
+	return evt, EventContains(evt, field, value)
+}

vendor/gitlab.com/etke.cc/linkpearl/justfile

@@ -3,9 +3,8 @@ default:
     @just --list --justfile {{ justfile() }}
 
 # update go deps
-update:
-    go get .
-    go get maunium.net/go/mautrix@latest
+update *flags:
+    go get {{flags}} .
     go mod tidy
 
 # run linter
@@ -16,11 +15,8 @@ lint:
 lintfix:
     golangci-lint run --fix ./...
 
-vuln:
-    govulncheck ./...
-
 # run unit tests
 test:
-    @go test -coverprofile=cover.out ./...
+    @go test -cover -coverprofile=cover.out -coverpkg=./... -covermode=set ./...
     @go tool cover -func=cover.out
     -@rm -f cover.out

vendor/gitlab.com/etke.cc/linkpearl/linkpearl.go

@@ -18,6 +18,8 @@ const (
 	DefaultMaxRetries = 10
 	// DefaultAccountDataCache size
 	DefaultAccountDataCache = 1000
+	// DefaultEventsLimit for methods like lp.Threads() and lp.FindEventBy()
+	DefaultEventsLimit = 1000
 )
 
 // Linkpearl object
@@ -29,9 +31,10 @@ type Linkpearl struct {
 	log zerolog.Logger
 	api *mautrix.Client
 
-	joinPermit func(*event.Event) bool
-	autoleave  bool
-	maxretries int
+	joinPermit  func(*event.Event) bool
+	autoleave   bool
+	maxretries  int
+	eventsLimit int
 }
 
 type ReqPresence struct {
@@ -46,6 +49,9 @@ func setDefaults(cfg *Config) {
 	if cfg.AccountDataCache == 0 {
 		cfg.AccountDataCache = DefaultAccountDataCache
 	}
+	if cfg.EventsLimit == 0 {
+		cfg.EventsLimit = DefaultEventsLimit
+	}
 	if cfg.JoinPermit == nil {
 		// By default, we approve all join requests
 		cfg.JoinPermit = func(*event.Event) bool { return true }
@@ -76,14 +82,15 @@ func New(cfg *Config) (*Linkpearl, error) {
 	}
 
 	lp := &Linkpearl{
-		db:         cfg.DB,
-		acc:        acc,
-		acr:        acr,
-		api:        api,
-		log:        cfg.Logger,
-		joinPermit: cfg.JoinPermit,
-		autoleave:  cfg.AutoLeave,
-		maxretries: cfg.MaxRetries,
+		db:          cfg.DB,
+		acc:         acc,
+		acr:         acr,
+		api:         api,
+		log:         cfg.Logger,
+		joinPermit:  cfg.JoinPermit,
+		autoleave:   cfg.AutoLeave,
+		maxretries:  cfg.MaxRetries,
+		eventsLimit: cfg.EventsLimit,
 	}
 
 	db, err := dbutil.NewWithDB(cfg.DB, cfg.Dialect)

vendor/gitlab.com/etke.cc/linkpearl/send.go

@@ -43,24 +43,37 @@ func (l *Linkpearl) SendNotice(roomID id.RoomID, message string, relates ...*eve
 }
 
 // SendFile to a matrix room
-func (l *Linkpearl) SendFile(roomID id.RoomID, req *mautrix.ReqUploadMedia, msgtype event.MessageType, relation *event.RelatesTo) error {
+func (l *Linkpearl) SendFile(roomID id.RoomID, req *mautrix.ReqUploadMedia, msgtype event.MessageType, relates ...*event.RelatesTo) error {
+	var relation *event.RelatesTo
+	if len(relates) > 0 {
+		relation = relates[0]
+	}
+
 	resp, err := l.GetClient().UploadMedia(*req)
 	if err != nil {
 		err = UnwrapError(err)
 		l.log.Error().Err(err).Str("file", req.FileName).Msg("cannot upload file")
 		return err
 	}
-	_, err = l.Send(roomID, &event.Content{
-		Parsed: &event.MessageEventContent{
-			MsgType:   msgtype,
-			Body:      req.FileName,
-			URL:       resp.ContentURI.CUString(),
-			RelatesTo: relation,
-		},
-	})
+	content := &event.MessageEventContent{
+		MsgType:   msgtype,
+		Body:      req.FileName,
+		URL:       resp.ContentURI.CUString(),
+		RelatesTo: relation,
+	}
+
+	_, err = l.Send(roomID, content)
 	err = UnwrapError(err)
 	if err != nil {
-		l.log.Error().Err(err).Str("file", req.FileName).Msg("cannot send uploaded file")
+		l.log.Error().Err(err).Str("roomID", roomID.String()).Str("retries", "1/2").Msg("cannot send file into the room")
+		if relation != nil {
+			content.RelatesTo = nil
+			_, err = l.Send(roomID, &content)
+			err = UnwrapError(err)
+			if err != nil {
+				l.log.Error().Err(UnwrapError(err)).Str("roomID", roomID.String()).Str("retries", "2/2").Msg("cannot send file into the room even without relations")
+			}
+		}
 	}
 
 	return err

vendor/gitlab.com/etke.cc/linkpearl/sync.go

@@ -1,6 +1,7 @@
 package linkpearl
 
 import (
+	"strings"
 	"time"
 
 	"maunium.net/go/mautrix"
@@ -65,14 +66,22 @@ func (l *Linkpearl) onInvite(evt *event.Event) {
 	l.tryLeave(evt.RoomID, 0)
 }
 
+// TODO: https://spec.matrix.org/v1.8/client-server-api/#post_matrixclientv3joinroomidoralias
+// endpoint supports server_name param and tells "The servers to attempt to join the room through. One of the servers must be participating in the room.",
+// meaning you can specify more than 1 server. It is not clear, what format should be used "example.com,example.org", or "example.com example.org", or whatever else.
+// Moreover, it is not clear if the following values can be used together with that field: l.api.UserID.Homeserver() and evt.Sender.Homeserver()
 func (l *Linkpearl) tryJoin(roomID id.RoomID, retry int) {
 	if retry >= l.maxretries {
 		return
 	}
 
-	_, err := l.api.JoinRoomByID(roomID)
+	_, err := l.api.JoinRoom(roomID.String(), "", nil)
+	err = UnwrapError(err)
 	if err != nil {
 		l.log.Error().Err(err).Str("roomID", roomID.String()).Msg("cannot join room")
+		if strings.HasPrefix(err.Error(), "403") || strings.HasPrefix(err.Error(), "M_FORBIDDEN") { // no permission to join, no need to retry
+			return
+		}
 		time.Sleep(5 * time.Second)
 		l.log.Error().Err(err).Str("roomID", roomID.String()).Int("retry", retry+1).Msg("trying to join again")
 		l.tryJoin(roomID, retry+1)
@@ -85,6 +94,7 @@ func (l *Linkpearl) tryLeave(roomID id.RoomID, retry int) {
 	}
 
 	_, err := l.api.LeaveRoom(roomID)
+	err = UnwrapError(err)
 	if err != nil {
 		l.log.Error().Err(err).Str("roomID", roomID.String()).Msg("cannot leave room")
 		time.Sleep(5 * time.Second)
@@ -99,6 +109,7 @@ func (l *Linkpearl) onEmpty(evt *event.Event) {
 	}
 
 	members, err := l.api.StateStore.GetRoomJoinedOrInvitedMembers(evt.RoomID)
+	err = UnwrapError(err)
 	if err != nil {
 		l.log.Error().Err(err).Str("roomID", evt.RoomID.String()).Msg("cannot get joined or invited members")
 		return

vendor/gitlab.com/etke.cc/linkpearl/utils.go

@@ -7,6 +7,17 @@ import (
 	"maunium.net/go/mautrix/id"
 )
 
+// EventRelatesTo uses evt as source for EventParent() and RelatesTo()
+func EventRelatesTo(evt *event.Event) *event.RelatesTo {
+	ParseContent(evt, nil)
+	relatable, ok := evt.Content.Parsed.(event.Relatable)
+	if !ok {
+		return nil
+	}
+
+	return RelatesTo(EventParent(evt.ID, relatable))
+}
+
 // RelatesTo returns relation object of a matrix event (either threads with reply-to fallback or plain reply-to)
 func RelatesTo(parentID id.EventID, noThreads ...bool) *event.RelatesTo {
 	if parentID == "" {
@@ -36,34 +47,54 @@ func RelatesTo(parentID id.EventID, noThreads ...bool) *event.RelatesTo {
 	}
 }
 
-// EventParent returns parent event ID (either from thread or from reply-to relation)
-func EventParent(currentID id.EventID, content *event.MessageEventContent) id.EventID {
-	if content == nil {
-		return currentID
+// GetParent is nil-safe version of evt.Content.AsMessage().RelatesTo.(GetThreadParent()|GetReplyTo())
+func GetParent(evt *event.Event) id.EventID {
+	ParseContent(evt, nil)
+	content, ok := evt.Content.Parsed.(event.Relatable)
+	if !ok {
+		return ""
 	}
 
-	relation := content.OptionalGetRelatesTo()
+	relation := content.GetRelatesTo()
 	if relation == nil {
-		return currentID
+		return ""
 	}
 
-	threadParent := relation.GetThreadParent()
-	if threadParent != "" {
-		return threadParent
+	if parentID := relation.GetThreadParent(); parentID != "" {
+		return parentID
+	}
+	if parentID := relation.GetReplyTo(); parentID != "" {
+		return parentID
 	}
 
-	replyParent := relation.GetReplyTo()
-	if replyParent != "" {
-		return replyParent
+	return ""
+}
+
+// EventParent returns parent event ID (either from thread or from reply-to relation), like GetRelatesTo(), but with content and default return value
+func EventParent(currentID id.EventID, content event.Relatable) id.EventID {
+	if parentID := GetParent(&event.Event{Content: event.Content{Parsed: content}}); parentID != "" {
+		return parentID
 	}
 
 	return currentID
 }
 
+// EventContains checks if raw event content contains specified field with specified values
+func EventContains[T comparable](evt *event.Event, field string, value T) bool {
+	if evt.Content.Raw == nil {
+		return false
+	}
+	return EventField[T](&evt.Content, field) == value
+}
+
 // EventField returns field value from raw event content
-func EventField[T any](content *event.Content, field string) T {
+func EventField[T comparable](content *event.Content, field string) T {
 	var zero T
-	raw := content.Raw[field]
+	raw, ok := content.Raw[field]
+	if !ok {
+		return zero
+	}
+
 	if raw == nil {
 		return zero
 	}
@@ -76,12 +107,13 @@ func EventField[T any](content *event.Content, field string) T {
 	return v
 }
 
-func ParseContent(evt *event.Event, eventType event.Type, log *zerolog.Logger) {
+// ParseContent parses event content according to evt.Type
+func ParseContent(evt *event.Event, log *zerolog.Logger) {
 	if evt.Content.Parsed != nil {
 		return
 	}
-	perr := evt.Content.ParseRaw(eventType)
-	if perr != nil {
+	perr := evt.Content.ParseRaw(evt.Type)
+	if perr != nil && log != nil {
 		log.Error().Err(perr).Msg("cannot parse event content")
 	}
 }

vendor/go.mau.fi/util/dbutil/database.go

@@ -199,7 +199,7 @@ func (db *Database) Close() error {
 	if db.ReadOnlyDB != nil {
 		err2 := db.ReadOnlyDB.Close()
 		if err == nil {
-			err = fmt.Errorf("closing read-only db failed: %w", err)
+			err = fmt.Errorf("closing read-only db failed: %w", err2)
 		} else {
 			err = fmt.Errorf("%w (closing read-only db also failed: %v)", err, err2)
 		}

vendor/go.mau.fi/util/jsontime/helpers.go

@@ -0,0 +1,59 @@
+// Copyright (c) 2023 Tulir Asokan
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this
+// file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+package jsontime
+
+import (
+	"time"
+)
+
+func UM(time time.Time) UnixMilli {
+	return UnixMilli{Time: time}
+}
+
+func UMInt(ts int64) UnixMilli {
+	return UM(time.UnixMilli(ts))
+}
+
+func UnixMilliNow() UnixMilli {
+	return UM(time.Now())
+}
+
+func UMicro(time time.Time) UnixMicro {
+	return UnixMicro{Time: time}
+}
+
+func UMicroInto(ts int64) UnixMicro {
+	return UMicro(time.UnixMicro(ts))
+}
+
+func UnixMicroNow() UnixMicro {
+	return UMicro(time.Now())
+}
+
+func UN(time time.Time) UnixNano {
+	return UnixNano{Time: time}
+}
+
+func UNInt(ts int64) UnixNano {
+	return UN(time.Unix(0, ts))
+}
+
+func UnixNanoNow() UnixNano {
+	return UN(time.Now())
+}
+
+func U(time time.Time) Unix {
+	return Unix{Time: time}
+}
+
+func UInt(ts int64) Unix {
+	return U(time.Unix(ts, 0))
+}
+
+func UnixNow() Unix {
+	return U(time.Now())
+}

vendor/go.mau.fi/util/jsontime/integer.go

@@ -0,0 +1,90 @@
+// Copyright (c) 2023 Tulir Asokan
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this
+// file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+package jsontime
+
+import (
+	"encoding/json"
+	"time"
+)
+
+func parseTime(data []byte, unixConv func(int64) time.Time, into *time.Time) error {
+	var val int64
+	err := json.Unmarshal(data, &val)
+	if err != nil {
+		return err
+	}
+	if val == 0 {
+		*into = time.Time{}
+	} else {
+		*into = unixConv(val)
+	}
+	return nil
+}
+
+type UnixMilli struct {
+	time.Time
+}
+
+func (um UnixMilli) MarshalJSON() ([]byte, error) {
+	if um.IsZero() {
+		return []byte{'0'}, nil
+	}
+	return json.Marshal(um.UnixMilli())
+}
+
+func (um *UnixMilli) UnmarshalJSON(data []byte) error {
+	return parseTime(data, time.UnixMilli, &um.Time)
+}
+
+type UnixMicro struct {
+	time.Time
+}
+
+func (um UnixMicro) MarshalJSON() ([]byte, error) {
+	if um.IsZero() {
+		return []byte{'0'}, nil
+	}
+	return json.Marshal(um.UnixMicro())
+}
+
+func (um *UnixMicro) UnmarshalJSON(data []byte) error {
+	return parseTime(data, time.UnixMicro, &um.Time)
+}
+
+type UnixNano struct {
+	time.Time
+}
+
+func (un UnixNano) MarshalJSON() ([]byte, error) {
+	if un.IsZero() {
+		return []byte{'0'}, nil
+	}
+	return json.Marshal(un.UnixNano())
+}
+
+func (un *UnixNano) UnmarshalJSON(data []byte) error {
+	return parseTime(data, func(i int64) time.Time {
+		return time.Unix(0, i)
+	}, &un.Time)
+}
+
+type Unix struct {
+	time.Time
+}
+
+func (u Unix) MarshalJSON() ([]byte, error) {
+	if u.IsZero() {
+		return []byte{'0'}, nil
+	}
+	return json.Marshal(u.Unix())
+}
+
+func (u *Unix) UnmarshalJSON(data []byte) error {
+	return parseTime(data, func(i int64) time.Time {
+		return time.Unix(i, 0)
+	}, &u.Time)
+}

vendor/go.mau.fi/util/jsontime/jsontime.go

@@ -1,86 +0,0 @@
-// Copyright (c) 2022 Tulir Asokan
-//
-// This Source Code Form is subject to the terms of the Mozilla Public
-// License, v. 2.0. If a copy of the MPL was not distributed with this
-// file, You can obtain one at http://mozilla.org/MPL/2.0/.
-
-package jsontime
-
-import (
-	"encoding/json"
-	"time"
-)
-
-func UM(time time.Time) UnixMilli {
-	return UnixMilli{Time: time}
-}
-
-func UMInt(ts int64) UnixMilli {
-	return UM(time.UnixMilli(ts))
-}
-
-func UnixMilliNow() UnixMilli {
-	return UM(time.Now())
-}
-
-type UnixMilli struct {
-	time.Time
-}
-
-func (um UnixMilli) MarshalJSON() ([]byte, error) {
-	if um.IsZero() {
-		return []byte{'0'}, nil
-	}
-	return json.Marshal(um.UnixMilli())
-}
-
-func (um *UnixMilli) UnmarshalJSON(data []byte) error {
-	var val int64
-	err := json.Unmarshal(data, &val)
-	if err != nil {
-		return err
-	}
-	if val == 0 {
-		um.Time = time.Time{}
-	} else {
-		um.Time = time.UnixMilli(val)
-	}
-	return nil
-}
-
-func U(time time.Time) Unix {
-	return Unix{Time: time}
-}
-
-func UInt(ts int64) Unix {
-	return U(time.Unix(ts, 0))
-}
-
-func UnixNow() Unix {
-	return U(time.Now())
-}
-
-type Unix struct {
-	time.Time
-}
-
-func (u Unix) MarshalJSON() ([]byte, error) {
-	if u.IsZero() {
-		return []byte{'0'}, nil
-	}
-	return json.Marshal(u.Unix())
-}
-
-func (u *Unix) UnmarshalJSON(data []byte) error {
-	var val int64
-	err := json.Unmarshal(data, &val)
-	if err != nil {
-		return err
-	}
-	if val == 0 {
-		u.Time = time.Time{}
-	} else {
-		u.Time = time.Unix(val, 0)
-	}
-	return nil
-}

vendor/go.mau.fi/util/jsontime/string.go

@@ -0,0 +1,95 @@
+// Copyright (c) 2023 Tulir Asokan
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this
+// file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+package jsontime
+
+import (
+	"encoding/json"
+	"strconv"
+	"time"
+)
+
+func parseTimeString(data []byte, unixConv func(int64) time.Time, into *time.Time) error {
+	var strVal string
+	err := json.Unmarshal(data, &strVal)
+	if err != nil {
+		return err
+	}
+	val, err := strconv.ParseInt(strVal, 10, 64)
+	if err != nil {
+		return err
+	}
+	if val == 0 {
+		*into = time.Time{}
+	} else {
+		*into = unixConv(val)
+	}
+	return nil
+}
+
+type UnixMilliString struct {
+	time.Time
+}
+
+func (um UnixMilliString) MarshalJSON() ([]byte, error) {
+	if um.IsZero() {
+		return []byte{'"', '0', '"'}, nil
+	}
+	return json.Marshal(strconv.FormatInt(um.UnixMilli(), 10))
+}
+
+func (um *UnixMilliString) UnmarshalJSON(data []byte) error {
+	return parseTimeString(data, time.UnixMilli, &um.Time)
+}
+
+type UnixMicroString struct {
+	time.Time
+}
+
+func (um UnixMicroString) MarshalJSON() ([]byte, error) {
+	if um.IsZero() {
+		return []byte{'"', '0', '"'}, nil
+	}
+	return json.Marshal(strconv.FormatInt(um.UnixMilli(), 10))
+}
+
+func (um *UnixMicroString) UnmarshalJSON(data []byte) error {
+	return parseTimeString(data, time.UnixMicro, &um.Time)
+}
+
+type UnixNanoString struct {
+	time.Time
+}
+
+func (um UnixNanoString) MarshalJSON() ([]byte, error) {
+	if um.IsZero() {
+		return []byte{'"', '0', '"'}, nil
+	}
+	return json.Marshal(um.UnixNano())
+}
+
+func (um *UnixNanoString) UnmarshalJSON(data []byte) error {
+	return parseTimeString(data, func(i int64) time.Time {
+		return time.Unix(0, i)
+	}, &um.Time)
+}
+
+type UnixString struct {
+	time.Time
+}
+
+func (u UnixString) MarshalJSON() ([]byte, error) {
+	if u.IsZero() {
+		return []byte{'"', '0', '"'}, nil
+	}
+	return json.Marshal(u.Unix())
+}
+
+func (u *UnixString) UnmarshalJSON(data []byte) error {
+	return parseTimeString(data, func(i int64) time.Time {
+		return time.Unix(i, 0)
+	}, &u.Time)
+}

vendor/golang.org/x/crypto/argon2/blamka_amd64.go

@@ -3,7 +3,6 @@
 // license that can be found in the LICENSE file.
 
 //go:build amd64 && gc && !purego
-// +build amd64,gc,!purego
 
 package argon2
 

vendor/golang.org/x/crypto/argon2/blamka_amd64.s

@@ -3,7 +3,6 @@
 // license that can be found in the LICENSE file.
 
 //go:build amd64 && gc && !purego
-// +build amd64,gc,!purego
 
 #include "textflag.h"
 

vendor/golang.org/x/crypto/argon2/blamka_ref.go

@@ -3,7 +3,6 @@
 // license that can be found in the LICENSE file.
 
 //go:build !amd64 || purego || !gc
-// +build !amd64 purego !gc
 
 package argon2
 

vendor/golang.org/x/crypto/blake2b/blake2bAVX2_amd64.go

@@ -3,7 +3,6 @@
 // license that can be found in the LICENSE file.
 
 //go:build go1.7 && amd64 && gc && !purego
-// +build go1.7,amd64,gc,!purego
 
 package blake2b
 

vendor/golang.org/x/crypto/blake2b/blake2bAVX2_amd64.s

@@ -3,7 +3,6 @@
 // license that can be found in the LICENSE file.
 
 //go:build go1.7 && amd64 && gc && !purego
-// +build go1.7,amd64,gc,!purego
 
 #include "textflag.h"
 

vendor/golang.org/x/crypto/blake2b/blake2b_amd64.go

@@ -3,7 +3,6 @@
 // license that can be found in the LICENSE file.
 
 //go:build !go1.7 && amd64 && gc && !purego
-// +build !go1.7,amd64,gc,!purego
 
 package blake2b
 

vendor/golang.org/x/crypto/blake2b/blake2b_amd64.s

@@ -3,7 +3,6 @@
 // license that can be found in the LICENSE file.
 
 //go:build amd64 && gc && !purego
-// +build amd64,gc,!purego
 
 #include "textflag.h"
 

vendor/golang.org/x/crypto/blake2b/blake2b_ref.go

@@ -3,7 +3,6 @@
 // license that can be found in the LICENSE file.
 
 //go:build !amd64 || purego || !gc
-// +build !amd64 purego !gc
 
 package blake2b
 

vendor/golang.org/x/crypto/blake2b/register.go

@@ -3,7 +3,6 @@
 // license that can be found in the LICENSE file.
 
 //go:build go1.9
-// +build go1.9
 
 package blake2b
 

vendor/golang.org/x/crypto/chacha20/chacha_arm64.go

@@ -2,8 +2,7 @@
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
-//go:build go1.11 && gc && !purego
-// +build go1.11,gc,!purego
+//go:build gc && !purego
 
 package chacha20
 

vendor/golang.org/x/crypto/chacha20/chacha_arm64.s

@@ -2,8 +2,7 @@
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
-//go:build go1.11 && gc && !purego
-// +build go1.11,gc,!purego
+//go:build gc && !purego
 
 #include "textflag.h"
 

vendor/golang.org/x/crypto/chacha20/chacha_noasm.go

@@ -2,8 +2,7 @@
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
-//go:build (!arm64 && !s390x && !ppc64le) || (arm64 && !go1.11) || !gc || purego
-// +build !arm64,!s390x,!ppc64le arm64,!go1.11 !gc purego
+//go:build (!arm64 && !s390x && !ppc64le) || !gc || purego
 
 package chacha20
 

vendor/golang.org/x/crypto/chacha20/chacha_ppc64le.go

@@ -3,7 +3,6 @@
 // license that can be found in the LICENSE file.
 
 //go:build gc && !purego
-// +build gc,!purego
 
 package chacha20
 

vendor/golang.org/x/crypto/chacha20/chacha_ppc64le.s

@@ -20,7 +20,6 @@
 // due to the calling conventions and initialization of constants.
 
 //go:build gc && !purego
-// +build gc,!purego
 
 #include "textflag.h"
 

vendor/golang.org/x/crypto/chacha20/chacha_s390x.go

@@ -3,7 +3,6 @@
 // license that can be found in the LICENSE file.
 
 //go:build gc && !purego
-// +build gc,!purego
 
 package chacha20
 

vendor/golang.org/x/crypto/chacha20/chacha_s390x.s

@@ -3,7 +3,6 @@
 // license that can be found in the LICENSE file.
 
 //go:build gc && !purego
-// +build gc,!purego
 
 #include "go_asm.h"
 #include "textflag.h"

vendor/golang.org/x/crypto/curve25519/internal/field/fe_amd64.go

@@ -1,7 +1,6 @@
 // Code generated by command: go run fe_amd64_asm.go -out ../fe_amd64.s -stubs ../fe_amd64.go -pkg field. DO NOT EDIT.
 
 //go:build amd64 && gc && !purego
-// +build amd64,gc,!purego
 
 package field
 

vendor/golang.org/x/crypto/curve25519/internal/field/fe_amd64.s

@@ -1,7 +1,6 @@
 // Code generated by command: go run fe_amd64_asm.go -out ../fe_amd64.s -stubs ../fe_amd64.go -pkg field. DO NOT EDIT.
 
 //go:build amd64 && gc && !purego
-// +build amd64,gc,!purego
 
 #include "textflag.h"
 

vendor/golang.org/x/crypto/curve25519/internal/field/fe_amd64_noasm.go

@@ -3,7 +3,6 @@
 // license that can be found in the LICENSE file.
 
 //go:build !amd64 || !gc || purego
-// +build !amd64 !gc purego
 
 package field
 

vendor/golang.org/x/crypto/curve25519/internal/field/fe_arm64.go

@@ -3,7 +3,6 @@
 // license that can be found in the LICENSE file.
 
 //go:build arm64 && gc && !purego
-// +build arm64,gc,!purego
 
 package field
 

vendor/golang.org/x/crypto/curve25519/internal/field/fe_arm64.s

@@ -3,7 +3,6 @@
 // license that can be found in the LICENSE file.
 
 //go:build arm64 && gc && !purego
-// +build arm64,gc,!purego
 
 #include "textflag.h"
 

vendor/golang.org/x/crypto/curve25519/internal/field/fe_arm64_noasm.go

@@ -3,7 +3,6 @@
 // license that can be found in the LICENSE file.
 
 //go:build !arm64 || !gc || purego
-// +build !arm64 !gc purego
 
 package field
 

vendor/golang.org/x/crypto/hkdf/hkdf.go

@@ -56,7 +56,9 @@ func (f *hkdf) Read(p []byte) (int, error) {
 
 	// Fill the rest of the buffer
 	for len(p) > 0 {
-		f.expander.Reset()
+		if f.counter > 1 {
+			f.expander.Reset()
+		}
 		f.expander.Write(f.prev)
 		f.expander.Write(f.info)
 		f.expander.Write([]byte{f.counter})

vendor/golang.org/x/crypto/internal/alias/alias.go

@@ -3,7 +3,6 @@
 // license that can be found in the LICENSE file.
 
 //go:build !purego
-// +build !purego
 
 // Package alias implements memory aliasing tests.
 package alias

vendor/golang.org/x/crypto/internal/alias/alias_purego.go

@@ -3,7 +3,6 @@
 // license that can be found in the LICENSE file.
 
 //go:build purego
-// +build purego
 
 // Package alias implements memory aliasing tests.
 package alias

vendor/golang.org/x/crypto/internal/poly1305/bits_compat.go

@@ -3,7 +3,6 @@
 // license that can be found in the LICENSE file.
 
 //go:build !go1.13
-// +build !go1.13
 
 package poly1305
 

vendor/golang.org/x/crypto/internal/poly1305/bits_go1.13.go

@@ -3,7 +3,6 @@
 // license that can be found in the LICENSE file.
 
 //go:build go1.13
-// +build go1.13
 
 package poly1305