From 99a89ef87a72692b5e3bbaae645c7a656d4304ab Mon Sep 17 00:00:00 2001 From: Aine Date: Wed, 16 Nov 2022 23:00:58 +0200 Subject: [PATCH] update deps; experiment: log security --- cmd/cmd.go | 13 +- go.mod | 10 +- go.sum | 22 +- vendor/github.com/mattn/go-sqlite3/README.md | 16 +- .../mattn/go-sqlite3/sqlite3-binding.c | 307 ++++++++++++------ .../mattn/go-sqlite3/sqlite3-binding.h | 6 +- vendor/github.com/mattn/go-sqlite3/sqlite3.go | 6 +- .../go-sqlite3/sqlite3_opt_math_functions.go | 14 + .../mattn/go-sqlite3/sqlite3_opt_os_trace.go | 15 + .../mattn/go-sqlite3/sqlite3_windows.go | 1 - vendor/github.com/yuin/goldmark/README.md | 1 + .../yuin/goldmark/extension/table.go | 4 + .../yuin/goldmark/parser/paragraph.go | 17 +- .../github.com/yuin/goldmark/parser/parser.go | 4 +- .../etke.cc/linkpearl/accountdata.go | 47 ++- .../etke.cc/linkpearl/config/config.go | 5 + .../gitlab.com/etke.cc/linkpearl/linkpearl.go | 5 + vendor/gitlab.com/etke.cc/linkpearl/send.go | 12 +- .../etke.cc/linkpearl/store/crypto.go | 32 +- .../etke.cc/linkpearl/store/linkpearl.go | 8 +- .../etke.cc/linkpearl/store/state.go | 8 +- .../etke.cc/linkpearl/store/storer.go | 14 +- vendor/gitlab.com/etke.cc/linkpearl/sync.go | 2 +- vendor/golang.org/x/crypto/ssh/common.go | 15 + vendor/golang.org/x/crypto/ssh/handshake.go | 21 +- vendor/golang.org/x/crypto/ssh/server.go | 13 +- vendor/maunium.net/go/mautrix/CHANGELOG.md | 27 ++ .../go/mautrix/appservice/appservice.go | 12 +- .../go/mautrix/appservice/eventprocessor.go | 23 +- .../maunium.net/go/mautrix/appservice/http.go | 12 +- .../go/mautrix/bridge/bridgeconfig/config.go | 12 +- vendor/maunium.net/go/mautrix/client.go | 15 + .../go/mautrix/crypto/encryptolm.go | 2 +- .../maunium.net/go/mautrix/crypto/machine.go | 9 +- .../go/mautrix/crypto/olm/utility.go | 10 +- .../go/mautrix/crypto/sql_store.go | 4 +- .../crypto/sql_store_upgrade/upgrade.go | 2 +- vendor/maunium.net/go/mautrix/error.go | 2 + vendor/maunium.net/go/mautrix/event/events.go | 7 +- .../maunium.net/go/mautrix/event/message.go | 6 +- .../maunium.net/go/mautrix/event/relations.go | 7 + vendor/maunium.net/go/mautrix/event/reply.go | 8 +- vendor/maunium.net/go/mautrix/requests.go | 56 +++- vendor/maunium.net/go/mautrix/responses.go | 26 ++ .../go/mautrix/util/dbutil/connlog.go | 90 ++++- .../go/mautrix/util/dbutil/database.go | 41 ++- .../maunium.net/go/mautrix/util/dbutil/log.go | 22 +- .../mautrix/util/dbutil/samples/04-notxn.sql | 4 + .../dbutil/samples/output/04-postgres.sql | 1 + .../util/dbutil/samples/output/04-sqlite3.sql | 1 + .../go/mautrix/util/dbutil/upgrades.go | 33 +- .../go/mautrix/util/dbutil/upgradetable.go | 42 ++- .../go/mautrix/util/jsontime/jsontime.go | 86 +++++ vendor/maunium.net/go/mautrix/version.go | 2 +- vendor/modules.txt | 11 +- 55 files changed, 883 insertions(+), 308 deletions(-) create mode 100644 vendor/github.com/mattn/go-sqlite3/sqlite3_opt_math_functions.go create mode 100644 vendor/github.com/mattn/go-sqlite3/sqlite3_opt_os_trace.go create mode 100644 vendor/maunium.net/go/mautrix/util/dbutil/samples/04-notxn.sql create mode 100644 vendor/maunium.net/go/mautrix/util/dbutil/samples/output/04-postgres.sql create mode 100644 vendor/maunium.net/go/mautrix/util/dbutil/samples/output/04-sqlite3.sql create mode 100644 vendor/maunium.net/go/mautrix/util/jsontime/jsontime.go diff --git a/cmd/cmd.go b/cmd/cmd.go index f3d2932..7cdf3cf 100644 --- a/cmd/cmd.go +++ b/cmd/cmd.go @@ -83,10 +83,15 @@ func initBot(cfg *config.Config) { Dialect: cfg.DB.Dialect, NoEncryption: cfg.NoEncryption, AccountDataSecret: cfg.DataSecret, - LPLogger: mxlog, - APILogger: logger.New("api.", "INFO"), - StoreLogger: logger.New("store.", "INFO"), - CryptoLogger: logger.New("olm.", "INFO"), + AccountDataLogReplace: map[string]string{ + "password": "", + "dkim.pem": "", + "dkim.pub": "", + }, + LPLogger: mxlog, + APILogger: logger.New("api.", "INFO"), + StoreLogger: logger.New("store.", "INFO"), + CryptoLogger: logger.New("olm.", "INFO"), }) if err != nil { // nolint // Fatal = panic, not os.Exit() diff --git a/go.mod b/go.mod index 6156fc8..f7f0f06 100644 --- a/go.mod +++ b/go.mod @@ -12,7 +12,7 @@ require ( github.com/getsentry/sentry-go v0.13.0 github.com/jhillyerd/enmime v0.10.0 github.com/lib/pq v1.10.7 - github.com/mattn/go-sqlite3 v1.14.15 + github.com/mattn/go-sqlite3 v1.14.16 github.com/mileusna/crontab v1.2.0 github.com/raja/argon2pw v1.0.2-0.20210910183755-a391af63bd39 gitlab.com/etke.cc/go/env v1.0.0 @@ -21,9 +21,9 @@ require ( gitlab.com/etke.cc/go/secgen v1.1.1 gitlab.com/etke.cc/go/trysmtp v1.0.0 gitlab.com/etke.cc/go/validator v1.0.3 - gitlab.com/etke.cc/linkpearl v0.0.0-20221115164843-97f1e49414d9 + gitlab.com/etke.cc/linkpearl v0.0.0-20221116205701-65547c5608e6 golang.org/x/net v0.2.0 - maunium.net/go/mautrix v0.12.2 + maunium.net/go/mautrix v0.12.3 ) require ( @@ -47,8 +47,8 @@ require ( github.com/tidwall/match v1.1.1 // indirect github.com/tidwall/pretty v1.2.1 // indirect github.com/tidwall/sjson v1.2.5 // indirect - github.com/yuin/goldmark v1.5.2 // indirect - golang.org/x/crypto v0.2.0 // indirect + github.com/yuin/goldmark v1.5.3 // indirect + golang.org/x/crypto v0.3.0 // indirect golang.org/x/sys v0.2.0 // indirect golang.org/x/text v0.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect diff --git a/go.sum b/go.sum index 465cbf0..9eb7521 100644 --- a/go.sum +++ b/go.sum @@ -50,8 +50,8 @@ github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/ github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= github.com/mattn/go-runewidth v0.0.12 h1:Y41i/hVW3Pgwr8gV+J23B9YEY0zxjptBuCWEaxmAOow= github.com/mattn/go-runewidth v0.0.12/go.mod h1:RAqKPSqVFrSLVXbA8x7dzmKdmGzieGRCM46jaSJTDAk= -github.com/mattn/go-sqlite3 v1.14.15 h1:vfoHhTN1af61xCRSWzFIWzx2YskyMTwHLrExkBOjvxI= -github.com/mattn/go-sqlite3 v1.14.15/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S2DGjv9HUNg= +github.com/mattn/go-sqlite3 v1.14.16 h1:yOQRA0RpS5PFz/oikGwBEqvAWhWg5ufRz4ETLjwpU1Y= +github.com/mattn/go-sqlite3 v1.14.16/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S2DGjv9HUNg= github.com/mikesmitty/edkey v0.0.0-20170222072505-3356ea4e686a h1:eU8j/ClY2Ty3qdHnn0TyW3ivFoPC/0F1gQZz8yTxbbE= github.com/mikesmitty/edkey v0.0.0-20170222072505-3356ea4e686a/go.mod h1:v8eSC2SMp9/7FTKUncp7fH9IwPfw+ysMObcEz5FWheQ= github.com/mileusna/crontab v1.2.0 h1:x9ZmE2A4p6CDqMEGQ+GbqsNtnmbdmWMQYShdQu8LvrU= @@ -76,7 +76,7 @@ github.com/ssor/bom v0.0.0-20170718123548-6386211fdfcf/go.mod h1:RJID2RhlZKId02n github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.8.0 h1:pSgiaMZlXftHpm5L7V1+rVB+AZJydKsMxsQBIJw4PKk= +github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= github.com/tidwall/gjson v1.14.2/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= github.com/tidwall/gjson v1.14.3 h1:9jvXn7olKEHU1S9vwoMGliaT8jq1vJ7IH/n9zD9Dnlw= github.com/tidwall/gjson v1.14.3/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= @@ -87,8 +87,8 @@ github.com/tidwall/pretty v1.2.1 h1:qjsOFOWWQl+N3RsoF5/ssm1pHmJJwhjlSbZ51I6wMl4= github.com/tidwall/pretty v1.2.1/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= github.com/tidwall/sjson v1.2.5 h1:kLy8mja+1c9jlljvWTlSazM7cKDRfJuR/bOJhcY5NcY= github.com/tidwall/sjson v1.2.5/go.mod h1:Fvgq9kS/6ociJEDnK0Fk1cpYF4FIW6ZF7LAe+6jwd28= -github.com/yuin/goldmark v1.5.2 h1:ALmeCk/px5FSm1MAcFBAsVKZjDuMVj8Tm7FFIlMJnqU= -github.com/yuin/goldmark v1.5.2/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= +github.com/yuin/goldmark v1.5.3 h1:3HUJmBFbQW9fhQOzMgseU134xfi6hU+mjWywx5Ty+/M= +github.com/yuin/goldmark v1.5.3/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= gitlab.com/etke.cc/go/env v1.0.0 h1:J98BwzOuELnjsVPFvz5wa79L7IoRV9CmrS41xLYXtSw= gitlab.com/etke.cc/go/env v1.0.0/go.mod h1:e1l4RM5MA1sc0R1w/RBDAESWRwgo5cOG9gx8BKUn2C4= gitlab.com/etke.cc/go/logger v1.1.0 h1:Yngp/DDLmJ0jJNLvLXrfan5Gi5QV+r7z6kCczTv8t4U= @@ -101,11 +101,11 @@ gitlab.com/etke.cc/go/trysmtp v1.0.0 h1:f/7gSmzohKniVeLSLevI+ZsySYcPUGkT9cRlOTwj gitlab.com/etke.cc/go/trysmtp v1.0.0/go.mod h1:KqRuIB2IPElEEbAxXmFyKtm7S5YiuEb4lxwWthccqyE= gitlab.com/etke.cc/go/validator v1.0.3 h1:qkMskwtA3Uiv1q7HTlNZaaZcIJTO4mp2p0KZAl53Xmo= gitlab.com/etke.cc/go/validator v1.0.3/go.mod h1:3vdssRG4LwgdTr9IHz9MjGSEO+3/FO9hXPGMuSeweJ8= -gitlab.com/etke.cc/linkpearl v0.0.0-20221115164843-97f1e49414d9 h1:U0xXVnRvXYxz/rndKIaksw5DwEV8KBBFjrV5k975Jiw= -gitlab.com/etke.cc/linkpearl v0.0.0-20221115164843-97f1e49414d9/go.mod h1:ClA7UlRUoeydy0a7AbJrGAdWYOX//twuThny198PA1k= +gitlab.com/etke.cc/linkpearl v0.0.0-20221116205701-65547c5608e6 h1:+HDT2/bx3Hug++aeDE/PaoRRcnKdYzEm6i2RlOAzPXo= +gitlab.com/etke.cc/linkpearl v0.0.0-20221116205701-65547c5608e6/go.mod h1:Dgtu0qvymNjjky4Bu5WC8+iSohcb5xZ9CtkD3ezDqIA= golang.org/x/crypto v0.0.0-20220518034528-6f7dac969898/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= -golang.org/x/crypto v0.2.0 h1:BRXPfhNivWL5Yq0BGQ39a2sW6t44aODpfxkWjYdzewE= -golang.org/x/crypto v0.2.0/go.mod h1:hebNnKkNXi2UzZN1eVRvBB7co0a+JxK6XbPiWVs/3J4= +golang.org/x/crypto v0.3.0 h1:a06MkbcxBrEFc0w0QIZWXrH/9cCX6KJyWbBOIwAn+7A= +golang.org/x/crypto v0.3.0/go.mod h1:hebNnKkNXi2UzZN1eVRvBB7co0a+JxK6XbPiWVs/3J4= golang.org/x/net v0.0.0-20210501142056-aec3718b3fa0/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220624214902-1bab6f366d9e/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= @@ -136,5 +136,5 @@ gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= maunium.net/go/maulogger/v2 v2.3.2 h1:1XmIYmMd3PoQfp9J+PaHhpt80zpfmMqaShzUTC7FwY0= maunium.net/go/maulogger/v2 v2.3.2/go.mod h1:TYWy7wKwz/tIXTpsx8G3mZseIRiC5DoMxSZazOHy68A= -maunium.net/go/mautrix v0.12.2 h1:HuIDgigR6VY2QUPyZADCwn8UZWYAqi31a77qd1jMPA4= -maunium.net/go/mautrix v0.12.2/go.mod h1:bCw45Qx/m9qsz7eazmbe7Rzq5ZbTPzwRE1UgX2S9DXs= +maunium.net/go/mautrix v0.12.3 h1:pUeO1ThhtZxE6XibGCzDhRuxwDIFNugsreVr1yYq96k= +maunium.net/go/mautrix v0.12.3/go.mod h1:uOUjkOjm2C+nQS3mr9B5ATjqemZfnPHvjdd1kZezAwg= diff --git a/vendor/github.com/mattn/go-sqlite3/README.md b/vendor/github.com/mattn/go-sqlite3/README.md index e455133..931b02e 100644 --- a/vendor/github.com/mattn/go-sqlite3/README.md +++ b/vendor/github.com/mattn/go-sqlite3/README.md @@ -1,7 +1,7 @@ go-sqlite3 ========== -[![GoDoc Reference](https://godoc.org/github.com/mattn/go-sqlite3?status.svg)](http://godoc.org/github.com/mattn/go-sqlite3) +[![Go Reference](https://pkg.go.dev/badge/github.com/mattn/go-sqlite3.svg)](https://pkg.go.dev/github.com/mattn/go-sqlite3) [![GitHub Actions](https://github.com/mattn/go-sqlite3/workflows/Go/badge.svg)](https://github.com/mattn/go-sqlite3/actions?query=workflow%3AGo) [![Financial Contributors on Open Collective](https://opencollective.com/mattn-go-sqlite3/all/badge.svg?label=financial+contributors)](https://opencollective.com/mattn-go-sqlite3) [![codecov](https://codecov.io/gh/mattn/go-sqlite3/branch/master/graph/badge.svg)](https://codecov.io/gh/mattn/go-sqlite3) @@ -172,15 +172,18 @@ go build --tags "icu json1 fts5 secure_delete" | International Components for Unicode | sqlite_icu | This option causes the International Components for Unicode or "ICU" extension to SQLite to be added to the build | | Introspect PRAGMAS | sqlite_introspect | This option adds some extra PRAGMA statements.
  • PRAGMA function_list
  • PRAGMA module_list
  • PRAGMA pragma_list
| | JSON SQL Functions | sqlite_json | When this option is defined in the amalgamation, the JSON SQL functions are added to the build automatically | +| Math Functions | sqlite_math_functions | This compile-time option enables built-in scalar math functions. For more information see [Built-In Mathematical SQL Functions](https://www.sqlite.org/lang_mathfunc.html) | +| OS Trace | sqlite_os_trace | This option enables OSTRACE() debug logging. This can be verbose and should not be used in production. | | Pre Update Hook | sqlite_preupdate_hook | Registers a callback function that is invoked prior to each INSERT, UPDATE, and DELETE operation on a database table. | | Secure Delete | sqlite_secure_delete | This compile-time option changes the default setting of the secure_delete pragma.

When this option is not used, secure_delete defaults to off. When this option is present, secure_delete defaults to on.

The secure_delete setting causes deleted content to be overwritten with zeros. There is a small performance penalty since additional I/O must occur.

On the other hand, secure_delete can prevent fragments of sensitive information from lingering in unused parts of the database file after it has been deleted. See the documentation on the secure_delete pragma for additional information | | Secure Delete (FAST) | sqlite_secure_delete_fast | For more information see [PRAGMA secure_delete](https://www.sqlite.org/pragma.html#pragma_secure_delete) | | Tracing / Debug | sqlite_trace | Activate trace functions | | User Authentication | sqlite_userauth | SQLite User Authentication see [User Authentication](#user-authentication) for more information. | +| Virtual Tables | sqlite_vtable | SQLite Virtual Tables see [SQLite Official VTABLE Documentation](https://www.sqlite.org/vtab.html) for more information, and a [full example here](https://github.com/mattn/go-sqlite3/tree/master/_example/vtable) | # Compilation -This package requires the `CGO_ENABLED=1` ennvironment variable if not set by default, and the presence of the `gcc` compiler. +This package requires the `CGO_ENABLED=1` environment variable if not set by default, and the presence of the `gcc` compiler. If you need to add additional CFLAGS or LDFLAGS to the build command, and do not want to modify this package, then this can be achieved by using the `CGO_CFLAGS` and `CGO_LDFLAGS` environment variables. @@ -216,14 +219,13 @@ This library can be cross-compiled. In some cases you are required to the `CC` environment variable with the cross compiler. ## Cross Compiling from MAC OSX -The simplest way to cross compile from OSX is to use [xgo](https://github.com/karalabe/xgo). +The simplest way to cross compile from OSX is to use [musl-cross](https://github.com/FiloSottile/homebrew-musl-cross). Steps: -- Install [xgo](https://github.com/karalabe/xgo) (`go get github.com/karalabe/xgo`). -- Ensure that your project is within your `GOPATH`. -- Run `xgo local/path/to/project`. +- Install [musl-cross](https://github.com/FiloSottile/homebrew-musl-cross) (`brew install FiloSottile/musl-cross/musl-cross`). +- Run `CC=x86_64-linux-musl-gcc CXX=x86_64-linux-musl-g++ GOARCH=amd64 GOOS=linux CGO_ENABLED=1 go build -ldflags "-linkmode external -extldflags -static"`. -Please refer to the project's [README](https://github.com/karalabe/xgo/blob/master/README.md) for further information. +Please refer to the project's [README](https://github.com/FiloSottile/homebrew-musl-cross#readme) for further information. # Google Cloud Platform diff --git a/vendor/github.com/mattn/go-sqlite3/sqlite3-binding.c b/vendor/github.com/mattn/go-sqlite3/sqlite3-binding.c index df04099..dd05f57 100644 --- a/vendor/github.com/mattn/go-sqlite3/sqlite3-binding.c +++ b/vendor/github.com/mattn/go-sqlite3/sqlite3-binding.c @@ -1,7 +1,7 @@ #ifndef USE_LIBSQLITE3 /****************************************************************************** ** This file is an amalgamation of many separate C source files from SQLite -** version 3.39.2. By combining all the individual C code files into this +** version 3.39.4. By combining all the individual C code files into this ** single large file, the entire code can be compiled as a single translation ** unit. This allows many compilers to do optimizations that would not be ** possible if the files were compiled separately. Performance improvements @@ -453,9 +453,9 @@ extern "C" { ** [sqlite3_libversion_number()], [sqlite3_sourceid()], ** [sqlite_version()] and [sqlite_source_id()]. */ -#define SQLITE_VERSION "3.39.2" -#define SQLITE_VERSION_NUMBER 3039002 -#define SQLITE_SOURCE_ID "2022-07-21 15:24:47 698edb77537b67c41adc68f9b892db56bcf9a55e00371a61420f3ddd668e6603" +#define SQLITE_VERSION "3.39.4" +#define SQLITE_VERSION_NUMBER 3039004 +#define SQLITE_SOURCE_ID "2022-09-29 15:55:41 a29f9949895322123f7c38fbe94c649a9d6e6c9cd0c3b41c96d694552f26b309" /* ** CAPI3REF: Run-Time Library Version Numbers @@ -13145,6 +13145,11 @@ struct fts5_api { /************** End of sqlite3.h *********************************************/ /************** Continuing where we left off in sqliteInt.h ******************/ +/* +** Reuse the STATIC_LRU for mutex access to sqlite3_temp_directory. +*/ +#define SQLITE_MUTEX_STATIC_TEMPDIR SQLITE_MUTEX_STATIC_VFS1 + /* ** Include the configuration header output by 'configure' if we're using the ** autoconf-based build @@ -29564,8 +29569,13 @@ SQLITE_PRIVATE void *sqlite3OomFault(sqlite3 *db){ } DisableLookaside; if( db->pParse ){ + Parse *pParse; sqlite3ErrorMsg(db->pParse, "out of memory"); db->pParse->rc = SQLITE_NOMEM_BKPT; + for(pParse=db->pParse->pOuterParse; pParse; pParse = pParse->pOuterParse){ + pParse->nErr++; + pParse->rc = SQLITE_NOMEM; + } } } return 0; @@ -33460,7 +33470,7 @@ SQLITE_PRIVATE void sqlite3ErrorMsg(Parse *pParse, const char *zFormat, ...){ va_list ap; sqlite3 *db = pParse->db; assert( db!=0 ); - assert( db->pParse==pParse ); + assert( db->pParse==pParse || db->pParse->pToplevel==pParse ); db->errByteOffset = -2; va_start(ap, zFormat); zMsg = sqlite3VMPrintf(db, zFormat, ap); @@ -41321,6 +41331,7 @@ static const char *unixTempFileDir(void){ static int unixGetTempname(int nBuf, char *zBuf){ const char *zDir; int iLimit = 0; + int rc = SQLITE_OK; /* It's odd to simulate an io-error here, but really this is just ** using the io-error infrastructure to test that SQLite handles this @@ -41329,18 +41340,26 @@ static int unixGetTempname(int nBuf, char *zBuf){ zBuf[0] = 0; SimulateIOError( return SQLITE_IOERR ); + sqlite3_mutex_enter(sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_TEMPDIR)); zDir = unixTempFileDir(); - if( zDir==0 ) return SQLITE_IOERR_GETTEMPPATH; - do{ - u64 r; - sqlite3_randomness(sizeof(r), &r); - assert( nBuf>2 ); - zBuf[nBuf-2] = 0; - sqlite3_snprintf(nBuf, zBuf, "%s/"SQLITE_TEMP_FILE_PREFIX"%llx%c", - zDir, r, 0); - if( zBuf[nBuf-2]!=0 || (iLimit++)>10 ) return SQLITE_ERROR; - }while( osAccess(zBuf,0)==0 ); - return SQLITE_OK; + if( zDir==0 ){ + rc = SQLITE_IOERR_GETTEMPPATH; + }else{ + do{ + u64 r; + sqlite3_randomness(sizeof(r), &r); + assert( nBuf>2 ); + zBuf[nBuf-2] = 0; + sqlite3_snprintf(nBuf, zBuf, "%s/"SQLITE_TEMP_FILE_PREFIX"%llx%c", + zDir, r, 0); + if( zBuf[nBuf-2]!=0 || (iLimit++)>10 ){ + rc = SQLITE_ERROR; + break; + } + }while( osAccess(zBuf,0)==0 ); + } + sqlite3_mutex_leave(sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_TEMPDIR)); + return rc; } #if SQLITE_ENABLE_LOCKING_STYLE && defined(__APPLE__) @@ -45479,10 +45498,12 @@ SQLITE_API int sqlite3_win32_set_directory8( const char *zValue /* New value for directory being set or reset */ ){ char **ppDirectory = 0; + int rc; #ifndef SQLITE_OMIT_AUTOINIT - int rc = sqlite3_initialize(); + rc = sqlite3_initialize(); if( rc ) return rc; #endif + sqlite3_mutex_enter(sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_TEMPDIR)); if( type==SQLITE_WIN32_DATA_DIRECTORY_TYPE ){ ppDirectory = &sqlite3_data_directory; }else if( type==SQLITE_WIN32_TEMP_DIRECTORY_TYPE ){ @@ -45497,14 +45518,19 @@ SQLITE_API int sqlite3_win32_set_directory8( if( zValue && zValue[0] ){ zCopy = sqlite3_mprintf("%s", zValue); if ( zCopy==0 ){ - return SQLITE_NOMEM_BKPT; + rc = SQLITE_NOMEM_BKPT; + goto set_directory8_done; } } sqlite3_free(*ppDirectory); *ppDirectory = zCopy; - return SQLITE_OK; + rc = SQLITE_OK; + }else{ + rc = SQLITE_ERROR; } - return SQLITE_ERROR; +set_directory8_done: + sqlite3_mutex_leave(sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_TEMPDIR)); + return rc; } /* @@ -48278,6 +48304,18 @@ static int winMakeEndInDirSep(int nBuf, char *zBuf){ return 0; } +/* +** If sqlite3_temp_directory is not, take the mutex and return true. +** +** If sqlite3_temp_directory is NULL, omit the mutex and return false. +*/ +static int winTempDirDefined(void){ + sqlite3_mutex_enter(sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_TEMPDIR)); + if( sqlite3_temp_directory!=0 ) return 1; + sqlite3_mutex_leave(sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_TEMPDIR)); + return 0; +} + /* ** Create a temporary file name and store the resulting pointer into pzBuf. ** The pointer returned in pzBuf must be freed via sqlite3_free(). @@ -48314,20 +48352,23 @@ static int winGetTempname(sqlite3_vfs *pVfs, char **pzBuf){ */ nDir = nMax - (nPre + 15); assert( nDir>0 ); - if( sqlite3_temp_directory ){ + if( winTempDirDefined() ){ int nDirLen = sqlite3Strlen30(sqlite3_temp_directory); if( nDirLen>0 ){ if( !winIsDirSep(sqlite3_temp_directory[nDirLen-1]) ){ nDirLen++; } if( nDirLen>nDir ){ + sqlite3_mutex_leave(sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_TEMPDIR)); sqlite3_free(zBuf); OSTRACE(("TEMP-FILENAME rc=SQLITE_ERROR\n")); return winLogError(SQLITE_ERROR, 0, "winGetTempname1", 0); } sqlite3_snprintf(nMax, zBuf, "%s", sqlite3_temp_directory); } + sqlite3_mutex_leave(sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_TEMPDIR)); } + #if defined(__CYGWIN__) else{ static const char *azDirs[] = { @@ -49116,7 +49157,7 @@ static BOOL winIsVerbatimPathname( ** pathname into zOut[]. zOut[] will be at least pVfs->mxPathname ** bytes in size. */ -static int winFullPathname( +static int winFullPathnameNoMutex( sqlite3_vfs *pVfs, /* Pointer to vfs object */ const char *zRelative, /* Possibly relative input path */ int nFull, /* Size of output buffer in bytes */ @@ -49295,6 +49336,19 @@ static int winFullPathname( } #endif } +static int winFullPathname( + sqlite3_vfs *pVfs, /* Pointer to vfs object */ + const char *zRelative, /* Possibly relative input path */ + int nFull, /* Size of output buffer in bytes */ + char *zFull /* Output buffer */ +){ + int rc; + sqlite3_mutex *pMutex = sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_TEMPDIR); + sqlite3_mutex_enter(pMutex); + rc = winFullPathnameNoMutex(pVfs, zRelative, nFull, zFull); + sqlite3_mutex_leave(pMutex); + return rc; +} #ifndef SQLITE_OMIT_LOAD_EXTENSION /* @@ -51639,14 +51693,24 @@ SQLITE_PRIVATE void sqlite3PcacheClearSyncFlags(PCache *pCache){ */ SQLITE_PRIVATE void sqlite3PcacheMove(PgHdr *p, Pgno newPgno){ PCache *pCache = p->pCache; + sqlite3_pcache_page *pOther; assert( p->nRef>0 ); assert( newPgno>0 ); assert( sqlite3PcachePageSanity(p) ); pcacheTrace(("%p.MOVE %d -> %d\n",pCache,p->pgno,newPgno)); + pOther = sqlite3GlobalConfig.pcache2.xFetch(pCache->pCache, newPgno, 0); + if( pOther ){ + PgHdr *pXPage = (PgHdr*)pOther->pExtra; + assert( pXPage->nRef==0 ); + pXPage->nRef++; + pCache->nRefSum++; + sqlite3PcacheDrop(pXPage); + } sqlite3GlobalConfig.pcache2.xRekey(pCache->pCache, p->pPage, p->pgno,newPgno); p->pgno = newPgno; if( (p->flags&PGHDR_DIRTY) && (p->flags&PGHDR_NEED_SYNC) ){ pcacheManageDirtyList(p, PCACHE_DIRTYLIST_FRONT); + assert( sqlite3PcachePageSanity(p) ); } } @@ -53028,23 +53092,26 @@ static void pcache1Rekey( PCache1 *pCache = (PCache1 *)p; PgHdr1 *pPage = (PgHdr1 *)pPg; PgHdr1 **pp; - unsigned int h; + unsigned int hOld, hNew; assert( pPage->iKey==iOld ); assert( pPage->pCache==pCache ); + assert( iOld!=iNew ); /* The page number really is changing */ pcache1EnterMutex(pCache->pGroup); - h = iOld%pCache->nHash; - pp = &pCache->apHash[h]; + assert( pcache1FetchNoMutex(p, iOld, 0)==pPage ); /* pPg really is iOld */ + hOld = iOld%pCache->nHash; + pp = &pCache->apHash[hOld]; while( (*pp)!=pPage ){ pp = &(*pp)->pNext; } *pp = pPage->pNext; - h = iNew%pCache->nHash; + assert( pcache1FetchNoMutex(p, iNew, 0)==0 ); /* iNew not in cache */ + hNew = iNew%pCache->nHash; pPage->iKey = iNew; - pPage->pNext = pCache->apHash[h]; - pCache->apHash[h] = pPage; + pPage->pNext = pCache->apHash[hNew]; + pCache->apHash[hNew] = pPage; if( iNew>pCache->iMaxKey ){ pCache->iMaxKey = iNew; } @@ -59677,6 +59744,7 @@ static int pager_open_journal(Pager *pPager){ if( rc!=SQLITE_OK ){ sqlite3BitvecDestroy(pPager->pInJournal); pPager->pInJournal = 0; + pPager->journalOff = 0; }else{ assert( pPager->eState==PAGER_WRITER_LOCKED ); pPager->eState = PAGER_WRITER_CACHEMOD; @@ -61232,7 +61300,7 @@ SQLITE_PRIVATE int sqlite3PagerGetJournalMode(Pager *pPager){ SQLITE_PRIVATE int sqlite3PagerOkToChangeJournalMode(Pager *pPager){ assert( assert_pager_state(pPager) ); if( pPager->eState>=PAGER_WRITER_CACHEMOD ) return 0; - if( NEVER(isOpen(pPager->jfd) && pPager->journalOff>0) ) return 0; + if( isOpen(pPager->jfd) && pPager->journalOff>0 ) return 0; return 1; } @@ -68347,7 +68415,7 @@ static int defragmentPage(MemPage *pPage, int nMaxFrag){ if( iFree2+sz2 > usableSize ) return SQLITE_CORRUPT_PAGE(pPage); memmove(&data[iFree+sz+sz2], &data[iFree+sz], iFree2-(iFree+sz)); sz += sz2; - }else if( NEVER(iFree+sz>usableSize) ){ + }else if( iFree+sz>usableSize ){ return SQLITE_CORRUPT_PAGE(pPage); } @@ -74703,8 +74771,6 @@ static int balance_nonroot( Pgno pgno; /* Temp var to store a page number in */ u8 abDone[NB+2]; /* True after i'th new page is populated */ Pgno aPgno[NB+2]; /* Page numbers of new pages before shuffling */ - Pgno aPgOrder[NB+2]; /* Copy of aPgno[] used for sorting pages */ - u16 aPgFlags[NB+2]; /* flags field of new pages before shuffling */ CellArray b; /* Parsed information on cells being balanced */ memset(abDone, 0, sizeof(abDone)); @@ -75128,42 +75194,39 @@ static int balance_nonroot( ** of the table is closer to a linear scan through the file. That in turn ** helps the operating system to deliver pages from the disk more rapidly. ** - ** An O(n^2) insertion sort algorithm is used, but since n is never more - ** than (NB+2) (a small constant), that should not be a problem. + ** An O(N*N) sort algorithm is used, but since N is never more than NB+2 + ** (5), that is not a performance concern. ** ** When NB==3, this one optimization makes the database about 25% faster ** for large insertions and deletions. */ for(i=0; ipgno; - aPgFlags[i] = apNew[i]->pDbPage->flags; - for(j=0; jpgno; + assert( apNew[i]->pDbPage->flags & PGHDR_WRITEABLE ); + assert( apNew[i]->pDbPage->flags & PGHDR_DIRTY ); } - for(i=0; ipgno < apNew[iB]->pgno ) iB = j; } - pgno = aPgOrder[iBest]; - aPgOrder[iBest] = 0xffffffff; - if( iBest!=i ){ - if( iBest>i ){ - sqlite3PagerRekey(apNew[iBest]->pDbPage, pBt->nPage+iBest+1, 0); - } - sqlite3PagerRekey(apNew[i]->pDbPage, pgno, aPgFlags[iBest]); - apNew[i]->pgno = pgno; + + /* If apNew[i] has a page number that is bigger than any of the + ** subsequence apNew[i] entries, then swap apNew[i] with the subsequent + ** entry that has the smallest page number (which we know to be + ** entry apNew[iB]). + */ + if( iB!=i ){ + Pgno pgnoA = apNew[i]->pgno; + Pgno pgnoB = apNew[iB]->pgno; + Pgno pgnoTemp = (PENDING_BYTE/pBt->pageSize)+1; + u16 fgA = apNew[i]->pDbPage->flags; + u16 fgB = apNew[iB]->pDbPage->flags; + sqlite3PagerRekey(apNew[i]->pDbPage, pgnoTemp, fgB); + sqlite3PagerRekey(apNew[iB]->pDbPage, pgnoA, fgA); + sqlite3PagerRekey(apNew[i]->pDbPage, pgnoB, fgB); + apNew[i]->pgno = pgnoB; + apNew[iB]->pgno = pgnoA; } } @@ -81036,6 +81099,7 @@ SQLITE_PRIVATE int sqlite3VdbeAddFunctionCall( addr = sqlite3VdbeAddOp4(v, eCallCtx ? OP_PureFunc : OP_Function, p1, p2, p3, (char*)pCtx, P4_FUNCCTX); sqlite3VdbeChangeP5(v, eCallCtx & NC_SelfRef); + sqlite3MayAbort(pParse); return addr; } @@ -81371,6 +81435,7 @@ SQLITE_PRIVATE int sqlite3VdbeAssertMayAbort(Vdbe *v, int mayAbort){ || opcode==OP_VDestroy || opcode==OP_VCreate || opcode==OP_ParseSchema + || opcode==OP_Function || opcode==OP_PureFunc || ((opcode==OP_Halt || opcode==OP_HaltIfNull) && ((pOp->p1)!=SQLITE_OK && pOp->p2==OE_Abort)) ){ @@ -132705,6 +132770,7 @@ SQLITE_PRIVATE void sqlite3Pragma( ** */ case PragTyp_TEMP_STORE_DIRECTORY: { + sqlite3_mutex_enter(sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_TEMPDIR)); if( !zRight ){ returnSingleText(v, sqlite3_temp_directory); }else{ @@ -132714,6 +132780,7 @@ SQLITE_PRIVATE void sqlite3Pragma( rc = sqlite3OsAccess(db->pVfs, zRight, SQLITE_ACCESS_READWRITE, &res); if( rc!=SQLITE_OK || res==0 ){ sqlite3ErrorMsg(pParse, "not a writable directory"); + sqlite3_mutex_leave(sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_TEMPDIR)); goto pragma_out; } } @@ -132731,6 +132798,7 @@ SQLITE_PRIVATE void sqlite3Pragma( } #endif /* SQLITE_OMIT_WSD */ } + sqlite3_mutex_leave(sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_TEMPDIR)); break; } @@ -132749,6 +132817,7 @@ SQLITE_PRIVATE void sqlite3Pragma( ** */ case PragTyp_DATA_STORE_DIRECTORY: { + sqlite3_mutex_enter(sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_TEMPDIR)); if( !zRight ){ returnSingleText(v, sqlite3_data_directory); }else{ @@ -132758,6 +132827,7 @@ SQLITE_PRIVATE void sqlite3Pragma( rc = sqlite3OsAccess(db->pVfs, zRight, SQLITE_ACCESS_READWRITE, &res); if( rc!=SQLITE_OK || res==0 ){ sqlite3ErrorMsg(pParse, "not a writable directory"); + sqlite3_mutex_leave(sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_TEMPDIR)); goto pragma_out; } } @@ -132769,6 +132839,7 @@ SQLITE_PRIVATE void sqlite3Pragma( } #endif /* SQLITE_OMIT_WSD */ } + sqlite3_mutex_leave(sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_TEMPDIR)); break; } #endif @@ -137214,7 +137285,7 @@ static void generateSortTail( if( addrOnce ) sqlite3VdbeJumpHere(v, addrOnce); addr = 1 + sqlite3VdbeAddOp2(v, OP_SorterSort, iTab, addrBreak); VdbeCoverage(v); - codeOffset(v, p->iOffset, addrContinue); + assert( p->iLimit==0 && p->iOffset==0 ); sqlite3VdbeAddOp3(v, OP_SorterData, iTab, regSortOut, iSortTab); bSeq = 0; }else{ @@ -137222,6 +137293,9 @@ static void generateSortTail( codeOffset(v, p->iOffset, addrContinue); iSortTab = iTab; bSeq = 1; + if( p->iOffset>0 ){ + sqlite3VdbeAddOp2(v, OP_AddImm, p->iLimit, -1); + } } for(i=0, iCol=nKey+bSeq-1; ipPrior ){ - sqlite3SelectDelete(db, pSplit->pPrior); + sqlite3ParserAddCleanup(pParse, + (void(*)(sqlite3*,void*))sqlite3SelectDelete, pSplit->pPrior); } pSplit->pPrior = pPrior; pPrior->pNext = pSplit; @@ -140736,6 +140811,7 @@ static Table *isSimpleCount(Select *p, AggInfo *pAggInfo){ || p->pSrc->nSrc!=1 || p->pSrc->a[0].pSelect || pAggInfo->nFunc!=1 + || p->pHaving ){ return 0; } @@ -143973,6 +144049,23 @@ SQLITE_PRIVATE void sqlite3FinishTrigger( Vdbe *v; char *z; + /* If this is a new CREATE TABLE statement, and if shadow tables + ** are read-only, and the trigger makes a change to a shadow table, + ** then raise an error - do not allow the trigger to be created. */ + if( sqlite3ReadOnlyShadowTables(db) ){ + TriggerStep *pStep; + for(pStep=pTrig->step_list; pStep; pStep=pStep->pNext){ + if( pStep->zTarget!=0 + && sqlite3ShadowTableName(db, pStep->zTarget) + ){ + sqlite3ErrorMsg(pParse, + "trigger \"%s\" may not write to shadow table \"%s\"", + pTrig->zName, pStep->zTarget); + goto triggerfinish_cleanup; + } + } + } + /* Make an entry in the sqlite_schema table */ v = sqlite3GetVdbe(pParse); if( v==0 ) goto triggerfinish_cleanup; @@ -149784,7 +149877,8 @@ static int codeEqualityTerm( } sqlite3ExprDelete(db, pX); }else{ - aiMap = (int*)sqlite3DbMallocZero(pParse->db, sizeof(int)*nEq); + int n = sqlite3ExprVectorSize(pX->pLeft); + aiMap = (int*)sqlite3DbMallocZero(pParse->db, sizeof(int)*MAX(nEq,n)); eType = sqlite3FindInIndex(pParse, pX, IN_INDEX_LOOP, 0, aiMap, &iTab); } pX = pExpr; @@ -176937,7 +177031,7 @@ struct Fts3MultiSegReader { int nAdvance; /* How many seg-readers to advance */ Fts3SegFilter *pFilter; /* Pointer to filter object */ char *aBuffer; /* Buffer to merge doclists in */ - int nBuffer; /* Allocated size of aBuffer[] in bytes */ + i64 nBuffer; /* Allocated size of aBuffer[] in bytes */ int iColFilter; /* If >=0, filter for this column */ int bRestart; @@ -179633,7 +179727,7 @@ static int fts3TermSelectMerge( ** ** Similar padding is added in the fts3DoclistOrMerge() function. */ - pTS->aaOutput[0] = sqlite3_malloc(nDoclist + FTS3_VARINT_MAX + 1); + pTS->aaOutput[0] = sqlite3_malloc64((i64)nDoclist + FTS3_VARINT_MAX + 1); pTS->anOutput[0] = nDoclist; if( pTS->aaOutput[0] ){ memcpy(pTS->aaOutput[0], aDoclist, nDoclist); @@ -181121,7 +181215,7 @@ static int fts3EvalDeferredPhrase(Fts3Cursor *pCsr, Fts3Phrase *pPhrase){ nDistance = iPrev - nMaxUndeferred; } - aOut = (char *)sqlite3_malloc(nPoslist+8); + aOut = (char *)sqlite3Fts3MallocZero(nPoslist+FTS3_BUFFER_PADDING); if( !aOut ){ sqlite3_free(aPoslist); return SQLITE_NOMEM; @@ -181490,7 +181584,7 @@ static int fts3EvalIncrPhraseNext( if( bEof==0 ){ int nList = 0; int nByte = a[p->nToken-1].nList; - char *aDoclist = sqlite3_malloc(nByte+FTS3_BUFFER_PADDING); + char *aDoclist = sqlite3_malloc64((i64)nByte+FTS3_BUFFER_PADDING); if( !aDoclist ) return SQLITE_NOMEM; memcpy(aDoclist, a[p->nToken-1].pList, nByte+1); memset(&aDoclist[nByte], 0, FTS3_BUFFER_PADDING); @@ -185726,7 +185820,7 @@ static int porterNext( if( n>c->nAllocated ){ char *pNew; c->nAllocated = n+20; - pNew = sqlite3_realloc(c->zToken, c->nAllocated); + pNew = sqlite3_realloc64(c->zToken, c->nAllocated); if( !pNew ) return SQLITE_NOMEM; c->zToken = pNew; } @@ -186478,7 +186572,7 @@ static int simpleNext( if( n>c->nTokenAllocated ){ char *pNew; c->nTokenAllocated = n+20; - pNew = sqlite3_realloc(c->pToken, c->nTokenAllocated); + pNew = sqlite3_realloc64(c->pToken, c->nTokenAllocated); if( !pNew ) return SQLITE_NOMEM; c->pToken = pNew; } @@ -187640,7 +187734,7 @@ static int fts3PendingListAppendVarint( /* Allocate or grow the PendingList as required. */ if( !p ){ - p = sqlite3_malloc(sizeof(*p) + 100); + p = sqlite3_malloc64(sizeof(*p) + 100); if( !p ){ return SQLITE_NOMEM; } @@ -187649,14 +187743,14 @@ static int fts3PendingListAppendVarint( p->nData = 0; } else if( p->nData+FTS3_VARINT_MAX+1>p->nSpace ){ - int nNew = p->nSpace * 2; - p = sqlite3_realloc(p, sizeof(*p) + nNew); + i64 nNew = p->nSpace * 2; + p = sqlite3_realloc64(p, sizeof(*p) + nNew); if( !p ){ sqlite3_free(*pp); *pp = 0; return SQLITE_NOMEM; } - p->nSpace = nNew; + p->nSpace = (int)nNew; p->aData = (char *)&p[1]; } @@ -188213,7 +188307,7 @@ SQLITE_PRIVATE int sqlite3Fts3ReadBlock( int nByte = sqlite3_blob_bytes(p->pSegments); *pnBlob = nByte; if( paBlob ){ - char *aByte = sqlite3_malloc(nByte + FTS3_NODE_PADDING); + char *aByte = sqlite3_malloc64((i64)nByte + FTS3_NODE_PADDING); if( !aByte ){ rc = SQLITE_NOMEM; }else{ @@ -188330,7 +188424,7 @@ static int fts3SegReaderNext( int nTerm = fts3HashKeysize(pElem); if( (nTerm+1)>pReader->nTermAlloc ){ sqlite3_free(pReader->zTerm); - pReader->zTerm = (char*)sqlite3_malloc((nTerm+1)*2); + pReader->zTerm = (char*)sqlite3_malloc64(((i64)nTerm+1)*2); if( !pReader->zTerm ) return SQLITE_NOMEM; pReader->nTermAlloc = (nTerm+1)*2; } @@ -188338,7 +188432,7 @@ static int fts3SegReaderNext( pReader->zTerm[nTerm] = '\0'; pReader->nTerm = nTerm; - aCopy = (char*)sqlite3_malloc(nCopy); + aCopy = (char*)sqlite3_malloc64(nCopy); if( !aCopy ) return SQLITE_NOMEM; memcpy(aCopy, pList->aData, nCopy); pReader->nNode = pReader->nDoclist = nCopy; @@ -188625,7 +188719,7 @@ SQLITE_PRIVATE int sqlite3Fts3SegReaderNew( nExtra = nRoot + FTS3_NODE_PADDING; } - pReader = (Fts3SegReader *)sqlite3_malloc(sizeof(Fts3SegReader) + nExtra); + pReader = (Fts3SegReader *)sqlite3_malloc64(sizeof(Fts3SegReader) + nExtra); if( !pReader ){ return SQLITE_NOMEM; } @@ -188717,7 +188811,7 @@ SQLITE_PRIVATE int sqlite3Fts3SegReaderPending( if( nElem==nAlloc ){ Fts3HashElem **aElem2; nAlloc += 16; - aElem2 = (Fts3HashElem **)sqlite3_realloc( + aElem2 = (Fts3HashElem **)sqlite3_realloc64( aElem, nAlloc*sizeof(Fts3HashElem *) ); if( !aElem2 ){ @@ -189051,7 +189145,7 @@ static int fts3NodeAddTerm( ** this is not expected to be a serious problem. */ assert( pTree->aData==(char *)&pTree[1] ); - pTree->aData = (char *)sqlite3_malloc(nReq); + pTree->aData = (char *)sqlite3_malloc64(nReq); if( !pTree->aData ){ return SQLITE_NOMEM; } @@ -189069,7 +189163,7 @@ static int fts3NodeAddTerm( if( isCopyTerm ){ if( pTree->nMalloczMalloc, nTerm*2); + char *zNew = sqlite3_realloc64(pTree->zMalloc, (i64)nTerm*2); if( !zNew ){ return SQLITE_NOMEM; } @@ -189095,7 +189189,7 @@ static int fts3NodeAddTerm( ** now. Instead, the term is inserted into the parent of pTree. If pTree ** has no parent, one is created here. */ - pNew = (SegmentNode *)sqlite3_malloc(sizeof(SegmentNode) + p->nNodeSize); + pNew = (SegmentNode *)sqlite3_malloc64(sizeof(SegmentNode) + p->nNodeSize); if( !pNew ){ return SQLITE_NOMEM; } @@ -189233,7 +189327,7 @@ static int fts3SegWriterAdd( ){ int nPrefix; /* Size of term prefix in bytes */ int nSuffix; /* Size of term suffix in bytes */ - int nReq; /* Number of bytes required on leaf page */ + i64 nReq; /* Number of bytes required on leaf page */ int nData; SegmentWriter *pWriter = *ppWriter; @@ -189242,13 +189336,13 @@ static int fts3SegWriterAdd( sqlite3_stmt *pStmt; /* Allocate the SegmentWriter structure */ - pWriter = (SegmentWriter *)sqlite3_malloc(sizeof(SegmentWriter)); + pWriter = (SegmentWriter *)sqlite3_malloc64(sizeof(SegmentWriter)); if( !pWriter ) return SQLITE_NOMEM; memset(pWriter, 0, sizeof(SegmentWriter)); *ppWriter = pWriter; /* Allocate a buffer in which to accumulate data */ - pWriter->aData = (char *)sqlite3_malloc(p->nNodeSize); + pWriter->aData = (char *)sqlite3_malloc64(p->nNodeSize); if( !pWriter->aData ) return SQLITE_NOMEM; pWriter->nSize = p->nNodeSize; @@ -189323,7 +189417,7 @@ static int fts3SegWriterAdd( ** the buffer to make it large enough. */ if( nReq>pWriter->nSize ){ - char *aNew = sqlite3_realloc(pWriter->aData, nReq); + char *aNew = sqlite3_realloc64(pWriter->aData, nReq); if( !aNew ) return SQLITE_NOMEM; pWriter->aData = aNew; pWriter->nSize = nReq; @@ -189348,7 +189442,7 @@ static int fts3SegWriterAdd( */ if( isCopyTerm ){ if( nTerm>pWriter->nMalloc ){ - char *zNew = sqlite3_realloc(pWriter->zMalloc, nTerm*2); + char *zNew = sqlite3_realloc64(pWriter->zMalloc, (i64)nTerm*2); if( !zNew ){ return SQLITE_NOMEM; } @@ -189656,12 +189750,12 @@ static void fts3ColumnFilter( static int fts3MsrBufferData( Fts3MultiSegReader *pMsr, /* Multi-segment-reader handle */ char *pList, - int nList + i64 nList ){ if( nList>pMsr->nBuffer ){ char *pNew; pMsr->nBuffer = nList*2; - pNew = (char *)sqlite3_realloc(pMsr->aBuffer, pMsr->nBuffer); + pNew = (char *)sqlite3_realloc64(pMsr->aBuffer, pMsr->nBuffer); if( !pNew ) return SQLITE_NOMEM; pMsr->aBuffer = pNew; } @@ -189717,7 +189811,7 @@ SQLITE_PRIVATE int sqlite3Fts3MsrIncrNext( fts3SegReaderSort(pMsr->apSegment, nMerge, j, xCmp); if( nList>0 && fts3SegReaderIsPending(apSegment[0]) ){ - rc = fts3MsrBufferData(pMsr, pList, nList+1); + rc = fts3MsrBufferData(pMsr, pList, (i64)nList+1); if( rc!=SQLITE_OK ) return rc; assert( (pMsr->aBuffer[nList] & 0xFE)==0x00 ); pList = pMsr->aBuffer; @@ -189854,11 +189948,11 @@ SQLITE_PRIVATE int sqlite3Fts3MsrIncrRestart(Fts3MultiSegReader *pCsr){ return SQLITE_OK; } -static int fts3GrowSegReaderBuffer(Fts3MultiSegReader *pCsr, int nReq){ +static int fts3GrowSegReaderBuffer(Fts3MultiSegReader *pCsr, i64 nReq){ if( nReq>pCsr->nBuffer ){ char *aNew; pCsr->nBuffer = nReq*2; - aNew = sqlite3_realloc(pCsr->aBuffer, pCsr->nBuffer); + aNew = sqlite3_realloc64(pCsr->aBuffer, pCsr->nBuffer); if( !aNew ){ return SQLITE_NOMEM; } @@ -189949,7 +190043,8 @@ SQLITE_PRIVATE int sqlite3Fts3SegReaderStep( ){ pCsr->nDoclist = apSegment[0]->nDoclist; if( fts3SegReaderIsPending(apSegment[0]) ){ - rc = fts3MsrBufferData(pCsr, apSegment[0]->aDoclist, pCsr->nDoclist); + rc = fts3MsrBufferData(pCsr, apSegment[0]->aDoclist, + (i64)pCsr->nDoclist); pCsr->aDoclist = pCsr->aBuffer; }else{ pCsr->aDoclist = apSegment[0]->aDoclist; @@ -190002,7 +190097,8 @@ SQLITE_PRIVATE int sqlite3Fts3SegReaderStep( nByte = sqlite3Fts3VarintLen(iDelta) + (isRequirePos?nList+1:0); - rc = fts3GrowSegReaderBuffer(pCsr, nByte+nDoclist+FTS3_NODE_PADDING); + rc = fts3GrowSegReaderBuffer(pCsr, + (i64)nByte+nDoclist+FTS3_NODE_PADDING); if( rc ) return rc; if( isFirst ){ @@ -190028,7 +190124,7 @@ SQLITE_PRIVATE int sqlite3Fts3SegReaderStep( fts3SegReaderSort(apSegment, nMerge, j, xCmp); } if( nDoclist>0 ){ - rc = fts3GrowSegReaderBuffer(pCsr, nDoclist+FTS3_NODE_PADDING); + rc = fts3GrowSegReaderBuffer(pCsr, (i64)nDoclist+FTS3_NODE_PADDING); if( rc ) return rc; memset(&pCsr->aBuffer[nDoclist], 0, FTS3_NODE_PADDING); pCsr->aDoclist = pCsr->aBuffer; @@ -190741,7 +190837,7 @@ struct NodeReader { static void blobGrowBuffer(Blob *pBlob, int nMin, int *pRc){ if( *pRc==SQLITE_OK && nMin>pBlob->nAlloc ){ int nAlloc = nMin; - char *a = (char *)sqlite3_realloc(pBlob->a, nAlloc); + char *a = (char *)sqlite3_realloc64(pBlob->a, nAlloc); if( a ){ pBlob->nAlloc = nAlloc; pBlob->a = a; @@ -191538,7 +191634,7 @@ static int fts3RepackSegdirLevel( if( nIdx>=nAlloc ){ int *aNew; nAlloc += 16; - aNew = sqlite3_realloc(aIdx, nAlloc*sizeof(int)); + aNew = sqlite3_realloc64(aIdx, nAlloc*sizeof(int)); if( !aNew ){ rc = SQLITE_NOMEM; break; @@ -191912,7 +192008,7 @@ SQLITE_PRIVATE int sqlite3Fts3Incrmerge(Fts3Table *p, int nMerge, int nMin){ /* Allocate space for the cursor, filter and writer objects */ const int nAlloc = sizeof(*pCsr) + sizeof(*pFilter) + sizeof(*pWriter); - pWriter = (IncrmergeWriter *)sqlite3_malloc(nAlloc); + pWriter = (IncrmergeWriter *)sqlite3_malloc64(nAlloc); if( !pWriter ) return SQLITE_NOMEM; pFilter = (Fts3SegFilter *)&pWriter[1]; pCsr = (Fts3MultiSegReader *)&pFilter[1]; @@ -192548,7 +192644,7 @@ SQLITE_PRIVATE int sqlite3Fts3DeferredTokenList( return SQLITE_OK; } - pRet = (char *)sqlite3_malloc(p->pList->nData); + pRet = (char *)sqlite3_malloc64(p->pList->nData); if( !pRet ) return SQLITE_NOMEM; nSkip = sqlite3Fts3GetVarint(p->pList->aData, &dummy); @@ -192568,7 +192664,7 @@ SQLITE_PRIVATE int sqlite3Fts3DeferToken( int iCol /* Column that token must appear in (or -1) */ ){ Fts3DeferredToken *pDeferred; - pDeferred = sqlite3_malloc(sizeof(*pDeferred)); + pDeferred = sqlite3_malloc64(sizeof(*pDeferred)); if( !pDeferred ){ return SQLITE_NOMEM; } @@ -204147,7 +204243,7 @@ static int geopolyUpdate( sqlite3_free(p); nChange = 1; } - for(jj=1; jjnAux; jj++){ + for(jj=1; jj. +// +// Use of this source code is governed by an MIT-style +// license that can be found in the LICENSE file. + +// +build sqlite_math_functions + +package sqlite3 + +/* +#cgo CFLAGS: -DSQLITE_ENABLE_MATH_FUNCTIONS +#cgo LDFLAGS: -lm +*/ +import "C" diff --git a/vendor/github.com/mattn/go-sqlite3/sqlite3_opt_os_trace.go b/vendor/github.com/mattn/go-sqlite3/sqlite3_opt_os_trace.go new file mode 100644 index 0000000..9a30566 --- /dev/null +++ b/vendor/github.com/mattn/go-sqlite3/sqlite3_opt_os_trace.go @@ -0,0 +1,15 @@ +// Copyright (C) 2022 Yasuhiro Matsumoto . +// +// Use of this source code is governed by an MIT-style +// license that can be found in the LICENSE file. + +//go:build sqlite_os_trace +// +build sqlite_os_trace + +package sqlite3 + +/* +#cgo CFLAGS: -DSQLITE_FORCE_OS_TRACE=1 +#cgo CFLAGS: -DSQLITE_DEBUG_OS_TRACE=1 +*/ +import "C" diff --git a/vendor/github.com/mattn/go-sqlite3/sqlite3_windows.go b/vendor/github.com/mattn/go-sqlite3/sqlite3_windows.go index 9df1961..81aa2ab 100644 --- a/vendor/github.com/mattn/go-sqlite3/sqlite3_windows.go +++ b/vendor/github.com/mattn/go-sqlite3/sqlite3_windows.go @@ -12,7 +12,6 @@ package sqlite3 #cgo CFLAGS: -fno-stack-check #cgo CFLAGS: -fno-stack-protector #cgo CFLAGS: -mno-stack-arg-probe -#cgo LDFLAGS: -lmingwex -lmingw32 #cgo windows,386 CFLAGS: -D_USE_32BIT_TIME_T */ import "C" diff --git a/vendor/github.com/yuin/goldmark/README.md b/vendor/github.com/yuin/goldmark/README.md index dfcb6da..c99ecb5 100644 --- a/vendor/github.com/yuin/goldmark/README.md +++ b/vendor/github.com/yuin/goldmark/README.md @@ -445,6 +445,7 @@ Extensions - [goldmark-pikchr](https://github.com/jchenry/goldmark-pikchr): Adds support for rendering [Pikchr](https://pikchr.org/home/doc/trunk/homepage.md) diagrams in goldmark documents. - [goldmark-embed](https://github.com/13rac1/goldmark-embed): Adds support for rendering embeds from YouTube links. - [goldmark-latex](https://github.com/soypat/goldmark-latex): A $\LaTeX$ renderer that can be passed to `goldmark.WithRenderer()`. +- [goldmark-fences](https://github.com/stefanfritsch/goldmark-fences): Support for pandoc-style [fenced divs](https://pandoc.org/MANUAL.html#divs-and-spans) in goldmark. goldmark internal(for extension developers) diff --git a/vendor/github.com/yuin/goldmark/extension/table.go b/vendor/github.com/yuin/goldmark/extension/table.go index c637b99..48d0d68 100644 --- a/vendor/github.com/yuin/goldmark/extension/table.go +++ b/vendor/github.com/yuin/goldmark/extension/table.go @@ -122,6 +122,9 @@ func WithTableCellAlignMethod(a TableCellAlignMethod) TableOption { } func isTableDelim(bs []byte) bool { + if w, _ := util.IndentWidth(bs, 0); w > 3 { + return false + } for _, b := range bs { if !(util.IsSpace(b) || b == '-' || b == '|' || b == ':') { return false @@ -243,6 +246,7 @@ func (b *tableParagraphTransformer) parseRow(segment text.Segment, alignments [] } func (b *tableParagraphTransformer) parseDelimiter(segment text.Segment, reader text.Reader) []ast.Alignment { + line := segment.Value(reader.Source()) if !isTableDelim(line) { return nil diff --git a/vendor/github.com/yuin/goldmark/parser/paragraph.go b/vendor/github.com/yuin/goldmark/parser/paragraph.go index 2dd2b9a..9d3fa38 100644 --- a/vendor/github.com/yuin/goldmark/parser/paragraph.go +++ b/vendor/github.com/yuin/goldmark/parser/paragraph.go @@ -3,6 +3,7 @@ package parser import ( "github.com/yuin/goldmark/ast" "github.com/yuin/goldmark/text" + "github.com/yuin/goldmark/util" ) type paragraphParser struct { @@ -33,9 +34,8 @@ func (b *paragraphParser) Open(parent ast.Node, reader text.Reader, pc Context) } func (b *paragraphParser) Continue(node ast.Node, reader text.Reader, pc Context) State { - _, segment := reader.PeekLine() - segment = segment.TrimLeftSpace(reader.Source()) - if segment.IsEmpty() { + line, segment := reader.PeekLine() + if util.IsBlank(line) { return Close } node.Lines().Append(segment) @@ -44,13 +44,14 @@ func (b *paragraphParser) Continue(node ast.Node, reader text.Reader, pc Context } func (b *paragraphParser) Close(node ast.Node, reader text.Reader, pc Context) { - parent := node.Parent() - if parent == nil { - // paragraph has been transformed - return - } lines := node.Lines() if lines.Len() != 0 { + // trim leading spaces + for i := 0; i < lines.Len(); i++ { + l := lines.At(i) + lines.Set(i, l.TrimLeftSpace(reader.Source())) + } + // trim trailing spaces length := lines.Len() lastLine := node.Lines().At(length - 1) diff --git a/vendor/github.com/yuin/goldmark/parser/parser.go b/vendor/github.com/yuin/goldmark/parser/parser.go index c8d8aed..a823692 100644 --- a/vendor/github.com/yuin/goldmark/parser/parser.go +++ b/vendor/github.com/yuin/goldmark/parser/parser.go @@ -899,11 +899,13 @@ func (p *parser) closeBlocks(from, to int, reader text.Reader, pc Context) { blocks := pc.OpenedBlocks() for i := from; i >= to; i-- { node := blocks[i].Node - blocks[i].Parser.Close(blocks[i].Node, reader, pc) paragraph, ok := node.(*ast.Paragraph) if ok && node.Parent() != nil { p.transformParagraph(paragraph, reader, pc) } + if node.Parent() != nil { // closes only if node has not been transformed + blocks[i].Parser.Close(blocks[i].Node, reader, pc) + } } if from == len(blocks)-1 { blocks = blocks[0:to] diff --git a/vendor/gitlab.com/etke.cc/linkpearl/accountdata.go b/vendor/gitlab.com/etke.cc/linkpearl/accountdata.go index 4c5a223..955a357 100644 --- a/vendor/gitlab.com/etke.cc/linkpearl/accountdata.go +++ b/vendor/gitlab.com/etke.cc/linkpearl/accountdata.go @@ -10,18 +10,17 @@ import ( func (l *Linkpearl) GetAccountData(name string) (map[string]string, error) { cached, ok := l.acc.Get(name) if ok { - l.log.Debug("GetAccountData(%s) from cache (data): %+v", name, cached) + l.logAccountData(l.log.Debug, "GetAccountData(%q) cached:", cached, name) if cached == nil { return map[string]string{}, nil } return cached, nil } - l.log.Debug("GetAccountData(%s) from API", name) var data map[string]string err := l.GetClient().GetAccountData(name, &data) if err != nil { - l.log.Debug("GetAccountData(%s) from API (error): %v", name, err) + l.logAccountData(l.log.Debug, "GetAccountData(%q) error: %v", nil, name, err) data = map[string]string{} if strings.Contains(err.Error(), "M_NOT_FOUND") { l.acc.Add(name, data) @@ -30,7 +29,7 @@ func (l *Linkpearl) GetAccountData(name string) (map[string]string, error) { return data, err } data = l.decryptAccountData(data) - l.log.Debug("GetAccountData(%s) from API (data): %+v", name, data) + l.logAccountData(l.log.Debug, "GetAccountData(%q):", data, name) l.acc.Add(name, data) return data, err @@ -40,7 +39,7 @@ func (l *Linkpearl) GetAccountData(name string) (map[string]string, error) { func (l *Linkpearl) SetAccountData(name string, data map[string]string) error { l.acc.Add(name, data) - l.log.Debug("SetAccountData(%s) to API (data): %+v", name, data) + l.logAccountData(l.log.Debug, "SetAccountData(%q):", data, name) data = l.encryptAccountData(data) return l.GetClient().SetAccountData(name, data) } @@ -50,18 +49,17 @@ func (l *Linkpearl) GetRoomAccountData(roomID id.RoomID, name string) (map[strin key := roomID.String() + name cached, ok := l.acc.Get(key) if ok { - l.log.Debug("GetRoomAccountData(%s, %s) from cache (data): %+v", roomID, name, cached) + l.logAccountData(l.log.Debug, "GetRoomAccountData(%q, %q) cached:", cached, roomID, name) if cached == nil { return map[string]string{}, nil } return cached, nil } - l.log.Debug("GetRoomAccountData(%s, %s) from API", roomID, name) var data map[string]string err := l.GetClient().GetRoomAccountData(roomID, name, &data) if err != nil { - l.log.Debug("GetRoomAccountData(%s, %s) from API (error): %v", roomID, name, err) + l.logAccountData(l.log.Debug, "GetRoomAccountData(%q, %q) error: %v", nil, roomID, name, err) data = map[string]string{} if strings.Contains(err.Error(), "M_NOT_FOUND") { l.acc.Add(key, data) @@ -70,7 +68,7 @@ func (l *Linkpearl) GetRoomAccountData(roomID id.RoomID, name string) (map[strin return data, err } data = l.decryptAccountData(data) - l.log.Debug("GetRoomAccountData(%s,%s) from API (data): %+v", roomID, name, data) + l.logAccountData(l.log.Debug, "GetRoomAccountData(%q, %q):", data, roomID, name) l.acc.Add(key, data) return data, err @@ -81,7 +79,7 @@ func (l *Linkpearl) SetRoomAccountData(roomID id.RoomID, name string, data map[s key := roomID.String() + name l.acc.Add(key, data) - l.log.Debug("SetRoomAccountData(%s, %s) to API (data): %+v", roomID, name, data) + l.logAccountData(l.log.Debug, "SetRoomAccountData(%q, %q):", data, roomID, name) data = l.encryptAccountData(data) return l.GetClient().SetRoomAccountData(roomID, name, data) } @@ -95,11 +93,11 @@ func (l *Linkpearl) encryptAccountData(data map[string]string) map[string]string for k, v := range data { ek, err := l.acr.Encrypt(k) if err != nil { - l.log.Error("cannot encrypt account data (key=%s): %v", k, err) + l.log.Error("cannot encrypt account data (key=%q): %v", k, err) } ev, err := l.acr.Encrypt(v) if err != nil { - l.log.Error("cannot encrypt account data (key=%s): %v", k, err) + l.log.Error("cannot encrypt account data (key=%q): %v", k, err) } encrypted[ek] = ev // worst case: plaintext value } @@ -116,14 +114,35 @@ func (l *Linkpearl) decryptAccountData(data map[string]string) map[string]string for ek, ev := range data { k, err := l.acr.Decrypt(ek) if err != nil { - l.log.Error("cannot decrypt account data (key=%s): %v", k, err) + l.log.Error("cannot decrypt account data (key=%q): %v", k, err) } v, err := l.acr.Decrypt(ev) if err != nil { - l.log.Error("cannot decrypt account data (key=%s): %v", k, err) + l.log.Error("cannot decrypt account data (key=%q): %v", k, err) } decrypted[k] = v // worst case: encrypted value, usual case: migration from plaintext to encrypted account data } return decrypted } + +func (l *Linkpearl) logAccountData(method func(string, ...any), message string, data map[string]string, args ...any) { + if len(data) == 0 { + method(message, args...) + return + } + + safeData := make(map[string]string, len(data)) + for k, v := range data { + sv, ok := l.aclr[k] + if ok { + safeData[k] = sv + continue + } + + safeData[k] = v + } + args = append(args, safeData) + + method(message+" %+v", args...) +} diff --git a/vendor/gitlab.com/etke.cc/linkpearl/config/config.go b/vendor/gitlab.com/etke.cc/linkpearl/config/config.go index 1c3d161..1e46240 100644 --- a/vendor/gitlab.com/etke.cc/linkpearl/config/config.go +++ b/vendor/gitlab.com/etke.cc/linkpearl/config/config.go @@ -32,6 +32,11 @@ type Config struct { // AccountDataSecret (Password) for encryption AccountDataSecret string + // AccountDataLogReplace contains map of field name => value + // that will be used to replace mentioned account data fields with provided values + // when printing in logs (DEBUG, TRACE) + AccountDataLogReplace map[string]string + // MaxRetries for operations like auto join MaxRetries int diff --git a/vendor/gitlab.com/etke.cc/linkpearl/linkpearl.go b/vendor/gitlab.com/etke.cc/linkpearl/linkpearl.go index 57d4c0f..b9172f7 100644 --- a/vendor/gitlab.com/etke.cc/linkpearl/linkpearl.go +++ b/vendor/gitlab.com/etke.cc/linkpearl/linkpearl.go @@ -25,6 +25,7 @@ type Linkpearl struct { db *sql.DB acc *lru.Cache[string, map[string]string] acr *Crypter + aclr map[string]string log config.Logger api *mautrix.Client olm *crypto.OlmMachine @@ -41,6 +42,9 @@ type ReqPresence struct { } func setDefaults(cfg *config.Config) { + if cfg.AccountDataLogReplace == nil { + cfg.AccountDataLogReplace = make(map[string]string) + } if cfg.MaxRetries == 0 { cfg.MaxRetries = DefaultMaxRetries } @@ -80,6 +84,7 @@ func New(cfg *config.Config) (*Linkpearl, error) { db: cfg.DB, acc: acc, acr: acr, + aclr: cfg.AccountDataLogReplace, api: api, log: cfg.LPLogger, joinPermit: cfg.JoinPermit, diff --git a/vendor/gitlab.com/etke.cc/linkpearl/send.go b/vendor/gitlab.com/etke.cc/linkpearl/send.go index 43ce1d2..7ebb514 100644 --- a/vendor/gitlab.com/etke.cc/linkpearl/send.go +++ b/vendor/gitlab.com/etke.cc/linkpearl/send.go @@ -10,10 +10,10 @@ import ( // Send a message to the roomID and automatically try to encrypt it, if the destination room is encrypted func (l *Linkpearl) Send(roomID id.RoomID, content interface{}) (id.EventID, error) { if !l.store.IsEncrypted(roomID) { - l.log.Debug("room %s is not encrypted", roomID) + l.log.Debug("room %q is not encrypted", roomID) return l.SendPlaintext(roomID, content) } - l.log.Debug("room %s is encrypted", roomID) + l.log.Debug("room %q is encrypted", roomID) encrypted, err := l.EncryptEvent(roomID, content) if err != nil { @@ -28,7 +28,7 @@ func (l *Linkpearl) Send(roomID id.RoomID, content interface{}) (id.EventID, err func (l *Linkpearl) SendFile(roomID id.RoomID, req *mautrix.ReqUploadMedia, msgtype event.MessageType, relation *event.RelatesTo) error { resp, err := l.GetClient().UploadMedia(*req) if err != nil { - l.log.Error("cannot upload file %s: %v", req.FileName, err) + l.log.Error("cannot upload file %q: %v", req.FileName, err) return err } _, err = l.Send(roomID, &event.Content{ @@ -40,7 +40,7 @@ func (l *Linkpearl) SendFile(roomID id.RoomID, req *mautrix.ReqUploadMedia, msgt }, }) if err != nil { - l.log.Error("cannot send uploaded file: %s: %v", req.FileName, err) + l.log.Error("cannot send uploaded file: %q: %v", req.FileName, err) } return err @@ -48,7 +48,7 @@ func (l *Linkpearl) SendFile(roomID id.RoomID, req *mautrix.ReqUploadMedia, msgt // SendPlaintext sends plaintext event only func (l *Linkpearl) SendPlaintext(roomID id.RoomID, content interface{}) (id.EventID, error) { - l.log.Debug("sending plaintext event to %s: %+v", roomID, content) + l.log.Debug("sending plaintext event to %q: %+v", roomID, content) resp, err := l.api.SendMessageEvent(roomID, event.EventMessage, content) if err != nil { return "", err @@ -58,7 +58,7 @@ func (l *Linkpearl) SendPlaintext(roomID id.RoomID, content interface{}) (id.Eve // SendEncrypted sends encrypted event only func (l *Linkpearl) SendEncrypted(roomID id.RoomID, content interface{}) (id.EventID, error) { - l.log.Debug("sending encrypted event to %s: %+v", roomID, content) + l.log.Debug("sending encrypted event to %q: %+v", roomID, content) resp, err := l.api.SendMessageEvent(roomID, event.EventEncrypted, content) if err != nil { return "", err diff --git a/vendor/gitlab.com/etke.cc/linkpearl/store/crypto.go b/vendor/gitlab.com/etke.cc/linkpearl/store/crypto.go index 24bea91..1d5a409 100644 --- a/vendor/gitlab.com/etke.cc/linkpearl/store/crypto.go +++ b/vendor/gitlab.com/etke.cc/linkpearl/store/crypto.go @@ -42,43 +42,43 @@ func (s *Store) GetAccount() (*crypto.OlmAccount, error) { // HasSession returns whether there is an Olm session for the given sender key. func (s *Store) HasSession(key id.SenderKey) bool { - s.log.Debug("check if olm session exists for the key %s", key) + s.log.Debug("check if olm session exists for the key %q", key) return s.s.HasSession(key) } // GetSessions returns all the known Olm sessions for a sender key. func (s *Store) GetSessions(key id.SenderKey) (crypto.OlmSessionList, error) { - s.log.Debug("loading olm session for the key %s", key) + s.log.Debug("loading olm session for the key %q", key) return s.s.GetSessions(key) } // GetLatestSession retrieves the Olm session for a given sender key from the database that has the largest ID. func (s *Store) GetLatestSession(key id.SenderKey) (*crypto.OlmSession, error) { - s.log.Debug("loading latest session for the key %s", key) + s.log.Debug("loading latest session for the key %q", key) return s.s.GetLatestSession(key) } // AddSession persists an Olm session for a sender in the database. func (s *Store) AddSession(key id.SenderKey, session *crypto.OlmSession) error { - s.log.Debug("adding new olm session for the key %s", key) + s.log.Debug("adding new olm session for the key %q", key) return s.s.AddSession(key, session) } // UpdateSession replaces the Olm session for a sender in the database. func (s *Store) UpdateSession(key id.SenderKey, session *crypto.OlmSession) error { - s.log.Debug("update olm session for the key %s", key) + s.log.Debug("update olm session for the key %q", key) return s.s.UpdateSession(key, session) } // PutGroupSession stores an inbound Megolm group session for a room, sender and session. func (s *Store) PutGroupSession(roomID id.RoomID, senderKey id.SenderKey, sessionID id.SessionID, session *crypto.InboundGroupSession) error { - s.log.Debug("storing inbound group session for the room %s", roomID) + s.log.Debug("storing inbound group session for the room %q", roomID) return s.s.PutGroupSession(roomID, senderKey, sessionID, session) } // GetGroupSession retrieves an inbound Megolm group session for a room, sender and session. func (s *Store) GetGroupSession(roomID id.RoomID, senderKey id.SenderKey, sessionID id.SessionID) (*crypto.InboundGroupSession, error) { - s.log.Debug("loading inbound group session for the room %s", roomID) + s.log.Debug("loading inbound group session for the room %q", roomID) return s.s.GetGroupSession(roomID, senderKey, sessionID) } @@ -98,7 +98,7 @@ func (s *Store) GetWithheldGroupSession(roomID id.RoomID, senderKey id.SenderKey // GetGroupSessionsForRoom gets all the inbound Megolm sessions for a specific room. This is used for creating key // export files. Unlike GetGroupSession, this should not return any errors about withheld keys. func (s *Store) GetGroupSessionsForRoom(roomID id.RoomID) ([]*crypto.InboundGroupSession, error) { - s.log.Debug("loading group session for the room %s", roomID) + s.log.Debug("loading group session for the room %q", roomID) return s.s.GetGroupSessionsForRoom(roomID) } @@ -143,31 +143,31 @@ func (s *Store) ValidateMessageIndex(senderKey id.SenderKey, sessionID id.Sessio // GetDevices returns a map of device IDs to device identities, including the identity and signing keys, for a given user ID. func (s *Store) GetDevices(userID id.UserID) (map[id.DeviceID]*id.Device, error) { - s.log.Debug("loading devices of the %s", userID) + s.log.Debug("loading devices of the %q", userID) return s.s.GetDevices(userID) } // GetDevice returns the device dentity for a given user and device ID. func (s *Store) GetDevice(userID id.UserID, deviceID id.DeviceID) (*id.Device, error) { - s.log.Debug("loading device %s for the %s", deviceID, userID) + s.log.Debug("loading device %q for the %q", deviceID, userID) return s.s.GetDevice(userID, deviceID) } // FindDeviceByKey finds a specific device by its sender key. func (s *Store) FindDeviceByKey(userID id.UserID, identityKey id.IdentityKey) (*id.Device, error) { - s.log.Debug("loading device of the %s by the key %s", userID, identityKey) + s.log.Debug("loading device of the %q by the key %q", userID, identityKey) return s.s.FindDeviceByKey(userID, identityKey) } // PutDevice stores a single device for a user, replacing it if it exists already. func (s *Store) PutDevice(userID id.UserID, device *id.Device) error { - s.log.Debug("storing device of the %s", userID) + s.log.Debug("storing device of the %q", userID) return s.s.PutDevice(userID, device) } // PutDevices stores the device identity information for the given user ID. func (s *Store) PutDevices(userID id.UserID, devices map[id.DeviceID]*id.Device) error { - s.log.Debug("storing devices of the %s", userID) + s.log.Debug("storing devices of the %q", userID) return s.s.PutDevices(userID, devices) } @@ -179,13 +179,13 @@ func (s *Store) FilterTrackedUsers(users []id.UserID) ([]id.UserID, error) { // PutCrossSigningKey stores a cross-signing key of some user along with its usage. func (s *Store) PutCrossSigningKey(userID id.UserID, usage id.CrossSigningUsage, key id.Ed25519) error { - s.log.Debug("storing crosssigning key of the %s", userID) + s.log.Debug("storing crosssigning key of the %q", userID) return s.s.PutCrossSigningKey(userID, usage, key) } // GetCrossSigningKeys retrieves a user's stored cross-signing keys. func (s *Store) GetCrossSigningKeys(userID id.UserID) (map[id.CrossSigningUsage]id.CrossSigningKey, error) { - s.log.Debug("loading crosssigning keys of the %s", userID) + s.log.Debug("loading crosssigning keys of the %q", userID) return s.s.GetCrossSigningKeys(userID) } @@ -209,6 +209,6 @@ func (s *Store) IsKeySignedBy(userID id.UserID, key id.Ed25519, signerID id.User // DropSignaturesByKey deletes the signatures made by the given user and key from the store. It returns the number of signatures deleted. func (s *Store) DropSignaturesByKey(userID id.UserID, key id.Ed25519) (int64, error) { - s.log.Debug("removing signatures by the %s/%s", userID, key) + s.log.Debug("removing signatures by the %q/%q", userID, key) return s.s.DropSignaturesByKey(userID, key) } diff --git a/vendor/gitlab.com/etke.cc/linkpearl/store/linkpearl.go b/vendor/gitlab.com/etke.cc/linkpearl/store/linkpearl.go index 05be891..b570028 100644 --- a/vendor/gitlab.com/etke.cc/linkpearl/store/linkpearl.go +++ b/vendor/gitlab.com/etke.cc/linkpearl/store/linkpearl.go @@ -20,7 +20,7 @@ func (s *Store) IsEncrypted(roomID id.RoomID) bool { return false } - s.log.Debug("checking if room %s is encrypted", roomID) + s.log.Debug("checking if room %q is encrypted", roomID) return s.GetEncryptionEvent(roomID) != nil } @@ -79,7 +79,7 @@ func (s *Store) SetEncryptionEvent(evt *event.Event) { // SetMembership saves room members func (s *Store) SetMembership(evt *event.Event) { - s.log.Debug("saving membership event for %s", evt.RoomID) + s.log.Debug("saving membership event for %q", evt.RoomID) tx, err := s.db.Begin() if err != nil { s.log.Error("cannot begin transaction: %v", err) @@ -127,7 +127,7 @@ func (s *Store) SetMembership(evt *event.Event) { // GetRoomMembers ... func (s *Store) GetRoomMembers(roomID id.RoomID) []id.UserID { - s.log.Debug("loading room members of %s", roomID) + s.log.Debug("loading room members of %q", roomID) query := "SELECT user_id FROM room_members WHERE room_id = $1" rows, err := s.db.Query(query, roomID) users := make([]id.UserID, 0) @@ -148,7 +148,7 @@ func (s *Store) GetRoomMembers(roomID id.RoomID) []id.UserID { // SaveSession to DB func (s *Store) SaveSession(userID id.UserID, deviceID id.DeviceID, accessToken string) { - s.log.Debug("saving session credentials of %s/%s", userID, deviceID) + s.log.Debug("saving session credentials of %q/%q", userID, deviceID) tx, err := s.db.Begin() if err != nil { s.log.Error("cannot begin transaction: %v", err) diff --git a/vendor/gitlab.com/etke.cc/linkpearl/store/state.go b/vendor/gitlab.com/etke.cc/linkpearl/store/state.go index c096539..e14ba83 100644 --- a/vendor/gitlab.com/etke.cc/linkpearl/store/state.go +++ b/vendor/gitlab.com/etke.cc/linkpearl/store/state.go @@ -16,7 +16,7 @@ func (s *Store) GetEncryptionEvent(roomID id.RoomID) *event.EncryptionEventConte if !s.encryption { return nil } - s.log.Debug("finding encryption event of %s", roomID) + s.log.Debug("finding encryption event of %q", roomID) query := "SELECT encryption_event FROM rooms WHERE room_id = $1" row := s.db.QueryRow(query, roomID) @@ -28,7 +28,7 @@ func (s *Store) GetEncryptionEvent(roomID id.RoomID) *event.EncryptionEventConte } var encryptionEvent event.EncryptionEventContent if err := json.Unmarshal(encryptionEventJSON, &encryptionEvent); err != nil { - s.log.Debug("cannot unmarshal encryption event: %s", err) + s.log.Debug("cannot unmarshal encryption event: %q", err) return nil } @@ -40,12 +40,12 @@ func (s *Store) FindSharedRooms(userID id.UserID) []id.RoomID { if !s.encryption { return nil } - s.log.Debug("loading shared rooms for %s", userID) + s.log.Debug("loading shared rooms for %q", userID) query := "SELECT room_id FROM room_members WHERE user_id = $1" rows, queryErr := s.db.Query(query, userID) rooms := make([]id.RoomID, 0) if queryErr != nil { - s.log.Error("cannot load room members: %s", queryErr) + s.log.Error("cannot load room members: %q", queryErr) return rooms } defer rows.Close() diff --git a/vendor/gitlab.com/etke.cc/linkpearl/store/storer.go b/vendor/gitlab.com/etke.cc/linkpearl/store/storer.go index 7b62dce..f46c651 100644 --- a/vendor/gitlab.com/etke.cc/linkpearl/store/storer.go +++ b/vendor/gitlab.com/etke.cc/linkpearl/store/storer.go @@ -10,7 +10,7 @@ import ( // SaveFilterID to DB func (s *Store) SaveFilterID(userID id.UserID, filterID string) { - s.log.Debug("saving filter ID %s for %s", filterID, userID) + s.log.Debug("saving filter ID %q for %q", filterID, userID) tx, err := s.db.Begin() if err != nil { s.log.Error("cannot begin transaction: %v", err) @@ -52,12 +52,12 @@ func (s *Store) SaveFilterID(userID id.UserID, filterID string) { // LoadFilterID from DB func (s *Store) LoadFilterID(userID id.UserID) string { - s.log.Debug("loading filter ID for %s", userID) + s.log.Debug("loading filter ID for %q", userID) query := "SELECT filter_id FROM user_filter_ids WHERE user_id = $1" row := s.db.QueryRow(query, userID) var filterID string if err := row.Scan(&filterID); err != nil { - s.log.Error("cannot load filter ID: %s", err) + s.log.Error("cannot load filter ID: %q", err) return "" } return filterID @@ -65,7 +65,7 @@ func (s *Store) LoadFilterID(userID id.UserID) string { // SaveNextBatch to DB func (s *Store) SaveNextBatch(userID id.UserID, nextBatchToken string) { - s.log.Debug("saving next batch token for %s", userID) + s.log.Debug("saving next batch token for %q", userID) tx, err := s.db.Begin() if err != nil { s.log.Error("cannot begin transaction: %v", err) @@ -103,7 +103,7 @@ func (s *Store) SaveNextBatch(userID id.UserID, nextBatchToken string) { // LoadNextBatch from DB func (s *Store) LoadNextBatch(userID id.UserID) string { - s.log.Debug("loading next batch token for %s", userID) + s.log.Debug("loading next batch token for %q", userID) query := "SELECT next_batch_token FROM user_batch_tokens WHERE user_id = $1" row := s.db.QueryRow(query, userID) var batchToken string @@ -116,11 +116,11 @@ func (s *Store) LoadNextBatch(userID id.UserID) string { // SaveRoom to DB, not implemented func (s *Store) SaveRoom(room *mautrix.Room) { - s.log.Debug("saving room %s (stub, not implemented)", room.ID) + s.log.Debug("saving room %q (stub, not implemented)", room.ID) } // LoadRoom from DB, not implemented func (s *Store) LoadRoom(roomID id.RoomID) *mautrix.Room { - s.log.Debug("loading room %s (stub, not implemented)", roomID) + s.log.Debug("loading room %q (stub, not implemented)", roomID) return mautrix.NewRoom(roomID) } diff --git a/vendor/gitlab.com/etke.cc/linkpearl/sync.go b/vendor/gitlab.com/etke.cc/linkpearl/sync.go index 1186757..d3618a8 100644 --- a/vendor/gitlab.com/etke.cc/linkpearl/sync.go +++ b/vendor/gitlab.com/etke.cc/linkpearl/sync.go @@ -78,7 +78,7 @@ func (l *Linkpearl) tryJoin(roomID id.RoomID, retry int) { _, err := l.api.JoinRoomByID(roomID) if err != nil { - l.log.Error("cannot join the room %s: %v", roomID, err) + l.log.Error("cannot join the room %q: %v", roomID, err) time.Sleep(5 * time.Second) l.log.Debug("trying to join again (%d/%d)", retry+1, l.maxretries) l.tryJoin(roomID, retry+1) diff --git a/vendor/golang.org/x/crypto/ssh/common.go b/vendor/golang.org/x/crypto/ssh/common.go index 7a5ff2d..c796427 100644 --- a/vendor/golang.org/x/crypto/ssh/common.go +++ b/vendor/golang.org/x/crypto/ssh/common.go @@ -10,6 +10,7 @@ import ( "fmt" "io" "math" + "strings" "sync" _ "crypto/sha1" @@ -118,6 +119,20 @@ func algorithmsForKeyFormat(keyFormat string) []string { } } +// supportedPubKeyAuthAlgos specifies the supported client public key +// authentication algorithms. Note that this doesn't include certificate types +// since those use the underlying algorithm. This list is sent to the client if +// it supports the server-sig-algs extension. Order is irrelevant. +var supportedPubKeyAuthAlgos = []string{ + KeyAlgoED25519, + KeyAlgoSKED25519, KeyAlgoSKECDSA256, + KeyAlgoECDSA256, KeyAlgoECDSA384, KeyAlgoECDSA521, + KeyAlgoRSASHA256, KeyAlgoRSASHA512, KeyAlgoRSA, + KeyAlgoDSA, +} + +var supportedPubKeyAuthAlgosList = strings.Join(supportedPubKeyAuthAlgos, ",") + // unexpectedMessageError results when the SSH message that we received didn't // match what we wanted. func unexpectedMessageError(expected, got uint8) error { diff --git a/vendor/golang.org/x/crypto/ssh/handshake.go b/vendor/golang.org/x/crypto/ssh/handshake.go index 653dc4d..2b84c35 100644 --- a/vendor/golang.org/x/crypto/ssh/handshake.go +++ b/vendor/golang.org/x/crypto/ssh/handshake.go @@ -615,7 +615,8 @@ func (t *handshakeTransport) enterKeyExchange(otherInitPacket []byte) error { return err } - if t.sessionID == nil { + firstKeyExchange := t.sessionID == nil + if firstKeyExchange { t.sessionID = result.H } result.SessionID = t.sessionID @@ -626,6 +627,24 @@ func (t *handshakeTransport) enterKeyExchange(otherInitPacket []byte) error { if err = t.conn.writePacket([]byte{msgNewKeys}); err != nil { return err } + + // On the server side, after the first SSH_MSG_NEWKEYS, send a SSH_MSG_EXT_INFO + // message with the server-sig-algs extension if the client supports it. See + // RFC 8308, Sections 2.4 and 3.1. + if !isClient && firstKeyExchange && contains(clientInit.KexAlgos, "ext-info-c") { + extInfo := &extInfoMsg{ + NumExtensions: 1, + Payload: make([]byte, 0, 4+15+4+len(supportedPubKeyAuthAlgosList)), + } + extInfo.Payload = appendInt(extInfo.Payload, len("server-sig-algs")) + extInfo.Payload = append(extInfo.Payload, "server-sig-algs"...) + extInfo.Payload = appendInt(extInfo.Payload, len(supportedPubKeyAuthAlgosList)) + extInfo.Payload = append(extInfo.Payload, supportedPubKeyAuthAlgosList...) + if err := t.conn.writePacket(Marshal(extInfo)); err != nil { + return err + } + } + if packet, err := t.conn.readPacket(); err != nil { return err } else if packet[0] != msgNewKeys { diff --git a/vendor/golang.org/x/crypto/ssh/server.go b/vendor/golang.org/x/crypto/ssh/server.go index 2260b20..9e38702 100644 --- a/vendor/golang.org/x/crypto/ssh/server.go +++ b/vendor/golang.org/x/crypto/ssh/server.go @@ -291,15 +291,6 @@ func (s *connection) serverHandshake(config *ServerConfig) (*Permissions, error) return perms, err } -func isAcceptableAlgo(algo string) bool { - switch algo { - case KeyAlgoRSA, KeyAlgoRSASHA256, KeyAlgoRSASHA512, KeyAlgoDSA, KeyAlgoECDSA256, KeyAlgoECDSA384, KeyAlgoECDSA521, KeyAlgoSKECDSA256, KeyAlgoED25519, KeyAlgoSKED25519, - CertAlgoRSAv01, CertAlgoDSAv01, CertAlgoECDSA256v01, CertAlgoECDSA384v01, CertAlgoECDSA521v01, CertAlgoSKECDSA256v01, CertAlgoED25519v01, CertAlgoSKED25519v01: - return true - } - return false -} - func checkSourceAddress(addr net.Addr, sourceAddrs string) error { if addr == nil { return errors.New("ssh: no address known for client, but source-address match required") @@ -514,7 +505,7 @@ userAuthLoop: return nil, parseError(msgUserAuthRequest) } algo := string(algoBytes) - if !isAcceptableAlgo(algo) { + if !contains(supportedPubKeyAuthAlgos, underlyingAlgo(algo)) { authErr = fmt.Errorf("ssh: algorithm %q not accepted", algo) break } @@ -572,7 +563,7 @@ userAuthLoop: // algorithm name that corresponds to algo with // sig.Format. This is usually the same, but // for certs, the names differ. - if !isAcceptableAlgo(sig.Format) { + if !contains(supportedPubKeyAuthAlgos, sig.Format) { authErr = fmt.Errorf("ssh: algorithm %q not accepted", sig.Format) break } diff --git a/vendor/maunium.net/go/mautrix/CHANGELOG.md b/vendor/maunium.net/go/mautrix/CHANGELOG.md index 1422fdf..44f35b0 100644 --- a/vendor/maunium.net/go/mautrix/CHANGELOG.md +++ b/vendor/maunium.net/go/mautrix/CHANGELOG.md @@ -1,3 +1,30 @@ +## v0.12.3 (2022-11-16) + +* **Breaking change:** Added logging for row iteration in the dbutil package. + This changes the return type of `Query` methods from `*sql.Rows` to a new + `dbutil.Rows` interface. +* Added flag to disable wrapping database upgrades in a transaction (e.g. to + allow setting `PRAGMA`s for advanced table mutations on SQLite). +* Deprecated `MessageEventContent.GetReplyTo` in favor of directly using + `RelatesTo.GetReplyTo`. RelatesTo methods are nil-safe, so checking if + RelatesTo is nil is not necessary for using those methods. +* Added wrapper for space hierarchyendpoint (thanks to [@mgcm] in [#100]). +* Added bridge config option to handle transactions asynchronously. +* Added separate channels for to-device events in appservice transaction + handler to avoid blocking to-device events behind normal events. +* Added `RelatesTo.GetNonFallbackReplyTo` utility method to get the reply event + ID, unless the reply is a thread fallback. +* Added `event.TextToHTML` as an utility method to HTML-escape a string and + replace newlines with `
`. +* Added check to bridge encryption helper to make sure the e2ee keys are still + on the server. Synapse is known to sometimes lose keys randomly. +* Changed bridge crypto syncer to crash on `M_UNKNOWN_TOKEN` errors instead of + retrying forever pointlessly. +* Fixed verifying signatures of fallback one-time keys. + +[@mgcm]: https://github.com/mgcm +[#100]: https://github.com/mautrix/go/pull/100 + ## v0.12.2 (2022-10-16) * Added utility method to redact bridge commands. diff --git a/vendor/maunium.net/go/mautrix/appservice/appservice.go b/vendor/maunium.net/go/mautrix/appservice/appservice.go index 6f30125..c14ced2 100644 --- a/vendor/maunium.net/go/mautrix/appservice/appservice.go +++ b/vendor/maunium.net/go/mautrix/appservice/appservice.go @@ -96,11 +96,12 @@ type AppService struct { txnIDC *TransactionIDCache - Events chan *event.Event `yaml:"-"` - DeviceLists chan *mautrix.DeviceLists `yaml:"-"` - OTKCounts chan *mautrix.OTKCount `yaml:"-"` - QueryHandler QueryHandler `yaml:"-"` - StateStore StateStore `yaml:"-"` + Events chan *event.Event `yaml:"-"` + ToDeviceEvents chan *event.Event `yaml:"-"` + DeviceLists chan *mautrix.DeviceLists `yaml:"-"` + OTKCounts chan *mautrix.OTKCount `yaml:"-"` + QueryHandler QueryHandler `yaml:"-"` + StateStore StateStore `yaml:"-"` Router *mux.Router `yaml:"-"` UserAgent string `yaml:"-"` @@ -275,6 +276,7 @@ func (as *AppService) BotClient() *mautrix.Client { // Init initializes the logger and loads the registration of this appservice. func (as *AppService) Init() (bool, error) { as.Events = make(chan *event.Event, EventChannelSize) + as.ToDeviceEvents = make(chan *event.Event, EventChannelSize) as.OTKCounts = make(chan *mautrix.OTKCount, OTKChannelSize) as.DeviceLists = make(chan *mautrix.DeviceLists, EventChannelSize) as.QueryHandler = &QueryHandlerStub{} diff --git a/vendor/maunium.net/go/mautrix/appservice/eventprocessor.go b/vendor/maunium.net/go/mautrix/appservice/eventprocessor.go index bb41ec6..d8d5ef1 100644 --- a/vendor/maunium.net/go/mautrix/appservice/eventprocessor.go +++ b/vendor/maunium.net/go/mautrix/appservice/eventprocessor.go @@ -137,12 +137,22 @@ func (ep *EventProcessor) Dispatch(evt *event.Event) { } } } - -func (ep *EventProcessor) Start() { +func (ep *EventProcessor) startEvents() { for { select { case evt := <-ep.as.Events: ep.Dispatch(evt) + case <-ep.stop: + return + } + } +} + +func (ep *EventProcessor) startEncryption() { + for { + select { + case evt := <-ep.as.ToDeviceEvents: + ep.Dispatch(evt) case otk := <-ep.as.OTKCounts: ep.DispatchOTK(otk) case dl := <-ep.as.DeviceLists: @@ -153,6 +163,11 @@ func (ep *EventProcessor) Start() { } } -func (ep *EventProcessor) Stop() { - ep.stop <- struct{}{} +func (ep *EventProcessor) Start() { + go ep.startEvents() + go ep.startEncryption() +} + +func (ep *EventProcessor) Stop() { + close(ep.stop) } diff --git a/vendor/maunium.net/go/mautrix/appservice/http.go b/vendor/maunium.net/go/mautrix/appservice/http.go index 55f87c5..fd19296 100644 --- a/vendor/maunium.net/go/mautrix/appservice/http.go +++ b/vendor/maunium.net/go/mautrix/appservice/http.go @@ -206,13 +206,19 @@ func (as *AppService) handleEvents(evts []*event.Event, defaultTypeClass event.T } if evt.Type.IsState() { - // TODO remove this check after https://github.com/matrix-org/synapse/pull/11265 + // TODO remove this check after making sure the log doesn't happen historical, ok := evt.Content.Raw["org.matrix.msc2716.historical"].(bool) - if !ok || !historical { + if ok && historical { + as.Log.Warnfln("Received historical state event %s (%s/%s)", evt.ID, evt.Type.Type, evt.GetStateKey()) + } else { as.UpdateState(evt) } } - as.Events <- evt + if evt.Type.Class == event.ToDeviceEventType { + as.ToDeviceEvents <- evt + } else { + as.Events <- evt + } } } diff --git a/vendor/maunium.net/go/mautrix/bridge/bridgeconfig/config.go b/vendor/maunium.net/go/mautrix/bridge/bridgeconfig/config.go index 578978f..d51d95a 100644 --- a/vendor/maunium.net/go/mautrix/bridge/bridgeconfig/config.go +++ b/vendor/maunium.net/go/mautrix/bridge/bridgeconfig/config.go @@ -58,16 +58,17 @@ type AppserviceConfig struct { ASToken string `yaml:"as_token"` HSToken string `yaml:"hs_token"` - EphemeralEvents bool `yaml:"ephemeral_events"` + EphemeralEvents bool `yaml:"ephemeral_events"` + AsyncTransactions bool `yaml:"async_transactions"` } -func (config *BaseConfig) MakeUserIDRegex() *regexp.Regexp { - usernamePlaceholder := util.RandomString(16) +func (config *BaseConfig) MakeUserIDRegex(matcher string) *regexp.Regexp { + usernamePlaceholder := strings.ToLower(util.RandomString(16)) usernameTemplate := fmt.Sprintf("@%s:%s", config.Bridge.FormatUsername(usernamePlaceholder), config.Homeserver.Domain) usernameTemplate = regexp.QuoteMeta(usernameTemplate) - usernameTemplate = strings.Replace(usernameTemplate, usernamePlaceholder, ".+", 1) + usernameTemplate = strings.Replace(usernameTemplate, usernamePlaceholder, matcher, 1) usernameTemplate = fmt.Sprintf("^%s$", usernameTemplate) return regexp.MustCompile(usernameTemplate) } @@ -84,7 +85,7 @@ func (config *BaseConfig) GenerateRegistration() *appservice.Registration { regexp.QuoteMeta(config.AppService.Bot.Username), regexp.QuoteMeta(config.Homeserver.Domain))) registration.Namespaces.UserIDs.Register(botRegex, true) - registration.Namespaces.UserIDs.Register(config.MakeUserIDRegex(), true) + registration.Namespaces.UserIDs.Register(config.MakeUserIDRegex(".*"), true) return registration } @@ -230,6 +231,7 @@ func doUpgrade(helper *up.Helper) { helper.Copy(up.Str, "appservice", "bot", "displayname") helper.Copy(up.Str, "appservice", "bot", "avatar") helper.Copy(up.Bool, "appservice", "ephemeral_events") + helper.Copy(up.Bool, "appservice", "async_transactions") helper.Copy(up.Str, "appservice", "as_token") helper.Copy(up.Str, "appservice", "hs_token") diff --git a/vendor/maunium.net/go/mautrix/client.go b/vendor/maunium.net/go/mautrix/client.go index 1260466..bff7476 100644 --- a/vendor/maunium.net/go/mautrix/client.go +++ b/vendor/maunium.net/go/mautrix/client.go @@ -1468,6 +1468,18 @@ func (cli *Client) JoinedRooms() (resp *RespJoinedRooms, err error) { return } +// Hierarchy returns a list of rooms that are in the room's hierarchy. See https://spec.matrix.org/v1.4/client-server-api/#get_matrixclientv1roomsroomidhierarchy +// +// The hierarchy API is provided to walk the space tree and discover the rooms with their aesthetic details. works in a depth-first manner: +// when it encounters another space as a child it recurses into that space before returning non-space children. +// +// The second function parameter specifies query parameters to limit the response. No query parameters will be added if it's nil. +func (cli *Client) Hierarchy(roomID id.RoomID, req *ReqHierarchy) (resp *RespHierarchy, err error) { + urlPath := cli.BuildURLWithQuery(ClientURLPath{"v1", "rooms", roomID, "hierarchy"}, req.Query()) + _, err = cli.MakeRequest(http.MethodGet, urlPath, nil, &resp) + return +} + // Messages returns a list of message and state events for a room. It uses // pagination query parameters to paginate history in the room. // See https://spec.matrix.org/v1.2/client-server-api/#get_matrixclientv3roomsroomidmessages @@ -1760,6 +1772,9 @@ func (cli *Client) BatchSend(roomID id.RoomID, req *ReqBatchSend) (resp *RespBat if req.BeeperNewMessages { query["com.beeper.new_messages"] = "true" } + if req.BeeperMarkReadBy != "" { + query["com.beeper.mark_read_by"] = req.BeeperMarkReadBy.String() + } if len(req.BatchID) > 0 { query["batch_id"] = req.BatchID.String() } diff --git a/vendor/maunium.net/go/mautrix/crypto/encryptolm.go b/vendor/maunium.net/go/mautrix/crypto/encryptolm.go index 9182ee6..65987ab 100644 --- a/vendor/maunium.net/go/mautrix/crypto/encryptolm.go +++ b/vendor/maunium.net/go/mautrix/crypto/encryptolm.go @@ -97,7 +97,7 @@ func (mach *OlmMachine) createOutboundSessions(input map[id.UserID]map[id.Device continue } identity := input[userID][deviceID] - if ok, err := olm.VerifySignatureJSON(oneTimeKey, userID, deviceID.String(), identity.SigningKey); err != nil { + if ok, err := olm.VerifySignatureJSON(oneTimeKey.RawData, userID, deviceID.String(), identity.SigningKey); err != nil { mach.Log.Error("Failed to verify signature for %s of %s: %v", deviceID, userID, err) } else if !ok { mach.Log.Warn("Invalid signature for %s of %s", deviceID, userID) diff --git a/vendor/maunium.net/go/mautrix/crypto/machine.go b/vendor/maunium.net/go/mautrix/crypto/machine.go index b451148..86d1e3c 100644 --- a/vendor/maunium.net/go/mautrix/crypto/machine.go +++ b/vendor/maunium.net/go/mautrix/crypto/machine.go @@ -445,15 +445,20 @@ func (mach *OlmMachine) WaitForSession(roomID id.RoomID, senderKey id.SenderKey, mach.keyWaitersLock.Lock() ch, ok := mach.keyWaiters[sessionID] if !ok { - ch := make(chan struct{}) + ch = make(chan struct{}) mach.keyWaiters[sessionID] = ch } mach.keyWaitersLock.Unlock() + // Handle race conditions where a session appears between the failed decryption and WaitForSession call. + sess, err := mach.CryptoStore.GetGroupSession(roomID, senderKey, sessionID) + if sess != nil || errors.Is(err, ErrGroupSessionWithheld) { + return true + } select { case <-ch: return true case <-time.After(timeout): - sess, err := mach.CryptoStore.GetGroupSession(roomID, senderKey, sessionID) + sess, err = mach.CryptoStore.GetGroupSession(roomID, senderKey, sessionID) // Check if the session somehow appeared in the store without telling us // We accept withheld sessions as received, as then the decryption attempt will show the error. return sess != nil || errors.Is(err, ErrGroupSessionWithheld) diff --git a/vendor/maunium.net/go/mautrix/crypto/olm/utility.go b/vendor/maunium.net/go/mautrix/crypto/olm/utility.go index ebac6cb..5d2564b 100644 --- a/vendor/maunium.net/go/mautrix/crypto/olm/utility.go +++ b/vendor/maunium.net/go/mautrix/crypto/olm/utility.go @@ -107,9 +107,13 @@ func (u *Utility) VerifySignature(message string, key id.Ed25519, signature stri // https://matrix.org/speculator/spec/drafts%2Fe2e/appendices.html#signing-json // If the _obj is a struct, the `json` tags will be honored. func (u *Utility) VerifySignatureJSON(obj interface{}, userID id.UserID, keyName string, key id.Ed25519) (bool, error) { - objJSON, err := json.Marshal(obj) - if err != nil { - return false, err + var err error + objJSON, ok := obj.(json.RawMessage) + if !ok { + objJSON, err = json.Marshal(obj) + if err != nil { + return false, err + } } sig := gjson.GetBytes(objJSON, util.GJSONPath("signatures", string(userID), fmt.Sprintf("ed25519:%s", keyName))) if !sig.Exists() || sig.Type != gjson.String { diff --git a/vendor/maunium.net/go/mautrix/crypto/sql_store.go b/vendor/maunium.net/go/mautrix/crypto/sql_store.go index a96ee9d..b72b4a4 100644 --- a/vendor/maunium.net/go/mautrix/crypto/sql_store.go +++ b/vendor/maunium.net/go/mautrix/crypto/sql_store.go @@ -302,7 +302,7 @@ func (store *SQLCryptoStore) GetWithheldGroupSession(roomID id.RoomID, senderKey }, nil } -func (store *SQLCryptoStore) scanGroupSessionList(rows *sql.Rows) (result []*InboundGroupSession, err error) { +func (store *SQLCryptoStore) scanGroupSessionList(rows dbutil.Rows) (result []*InboundGroupSession, err error) { for rows.Next() { var roomID id.RoomID var signingKey, senderKey, forwardingChains sql.NullString @@ -577,7 +577,7 @@ func (store *SQLCryptoStore) PutDevices(userID id.UserID, devices map[id.DeviceI // FilterTrackedUsers finds all the user IDs out of the given ones for which the database contains identity information. func (store *SQLCryptoStore) FilterTrackedUsers(users []id.UserID) ([]id.UserID, error) { - var rows *sql.Rows + var rows dbutil.Rows var err error if store.DB.Dialect == dbutil.Postgres && PostgresArrayWrapper != nil { rows, err = store.DB.Query("SELECT user_id FROM crypto_tracked_user WHERE user_id = ANY($1)", PostgresArrayWrapper(users)) diff --git a/vendor/maunium.net/go/mautrix/crypto/sql_store_upgrade/upgrade.go b/vendor/maunium.net/go/mautrix/crypto/sql_store_upgrade/upgrade.go index ff1034d..3a3a884 100644 --- a/vendor/maunium.net/go/mautrix/crypto/sql_store_upgrade/upgrade.go +++ b/vendor/maunium.net/go/mautrix/crypto/sql_store_upgrade/upgrade.go @@ -22,7 +22,7 @@ const VersionTableName = "crypto_version" var fs embed.FS func init() { - Table.Register(-1, 3, "Unsupported version", func(tx dbutil.Transaction, database *dbutil.Database) error { + Table.Register(-1, 3, "Unsupported version", false, func(tx dbutil.Execable, database *dbutil.Database) error { return fmt.Errorf("upgrading from versions 1 and 2 of the crypto store is no longer supported in mautrix-go v0.12+") }) Table.RegisterFS(fs) diff --git a/vendor/maunium.net/go/mautrix/error.go b/vendor/maunium.net/go/mautrix/error.go index 2fc0dc2..36fa74f 100644 --- a/vendor/maunium.net/go/mautrix/error.go +++ b/vendor/maunium.net/go/mautrix/error.go @@ -58,6 +58,8 @@ var ( // The client attempted to join a room that has a version the server does not support. // Inspect the room_version property of the error response for the room's version. MIncompatibleRoomVersion = RespError{ErrCode: "M_INCOMPATIBLE_ROOM_VERSION"} + // The client specified a parameter that has the wrong value. + MInvalidParam = RespError{ErrCode: "M_INVALID_PARAM"} ) // HTTPError An HTTP Error response, which may wrap an underlying native Go Error. diff --git a/vendor/maunium.net/go/mautrix/event/events.go b/vendor/maunium.net/go/mautrix/event/events.go index 7808ba9..29608b5 100644 --- a/vendor/maunium.net/go/mautrix/event/events.go +++ b/vendor/maunium.net/go/mautrix/event/events.go @@ -124,9 +124,10 @@ func (evt *Event) GetStateKey() string { } type StrippedState struct { - Content Content `json:"content"` - Type Type `json:"type"` - StateKey string `json:"state_key"` + Content Content `json:"content"` + Type Type `json:"type"` + StateKey string `json:"state_key"` + Sender id.UserID `json:"sender"` } type Unsigned struct { diff --git a/vendor/maunium.net/go/mautrix/event/message.go b/vendor/maunium.net/go/mautrix/event/message.go index 54e2a13..9412a7a 100644 --- a/vendor/maunium.net/go/mautrix/event/message.go +++ b/vendor/maunium.net/go/mautrix/event/message.go @@ -138,9 +138,13 @@ func (content *MessageEventContent) SetEdit(original id.EventID) { } } +func TextToHTML(text string) string { + return strings.ReplaceAll(html.EscapeString(text), "\n", "
") +} + func (content *MessageEventContent) EnsureHasHTML() { if len(content.FormattedBody) == 0 || content.Format != FormatHTML { - content.FormattedBody = strings.ReplaceAll(html.EscapeString(content.Body), "\n", "
") + content.FormattedBody = TextToHTML(content.Body) content.Format = FormatHTML } } diff --git a/vendor/maunium.net/go/mautrix/event/relations.go b/vendor/maunium.net/go/mautrix/event/relations.go index bd43cc1..0248323 100644 --- a/vendor/maunium.net/go/mautrix/event/relations.go +++ b/vendor/maunium.net/go/mautrix/event/relations.go @@ -70,6 +70,13 @@ func (rel *RelatesTo) GetReplyTo() id.EventID { return "" } +func (rel *RelatesTo) GetNonFallbackReplyTo() id.EventID { + if rel != nil && rel.InReplyTo != nil && !rel.IsFallingBack { + return rel.InReplyTo.EventID + } + return "" +} + func (rel *RelatesTo) GetAnnotationID() id.EventID { if rel != nil && rel.Type == RelAnnotation { return rel.EventID diff --git a/vendor/maunium.net/go/mautrix/event/reply.go b/vendor/maunium.net/go/mautrix/event/reply.go index c3f5af8..e1844a4 100644 --- a/vendor/maunium.net/go/mautrix/event/reply.go +++ b/vendor/maunium.net/go/mautrix/event/reply.go @@ -35,7 +35,7 @@ func TrimReplyFallbackText(text string) string { } func (content *MessageEventContent) RemoveReplyFallback() { - if len(content.GetReplyTo()) > 0 && !content.replyFallbackRemoved { + if len(content.RelatesTo.GetReplyTo()) > 0 && !content.replyFallbackRemoved { if content.Format == FormatHTML { content.FormattedBody = TrimReplyFallbackHTML(content.FormattedBody) } @@ -44,11 +44,9 @@ func (content *MessageEventContent) RemoveReplyFallback() { } } +// Deprecated: RelatesTo methods are nil-safe, so RelatesTo.GetReplyTo can be used directly func (content *MessageEventContent) GetReplyTo() id.EventID { - if content.RelatesTo != nil { - return content.RelatesTo.GetReplyTo() - } - return "" + return content.RelatesTo.GetReplyTo() } const ReplyFormat = `
In reply to %s
%s
` diff --git a/vendor/maunium.net/go/mautrix/requests.go b/vendor/maunium.net/go/mautrix/requests.go index 64a1ab9..b5c681d 100644 --- a/vendor/maunium.net/go/mautrix/requests.go +++ b/vendor/maunium.net/go/mautrix/requests.go @@ -2,6 +2,7 @@ package mautrix import ( "encoding/json" + "strconv" "maunium.net/go/mautrix/event" "maunium.net/go/mautrix/id" @@ -172,10 +173,15 @@ type ReqAliasCreate struct { } type OneTimeKey struct { - Key id.Curve25519 `json:"key"` - IsSigned bool `json:"-"` - Signatures Signatures `json:"signatures,omitempty"` - Unsigned map[string]interface{} `json:"unsigned,omitempty"` + Key id.Curve25519 `json:"key"` + Fallback bool `json:"fallback,omitempty"` + Signatures Signatures `json:"signatures,omitempty"` + Unsigned map[string]any `json:"unsigned,omitempty"` + IsSigned bool `json:"-"` + + // Raw data in the one-time key. This must be used for signature verification to ensure unrecognized fields + // aren't thrown away (because that would invalidate the signature). + RawData json.RawMessage `json:"-"` } type serializableOTK OneTimeKey @@ -188,6 +194,7 @@ func (otk *OneTimeKey) UnmarshalJSON(data []byte) (err error) { otk.IsSigned = false } else { err = json.Unmarshal(data, (*serializableOTK)(otk)) + otk.RawData = data otk.IsSigned = true } return err @@ -319,7 +326,8 @@ type ReqBatchSend struct { PrevEventID id.EventID `json:"-"` BatchID id.BatchID `json:"-"` - BeeperNewMessages bool `json:"-"` + BeeperNewMessages bool `json:"-"` + BeeperMarkReadBy id.UserID `json:"-"` StateEventsAtStart []*event.Event `json:"state_events_at_start"` Events []*event.Event `json:"events"` @@ -334,3 +342,41 @@ type ReqSetReadMarkers struct { BeeperReadPrivateExtra interface{} `json:"com.beeper.read.private.extra"` BeeperFullyReadExtra interface{} `json:"com.beeper.fully_read.extra"` } + +// ReqHierarchy contains the parameters for https://spec.matrix.org/v1.4/client-server-api/#get_matrixclientv1roomsroomidhierarchy +// +// As it's a GET method, there is no JSON body, so this is only query parameters. +type ReqHierarchy struct { + // A pagination token from a previous Hierarchy call. + // If specified, max_depth and suggested_only cannot be changed from the first request. + From string + // Limit for the maximum number of rooms to include per response. + // The server will apply a default value if a limit isn't provided. + Limit int + // Limit for how far to go into the space. When reached, no further child rooms will be returned. + // The server will apply a default value if a max depth isn't provided. + MaxDepth *int + // Flag to indicate whether the server should only consider suggested rooms. + // Suggested rooms are annotated in their m.space.child event contents. + SuggestedOnly bool +} + +func (req *ReqHierarchy) Query() map[string]string { + query := map[string]string{} + if req == nil { + return query + } + if req.From != "" { + query["from"] = req.From + } + if req.Limit > 0 { + query["limit"] = strconv.Itoa(req.Limit) + } + if req.MaxDepth != nil { + query["max_depth"] = strconv.Itoa(*req.MaxDepth) + } + if req.SuggestedOnly { + query["suggested_only"] = "true" + } + return query +} diff --git a/vendor/maunium.net/go/mautrix/responses.go b/vendor/maunium.net/go/mautrix/responses.go index 7c2cc98..119c374 100644 --- a/vendor/maunium.net/go/mautrix/responses.go +++ b/vendor/maunium.net/go/mautrix/responses.go @@ -12,6 +12,7 @@ import ( "maunium.net/go/mautrix/event" "maunium.net/go/mautrix/id" "maunium.net/go/mautrix/util" + "maunium.net/go/mautrix/util/jsontime" ) // RespWhoami is the JSON response for https://spec.matrix.org/v1.2/client-server-api/#get_matrixclientv3accountwhoami @@ -514,3 +515,28 @@ func (vers *CapRoomVersions) IsAvailable(version string) bool { _, available := vers.Available[version] return available } + +// RespHierarchy is the JSON response for https://spec.matrix.org/v1.4/client-server-api/#get_matrixclientv1roomsroomidhierarchy +type RespHierarchy struct { + NextBatch string `json:"next_batch,omitempty"` + Rooms []ChildRoomsChunk `json:"rooms"` +} + +type ChildRoomsChunk struct { + AvatarURL id.ContentURI `json:"avatar_url,omitempty"` + CanonicalAlias id.RoomAlias `json:"canonical_alias,omitempty"` + ChildrenState []StrippedStateWithTime `json:"children_state"` + GuestCanJoin bool `json:"guest_can_join"` + JoinRule event.JoinRule `json:"join_rule,omitempty"` + Name string `json:"name,omitempty"` + NumJoinedMembers int `json:"num_joined_members"` + RoomID id.RoomID `json:"room_id"` + RoomType event.RoomType `json:"room_type"` + Topic string `json:"topic,omitempty"` + WorldReadble bool `json:"world_readable"` +} + +type StrippedStateWithTime struct { + event.StrippedState + Timestamp jsontime.UnixMilli `json:"origin_server_ts"` +} diff --git a/vendor/maunium.net/go/mautrix/util/dbutil/connlog.go b/vendor/maunium.net/go/mautrix/util/dbutil/connlog.go index 5a55af7..ffa370a 100644 --- a/vendor/maunium.net/go/mautrix/util/dbutil/connlog.go +++ b/vendor/maunium.net/go/mautrix/util/dbutil/connlog.go @@ -15,7 +15,7 @@ import ( // LoggingExecable is a wrapper for anything with database Exec methods (i.e. sql.Conn, sql.DB and sql.Tx) // that can preprocess queries (e.g. replacing $ with ? on SQLite) and log query durations. type LoggingExecable struct { - UnderlyingExecable Execable + UnderlyingExecable UnderlyingExecable db *Database } @@ -23,23 +23,30 @@ func (le *LoggingExecable) ExecContext(ctx context.Context, query string, args . start := time.Now() query = le.db.mutateQuery(query) res, err := le.UnderlyingExecable.ExecContext(ctx, query, args...) - le.db.Log.QueryTiming(ctx, "Exec", query, args, time.Since(start)) + le.db.Log.QueryTiming(ctx, "Exec", query, args, -1, time.Since(start)) return res, err } -func (le *LoggingExecable) QueryContext(ctx context.Context, query string, args ...interface{}) (*sql.Rows, error) { +func (le *LoggingExecable) QueryContext(ctx context.Context, query string, args ...interface{}) (Rows, error) { start := time.Now() query = le.db.mutateQuery(query) rows, err := le.UnderlyingExecable.QueryContext(ctx, query, args...) - le.db.Log.QueryTiming(ctx, "Query", query, args, time.Since(start)) - return rows, err + le.db.Log.QueryTiming(ctx, "Query", query, args, -1, time.Since(start)) + return &LoggingRows{ + ctx: ctx, + db: le.db, + query: query, + args: args, + rs: rows, + start: start, + }, err } func (le *LoggingExecable) QueryRowContext(ctx context.Context, query string, args ...interface{}) *sql.Row { start := time.Now() query = le.db.mutateQuery(query) row := le.UnderlyingExecable.QueryRowContext(ctx, query, args...) - le.db.Log.QueryTiming(ctx, "QueryRow", query, args, time.Since(start)) + le.db.Log.QueryTiming(ctx, "QueryRow", query, args, -1, time.Since(start)) return row } @@ -47,7 +54,7 @@ func (le *LoggingExecable) Exec(query string, args ...interface{}) (sql.Result, return le.ExecContext(context.Background(), query, args...) } -func (le *LoggingExecable) Query(query string, args ...interface{}) (*sql.Rows, error) { +func (le *LoggingExecable) Query(query string, args ...interface{}) (Rows, error) { return le.QueryContext(context.Background(), query, args...) } @@ -66,7 +73,7 @@ type loggingDB struct { func (ld *loggingDB) BeginTx(ctx context.Context, opts *sql.TxOptions) (*LoggingTxn, error) { start := time.Now() tx, err := ld.db.RawDB.BeginTx(ctx, opts) - ld.db.Log.QueryTiming(ctx, "Begin", "", nil, time.Since(start)) + ld.db.Log.QueryTiming(ctx, "Begin", "", nil, -1, time.Since(start)) if err != nil { return nil, err } @@ -90,13 +97,76 @@ type LoggingTxn struct { func (lt *LoggingTxn) Commit() error { start := time.Now() err := lt.UnderlyingTx.Commit() - lt.db.Log.QueryTiming(lt.ctx, "Commit", "", nil, time.Since(start)) + lt.db.Log.QueryTiming(lt.ctx, "Commit", "", nil, -1, time.Since(start)) return err } func (lt *LoggingTxn) Rollback() error { start := time.Now() err := lt.UnderlyingTx.Rollback() - lt.db.Log.QueryTiming(lt.ctx, "Rollback", "", nil, time.Since(start)) + lt.db.Log.QueryTiming(lt.ctx, "Rollback", "", nil, -1, time.Since(start)) return err } + +type LoggingRows struct { + ctx context.Context + db *Database + query string + args []interface{} + rs Rows + start time.Time + nrows int +} + +func (lrs *LoggingRows) stopTiming() { + if !lrs.start.IsZero() { + lrs.db.Log.QueryTiming(lrs.ctx, "EndRows", lrs.query, lrs.args, lrs.nrows, time.Since(lrs.start)) + lrs.start = time.Time{} + } +} + +func (lrs *LoggingRows) Close() error { + err := lrs.rs.Close() + lrs.stopTiming() + return err +} + +func (lrs *LoggingRows) ColumnTypes() ([]*sql.ColumnType, error) { + return lrs.rs.ColumnTypes() +} + +func (lrs *LoggingRows) Columns() ([]string, error) { + return lrs.rs.Columns() +} + +func (lrs *LoggingRows) Err() error { + return lrs.rs.Err() +} + +func (lrs *LoggingRows) Next() bool { + hasNext := lrs.rs.Next() + + if !hasNext { + lrs.stopTiming() + } else { + lrs.nrows++ + } + + return hasNext +} + +func (lrs *LoggingRows) NextResultSet() bool { + hasNext := lrs.rs.NextResultSet() + + if !hasNext { + lrs.stopTiming() + } else { + lrs.nrows++ + } + + return hasNext +} + +func (lrs *LoggingRows) Scan(dest ...any) error { + return lrs.rs.Scan(dest...) +} diff --git a/vendor/maunium.net/go/mautrix/util/dbutil/database.go b/vendor/maunium.net/go/mautrix/util/dbutil/database.go index c7d5446..3e105fd 100644 --- a/vendor/maunium.net/go/mautrix/util/dbutil/database.go +++ b/vendor/maunium.net/go/mautrix/util/dbutil/database.go @@ -40,13 +40,23 @@ func ParseDialect(engine string) (Dialect, error) { switch strings.ToLower(engine) { case "postgres", "postgresql": return Postgres, nil - case "sqlite3", "sqlite", "litestream": + case "sqlite3", "sqlite", "litestream", "sqlite3-fk-wal": return SQLite, nil default: return DialectUnknown, fmt.Errorf("unknown dialect '%s'", engine) } } +type Rows interface { + Close() error + ColumnTypes() ([]*sql.ColumnType, error) + Columns() ([]string, error) + Err() error + Next() bool + NextResultSet() bool + Scan(...any) error +} + type Scannable interface { Scan(...interface{}) error } @@ -54,19 +64,32 @@ type Scannable interface { // Expected implementations of Scannable var ( _ Scannable = (*sql.Row)(nil) - _ Scannable = (*sql.Rows)(nil) + _ Scannable = (Rows)(nil) ) -type ContextExecable interface { +type UnderlyingContextExecable interface { ExecContext(ctx context.Context, query string, args ...interface{}) (sql.Result, error) QueryContext(ctx context.Context, query string, args ...interface{}) (*sql.Rows, error) QueryRowContext(ctx context.Context, query string, args ...interface{}) *sql.Row } +type ContextExecable interface { + ExecContext(ctx context.Context, query string, args ...interface{}) (sql.Result, error) + QueryContext(ctx context.Context, query string, args ...interface{}) (Rows, error) + QueryRowContext(ctx context.Context, query string, args ...interface{}) *sql.Row +} + +type UnderlyingExecable interface { + UnderlyingContextExecable + Exec(query string, args ...interface{}) (sql.Result, error) + Query(query string, args ...interface{}) (*sql.Rows, error) + QueryRow(query string, args ...interface{}) *sql.Row +} + type Execable interface { ContextExecable Exec(query string, args ...interface{}) (sql.Result, error) - Query(query string, args ...interface{}) (*sql.Rows, error) + Query(query string, args ...interface{}) (Rows, error) QueryRow(query string, args ...interface{}) *sql.Row } @@ -78,11 +101,11 @@ type Transaction interface { // Expected implementations of Execable var ( - _ Execable = (*sql.Tx)(nil) - _ Execable = (*sql.DB)(nil) - _ Execable = (*LoggingExecable)(nil) - _ Transaction = (*LoggingTxn)(nil) - _ ContextExecable = (*sql.Conn)(nil) + _ UnderlyingExecable = (*sql.Tx)(nil) + _ UnderlyingExecable = (*sql.DB)(nil) + _ Execable = (*LoggingExecable)(nil) + _ Transaction = (*LoggingTxn)(nil) + _ UnderlyingContextExecable = (*sql.Conn)(nil) ) type Database struct { diff --git a/vendor/maunium.net/go/mautrix/util/dbutil/log.go b/vendor/maunium.net/go/mautrix/util/dbutil/log.go index cc81927..31000a9 100644 --- a/vendor/maunium.net/go/mautrix/util/dbutil/log.go +++ b/vendor/maunium.net/go/mautrix/util/dbutil/log.go @@ -11,10 +11,10 @@ import ( ) type DatabaseLogger interface { - QueryTiming(ctx context.Context, method, query string, args []interface{}, duration time.Duration) + QueryTiming(ctx context.Context, method, query string, args []interface{}, nrows int, duration time.Duration) WarnUnsupportedVersion(current, latest int) PrepareUpgrade(current, latest int) - DoUpgrade(from, to int, message string) + DoUpgrade(from, to int, message string, txn bool) // Deprecated: legacy warning method, return errors instead Warn(msg string, args ...interface{}) } @@ -25,10 +25,11 @@ var NoopLogger DatabaseLogger = &noopLogger{} func (n noopLogger) WarnUnsupportedVersion(_, _ int) {} func (n noopLogger) PrepareUpgrade(_, _ int) {} -func (n noopLogger) DoUpgrade(_, _ int, _ string) {} +func (n noopLogger) DoUpgrade(_, _ int, _ string, _ bool) {} func (n noopLogger) Warn(msg string, args ...interface{}) {} -func (n noopLogger) QueryTiming(_ context.Context, _, _ string, _ []interface{}, _ time.Duration) {} +func (n noopLogger) QueryTiming(_ context.Context, _, _ string, _ []interface{}, _ int, _ time.Duration) { +} type mauLogger struct { l maulogger.Logger @@ -46,11 +47,11 @@ func (m mauLogger) PrepareUpgrade(current, latest int) { m.l.Infofln("Database currently on v%d, latest: v%d", current, latest) } -func (m mauLogger) DoUpgrade(from, to int, message string) { +func (m mauLogger) DoUpgrade(from, to int, message string, _ bool) { m.l.Infofln("Upgrading database from v%d to v%d: %s", from, to, message) } -func (m mauLogger) QueryTiming(_ context.Context, method, query string, _ []interface{}, duration time.Duration) { +func (m mauLogger) QueryTiming(_ context.Context, method, query string, _ []interface{}, _ int, duration time.Duration) { if duration > 1*time.Second { m.l.Warnfln("%s(%s) took %.3f seconds", method, query, duration.Seconds()) } @@ -90,17 +91,18 @@ func (z zeroLogger) PrepareUpgrade(current, latest int) { } } -func (z zeroLogger) DoUpgrade(from, to int, message string) { +func (z zeroLogger) DoUpgrade(from, to int, message string, txn bool) { z.l.Info(). Int("from", from). Int("to", to). + Bool("single_txn", txn). Str("description", message). Msg("Upgrading database") } var whitespaceRegex = regexp.MustCompile(`\s+`) -func (z zeroLogger) QueryTiming(ctx context.Context, method, query string, args []interface{}, duration time.Duration) { +func (z zeroLogger) QueryTiming(ctx context.Context, method, query string, args []interface{}, nrows int, duration time.Duration) { log := zerolog.Ctx(ctx) if log.GetLevel() == zerolog.Disabled { log = z.l @@ -108,6 +110,10 @@ func (z zeroLogger) QueryTiming(ctx context.Context, method, query string, args if log.GetLevel() != zerolog.TraceLevel && duration < 1*time.Second { return } + if nrows > -1 { + rowLog := log.With().Int("rows", nrows).Logger() + log = &rowLog + } query = strings.TrimSpace(whitespaceRegex.ReplaceAllLiteralString(query, " ")) log.Trace(). Int64("duration_µs", duration.Microseconds()). diff --git a/vendor/maunium.net/go/mautrix/util/dbutil/samples/04-notxn.sql b/vendor/maunium.net/go/mautrix/util/dbutil/samples/04-notxn.sql new file mode 100644 index 0000000..7a61a72 --- /dev/null +++ b/vendor/maunium.net/go/mautrix/util/dbutil/samples/04-notxn.sql @@ -0,0 +1,4 @@ +-- v4: Sample outside transaction +-- transaction: off + +INSERT INTO foo VALUES ('meow', '{}'); diff --git a/vendor/maunium.net/go/mautrix/util/dbutil/samples/output/04-postgres.sql b/vendor/maunium.net/go/mautrix/util/dbutil/samples/output/04-postgres.sql new file mode 100644 index 0000000..53c9742 --- /dev/null +++ b/vendor/maunium.net/go/mautrix/util/dbutil/samples/output/04-postgres.sql @@ -0,0 +1 @@ +INSERT INTO foo VALUES ('meow', '{}'); diff --git a/vendor/maunium.net/go/mautrix/util/dbutil/samples/output/04-sqlite3.sql b/vendor/maunium.net/go/mautrix/util/dbutil/samples/output/04-sqlite3.sql new file mode 100644 index 0000000..53c9742 --- /dev/null +++ b/vendor/maunium.net/go/mautrix/util/dbutil/samples/output/04-sqlite3.sql @@ -0,0 +1 @@ +INSERT INTO foo VALUES ('meow', '{}'); diff --git a/vendor/maunium.net/go/mautrix/util/dbutil/upgrades.go b/vendor/maunium.net/go/mautrix/util/dbutil/upgrades.go index baf875c..bb96528 100644 --- a/vendor/maunium.net/go/mautrix/util/dbutil/upgrades.go +++ b/vendor/maunium.net/go/mautrix/util/dbutil/upgrades.go @@ -12,13 +12,14 @@ import ( "fmt" ) -type upgradeFunc func(Transaction, *Database) error +type upgradeFunc func(Execable, *Database) error type upgrade struct { message string fn upgradeFunc - upgradesTo int + upgradesTo int + transaction bool } var ErrUnsupportedDatabaseVersion = fmt.Errorf("unsupported database schema version") @@ -93,7 +94,7 @@ func (db *Database) checkDatabaseOwner() error { return nil } -func (db *Database) setVersion(tx Transaction, version int) error { +func (db *Database) setVersion(tx Execable, version int) error { _, err := tx.Exec(fmt.Sprintf("DELETE FROM %s", db.VersionTable)) if err != nil { return err @@ -129,25 +130,33 @@ func (db *Database) Upgrade() error { version++ continue } - db.Log.DoUpgrade(logVersion, upgradeItem.upgradesTo, upgradeItem.message) + db.Log.DoUpgrade(logVersion, upgradeItem.upgradesTo, upgradeItem.message, upgradeItem.transaction) var tx Transaction - tx, err = db.Begin() - if err != nil { - return err + var upgradeConn Execable + if upgradeItem.transaction { + tx, err = db.Begin() + if err != nil { + return err + } + upgradeConn = tx + } else { + upgradeConn = db } - err = upgradeItem.fn(tx, db) + err = upgradeItem.fn(upgradeConn, db) if err != nil { return err } version = upgradeItem.upgradesTo logVersion = version - err = db.setVersion(tx, version) + err = db.setVersion(upgradeConn, version) if err != nil { return err } - err = tx.Commit() - if err != nil { - return err + if tx != nil { + err = tx.Commit() + if err != nil { + return err + } } } return nil diff --git a/vendor/maunium.net/go/mautrix/util/dbutil/upgradetable.go b/vendor/maunium.net/go/mautrix/util/dbutil/upgradetable.go index 9dd7275..a972eda 100644 --- a/vendor/maunium.net/go/mautrix/util/dbutil/upgradetable.go +++ b/vendor/maunium.net/go/mautrix/util/dbutil/upgradetable.go @@ -29,14 +29,14 @@ func (ut *UpgradeTable) extend(toSize int) { } } -func (ut *UpgradeTable) Register(from, to int, message string, fn upgradeFunc) { +func (ut *UpgradeTable) Register(from, to int, message string, txn bool, fn upgradeFunc) { if from < 0 { from += to } if from < 0 { panic("invalid from value in UpgradeTable.Register() call") } - upg := upgrade{message: message, fn: fn, upgradesTo: to} + upg := upgrade{message: message, fn: fn, upgradesTo: to, transaction: txn} if len(*ut) == from { *ut = append(*ut, upg) return @@ -57,7 +57,14 @@ func (ut *UpgradeTable) Register(from, to int, message string, fn upgradeFunc) { // -- v1: Message var upgradeHeaderRegex = regexp.MustCompile(`^-- (?:v(\d+) -> )?v(\d+): (.+)$`) -func parseFileHeader(file []byte) (from, to int, message string, lines [][]byte, err error) { +// To disable wrapping the upgrade in a single transaction, put `--transaction: off` on the second line. +// +// -- v5: Upgrade without transaction +// -- transaction: off +// // do dangerous stuff +var transactionDisableRegex = regexp.MustCompile(`^-- transaction: (\w*)`) + +func parseFileHeader(file []byte) (from, to int, message string, txn bool, lines [][]byte, err error) { lines = bytes.Split(file, []byte("\n")) if len(lines) < 2 { err = errors.New("upgrade file too short") @@ -81,6 +88,15 @@ func parseFileHeader(file []byte) (from, to int, message string, lines [][]byte, from = -1 } message = string(match[3]) + txn = true + match = transactionDisableRegex.FindSubmatch(lines[0]) + if match != nil { + lines = lines[1:] + if string(match[1]) != "off" { + err = fmt.Errorf("invalid value %q for transaction flag", match[1]) + } + txn = false + } } return } @@ -163,7 +179,7 @@ func (db *Database) filterSQLUpgrade(lines [][]byte) (string, error) { } func sqlUpgradeFunc(fileName string, lines [][]byte) upgradeFunc { - return func(tx Transaction, db *Database) error { + return func(tx Execable, db *Database) error { if skip, err := db.parseDialectFilter(lines[0]); err == nil && skip == skipNextLine { return nil } else if upgradeSQL, err := db.filterSQLUpgrade(lines); err != nil { @@ -176,7 +192,7 @@ func sqlUpgradeFunc(fileName string, lines [][]byte) upgradeFunc { } func splitSQLUpgradeFunc(sqliteData, postgresData string) upgradeFunc { - return func(tx Transaction, database *Database) (err error) { + return func(tx Execable, database *Database) (err error) { switch database.Dialect { case SQLite: _, err = tx.Exec(sqliteData) @@ -189,7 +205,7 @@ func splitSQLUpgradeFunc(sqliteData, postgresData string) upgradeFunc { } } -func parseSplitSQLUpgrade(name string, fs fullFS, skipNames map[string]struct{}) (from, to int, message string, fn upgradeFunc) { +func parseSplitSQLUpgrade(name string, fs fullFS, skipNames map[string]struct{}) (from, to int, message string, txn bool, fn upgradeFunc) { postgresName := fmt.Sprintf("%s.postgres.sql", name) sqliteName := fmt.Sprintf("%s.sqlite.sql", name) skipNames[postgresName] = struct{}{} @@ -202,11 +218,11 @@ func parseSplitSQLUpgrade(name string, fs fullFS, skipNames map[string]struct{}) if err != nil { panic(err) } - from, to, message, _, err = parseFileHeader(postgresData) + from, to, message, txn, _, err = parseFileHeader(postgresData) if err != nil { panic(fmt.Errorf("failed to parse header in %s: %w", postgresName, err)) } - sqliteFrom, sqliteTo, sqliteMessage, _, err := parseFileHeader(sqliteData) + sqliteFrom, sqliteTo, sqliteMessage, sqliteTxn, _, err := parseFileHeader(sqliteData) if err != nil { panic(fmt.Errorf("failed to parse header in %s: %w", sqliteName, err)) } @@ -214,6 +230,8 @@ func parseSplitSQLUpgrade(name string, fs fullFS, skipNames map[string]struct{}) panic(fmt.Errorf("mismatching versions in postgres and sqlite versions of %s: %d/%d -> %d/%d", name, from, sqliteFrom, to, sqliteTo)) } else if message != sqliteMessage { panic(fmt.Errorf("mismatching message in postgres and sqlite versions of %s: %q != %q", name, message, sqliteMessage)) + } else if txn != sqliteTxn { + panic(fmt.Errorf("mismatching transaction flag in postgres and sqlite versions of %s: %t != %t", name, txn, sqliteTxn)) } fn = splitSQLUpgradeFunc(string(sqliteData), string(postgresData)) return @@ -242,14 +260,14 @@ func (ut *UpgradeTable) RegisterFSPath(fs fullFS, dir string) { } else if _, skip := skipNames[file.Name()]; skip { // also do nothing } else if splitName := splitFileNameRegex.FindStringSubmatch(file.Name()); splitName != nil { - from, to, message, fn := parseSplitSQLUpgrade(splitName[1], fs, skipNames) - ut.Register(from, to, message, fn) + from, to, message, txn, fn := parseSplitSQLUpgrade(splitName[1], fs, skipNames) + ut.Register(from, to, message, txn, fn) } else if data, err := fs.ReadFile(filepath.Join(dir, file.Name())); err != nil { panic(err) - } else if from, to, message, lines, err := parseFileHeader(data); err != nil { + } else if from, to, message, txn, lines, err := parseFileHeader(data); err != nil { panic(fmt.Errorf("failed to parse header in %s: %w", file.Name(), err)) } else { - ut.Register(from, to, message, sqlUpgradeFunc(file.Name(), lines)) + ut.Register(from, to, message, txn, sqlUpgradeFunc(file.Name(), lines)) } } } diff --git a/vendor/maunium.net/go/mautrix/util/jsontime/jsontime.go b/vendor/maunium.net/go/mautrix/util/jsontime/jsontime.go new file mode 100644 index 0000000..b328233 --- /dev/null +++ b/vendor/maunium.net/go/mautrix/util/jsontime/jsontime.go @@ -0,0 +1,86 @@ +// Copyright (c) 2022 Tulir Asokan +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package jsontime + +import ( + "encoding/json" + "time" +) + +func UM(time time.Time) UnixMilli { + return UnixMilli{Time: time} +} + +func UMInt(ts int64) UnixMilli { + return UM(time.UnixMilli(ts)) +} + +func UnixMilliNow() UnixMilli { + return UM(time.Now()) +} + +type UnixMilli struct { + time.Time +} + +func (um UnixMilli) MarshalJSON() ([]byte, error) { + if um.IsZero() { + return []byte{'0'}, nil + } + return json.Marshal(um.UnixMilli()) +} + +func (um *UnixMilli) UnmarshalJSON(data []byte) error { + var val int64 + err := json.Unmarshal(data, &val) + if err != nil { + return err + } + if val == 0 { + um.Time = time.Time{} + } else { + um.Time = time.UnixMilli(val) + } + return nil +} + +func U(time time.Time) Unix { + return Unix{Time: time} +} + +func UInt(ts int64) Unix { + return U(time.Unix(ts, 0)) +} + +func UnixNow() Unix { + return U(time.Now()) +} + +type Unix struct { + time.Time +} + +func (u Unix) MarshalJSON() ([]byte, error) { + if u.IsZero() { + return []byte{'0'}, nil + } + return json.Marshal(u.Unix()) +} + +func (u *Unix) UnmarshalJSON(data []byte) error { + var val int64 + err := json.Unmarshal(data, &val) + if err != nil { + return err + } + if val == 0 { + u.Time = time.Time{} + } else { + u.Time = time.Unix(val, 0) + } + return nil +} diff --git a/vendor/maunium.net/go/mautrix/version.go b/vendor/maunium.net/go/mautrix/version.go index dd930a6..18bf6bf 100644 --- a/vendor/maunium.net/go/mautrix/version.go +++ b/vendor/maunium.net/go/mautrix/version.go @@ -1,5 +1,5 @@ package mautrix -const Version = "v0.12.2" +const Version = "v0.12.3" var DefaultUserAgent = "mautrix-go/" + Version diff --git a/vendor/modules.txt b/vendor/modules.txt index bf91458..8ae4482 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -60,7 +60,7 @@ github.com/mattn/go-isatty # github.com/mattn/go-runewidth v0.0.12 ## explicit; go 1.9 github.com/mattn/go-runewidth -# github.com/mattn/go-sqlite3 v1.14.15 +# github.com/mattn/go-sqlite3 v1.14.16 ## explicit; go 1.16 github.com/mattn/go-sqlite3 # github.com/mikesmitty/edkey v0.0.0-20170222072505-3356ea4e686a @@ -101,7 +101,7 @@ github.com/tidwall/pretty # github.com/tidwall/sjson v1.2.5 ## explicit; go 1.14 github.com/tidwall/sjson -# github.com/yuin/goldmark v1.5.2 +# github.com/yuin/goldmark v1.5.3 ## explicit; go 1.18 github.com/yuin/goldmark github.com/yuin/goldmark/ast @@ -130,12 +130,12 @@ gitlab.com/etke.cc/go/trysmtp # gitlab.com/etke.cc/go/validator v1.0.3 ## explicit; go 1.18 gitlab.com/etke.cc/go/validator -# gitlab.com/etke.cc/linkpearl v0.0.0-20221115164843-97f1e49414d9 +# gitlab.com/etke.cc/linkpearl v0.0.0-20221116205701-65547c5608e6 ## explicit; go 1.18 gitlab.com/etke.cc/linkpearl gitlab.com/etke.cc/linkpearl/config gitlab.com/etke.cc/linkpearl/store -# golang.org/x/crypto v0.2.0 +# golang.org/x/crypto v0.3.0 ## explicit; go 1.17 golang.org/x/crypto/argon2 golang.org/x/crypto/blake2b @@ -180,7 +180,7 @@ gopkg.in/yaml.v3 # maunium.net/go/maulogger/v2 v2.3.2 ## explicit; go 1.11 maunium.net/go/maulogger/v2 -# maunium.net/go/mautrix v0.12.2 +# maunium.net/go/mautrix v0.12.3 ## explicit; go 1.18 maunium.net/go/mautrix maunium.net/go/mautrix/appservice @@ -202,3 +202,4 @@ maunium.net/go/mautrix/util maunium.net/go/mautrix/util/base58 maunium.net/go/mautrix/util/configupgrade maunium.net/go/mautrix/util/dbutil +maunium.net/go/mautrix/util/jsontime