7 Commits

Author SHA1 Message Date
Erik Johnston
29037e3ca1 Fix clippy lints 2022-06-06 09:59:17 +01:00
reivilibre
04ac0529e1 Merge pull request #86 from saces/saces/fixlogfile 2022-03-16 13:58:37 +00:00
saces
d6df1ac5a4 remove log_file leftovers
PR #74 (Log to stderr not to a file) did not remove all references to
log_file, the file was still created, but remained empty.

The synapse_auto_compressor failed if run in a read only environment.

Signed-off-by: saces <saces@c-base.org>
2022-03-13 00:13:33 +01:00
saces
32e43da3e8 Add Docker files (#83)
Signed-off-by: saces <saces@c-base.org>
2022-02-17 10:28:43 +00:00
Jan Alexander Steffens
4c7316311b Update dependencies, use tikv-jemallocator (#73)
Signed-off-by: Jan Alexander Steffens (heftig) <heftig@archlinux.org>
2021-10-26 08:54:56 +01:00
Jan Alexander Steffens
83e8dedfa9 lib: New argument -N to suppress verification (#26)
Signed-off-by: Jan Alexander Steffens (heftig) <heftig@archlinux.org>
2021-10-26 08:50:34 +01:00
Sean Quah
38d800a775 Configure @matrix-org/synapse-core to be the code owner for the repo (#76)
Signed-off-by: Sean Quah <seanq@element.io>
2021-10-25 14:32:07 +01:00
18 changed files with 268 additions and 225 deletions

3
.dockerignore Normal file
View File

@@ -0,0 +1,3 @@
.git
.github
/target

2
.github/CODEOWNERS vendored Normal file
View File

@@ -0,0 +1,2 @@
# Automatically request reviews from the synapse-core team when a pull request comes in.
* @matrix-org/synapse-core

3
.vscode/settings.json vendored Normal file
View File

@@ -0,0 +1,3 @@
{
"rust-analyzer.checkOnSave.command": "clippy"
}

307
Cargo.lock generated
View File

@@ -28,15 +28,15 @@ checksum = "61604a8f862e1d5c3229fdd78f8b02c68dcf73a4c4b05fd636d12240aaa242c1"
[[package]] [[package]]
name = "arc-swap" name = "arc-swap"
version = "1.3.2" version = "1.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b5ab7d9e73059c86c36473f459b52adbd99c3554a4fec492caef460806006f00" checksum = "e6df5aef5c5830360ce5218cecb8f018af3438af5686ae945094affc86fdec63"
[[package]] [[package]]
name = "async-trait" name = "async-trait"
version = "0.1.50" version = "0.1.51"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0b98e84bbb4cbcdd97da190ba0c58a1bb0de2c1fdf67d159e192ed766aeca722" checksum = "44318e776df68115a881de9a8fd1b9e53368d7a4a5ce4cc48517da3393233a5e"
dependencies = [ dependencies = [
"proc-macro2", "proc-macro2",
"quote", "quote",
@@ -68,9 +68,9 @@ checksum = "904dfeac50f3cdaba28fc6f57fdcddb75f49ed61346676a78c4ffe55877802fd"
[[package]] [[package]]
name = "bitflags" name = "bitflags"
version = "1.2.1" version = "1.3.2"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cf1de2fe8c75bc145a2f577add951f8134889b4795d47466a54a5c846d691693" checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a"
[[package]] [[package]]
name = "block-buffer" name = "block-buffer"
@@ -89,15 +89,15 @@ checksum = "14c189c53d098945499cdfa7ecc63567cf3886b3332b312a5b4585d8d3a6a610"
[[package]] [[package]]
name = "bytes" name = "bytes"
version = "1.0.1" version = "1.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b700ce4376041dcd0a327fd0097c41095743c4c8af8887265942faf1100bd040" checksum = "c4872d67bab6358e59559027aa3b9157c53d9358c51423c17554809a8858e0f8"
[[package]] [[package]]
name = "cc" name = "cc"
version = "1.0.68" version = "1.0.71"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4a72c244c1ff497a746a7e1fb3d14bd08420ecda70c8f25c7112f2781652d787" checksum = "79c2681d6594606957bbb8631c4b90a7fcaaa72cdb714743a437b156d6a7eedd"
[[package]] [[package]]
name = "cfg-if" name = "cfg-if"
@@ -139,22 +139,22 @@ dependencies = [
[[package]] [[package]]
name = "console" name = "console"
version = "0.14.1" version = "0.15.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3993e6445baa160675931ec041a5e03ca84b9c6e32a056150d3aa2bdda0a1f45" checksum = "a28b32d32ca44b70c3e4acd7db1babf555fa026e385fb95f18028f88848b3c31"
dependencies = [ dependencies = [
"encode_unicode", "encode_unicode",
"lazy_static",
"libc", "libc",
"once_cell",
"terminal_size", "terminal_size",
"winapi", "winapi",
] ]
[[package]] [[package]]
name = "cpufeatures" name = "cpufeatures"
version = "0.1.5" version = "0.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "66c99696f6c9dd7f35d486b9d04d7e6e202aa3e8c40d553f2fdf5e7e0c6a71ef" checksum = "95059428f66df56b63431fdb4e1947ed2190586af5c5a8a8b71122bdf5a7f469"
dependencies = [ dependencies = [
"libc", "libc",
] ]
@@ -171,9 +171,9 @@ dependencies = [
[[package]] [[package]]
name = "crossbeam-deque" name = "crossbeam-deque"
version = "0.8.0" version = "0.8.1"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "94af6efb46fef72616855b036a624cf27ba656ffc9be1b9a3c931cfc7749a9a9" checksum = "6455c0ca19f0d2fbf751b908d5c55c1f5cbc65e03c4225427254b46890bdde1e"
dependencies = [ dependencies = [
"cfg-if", "cfg-if",
"crossbeam-epoch", "crossbeam-epoch",
@@ -205,9 +205,9 @@ dependencies = [
[[package]] [[package]]
name = "crypto-mac" name = "crypto-mac"
version = "0.10.0" version = "0.11.1"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4857fd85a0c34b3c3297875b747c1e02e06b6a0ea32dd892d8192b9ce0813ea6" checksum = "b1d1a86f49236c215f271d40892d5fc950490551400b02ef360692c29815c714"
dependencies = [ dependencies = [
"generic-array", "generic-array",
"subtle", "subtle",
@@ -276,9 +276,9 @@ checksum = "2022715d62ab30faffd124d40b76f4134a550a87792276512b18d63272333394"
[[package]] [[package]]
name = "futures" name = "futures"
version = "0.3.15" version = "0.3.17"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0e7e43a803dae2fa37c1f6a8fe121e1f7bf9548b4dfc0522a42f34145dadfc27" checksum = "a12aa0eb539080d55c3f2d45a67c3b58b6b0773c1a3ca2dfec66d58c97fd66ca"
dependencies = [ dependencies = [
"futures-channel", "futures-channel",
"futures-core", "futures-core",
@@ -291,9 +291,9 @@ dependencies = [
[[package]] [[package]]
name = "futures-channel" name = "futures-channel"
version = "0.3.15" version = "0.3.17"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e682a68b29a882df0545c143dc3646daefe80ba479bcdede94d5a703de2871e2" checksum = "5da6ba8c3bb3c165d3c7319fc1cc8304facf1fb8db99c5de877183c08a273888"
dependencies = [ dependencies = [
"futures-core", "futures-core",
"futures-sink", "futures-sink",
@@ -301,15 +301,15 @@ dependencies = [
[[package]] [[package]]
name = "futures-core" name = "futures-core"
version = "0.3.15" version = "0.3.17"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0402f765d8a89a26043b889b26ce3c4679d268fa6bb22cd7c6aad98340e179d1" checksum = "88d1c26957f23603395cd326b0ffe64124b818f4449552f960d815cfba83a53d"
[[package]] [[package]]
name = "futures-executor" name = "futures-executor"
version = "0.3.15" version = "0.3.17"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "badaa6a909fac9e7236d0620a2f57f7664640c56575b71a7552fbd68deafab79" checksum = "45025be030969d763025784f7f355043dc6bc74093e4ecc5000ca4dc50d8745c"
dependencies = [ dependencies = [
"futures-core", "futures-core",
"futures-task", "futures-task",
@@ -318,15 +318,15 @@ dependencies = [
[[package]] [[package]]
name = "futures-io" name = "futures-io"
version = "0.3.15" version = "0.3.17"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "acc499defb3b348f8d8f3f66415835a9131856ff7714bf10dadfc4ec4bdb29a1" checksum = "522de2a0fe3e380f1bc577ba0474108faf3f6b18321dbf60b3b9c39a75073377"
[[package]] [[package]]
name = "futures-macro" name = "futures-macro"
version = "0.3.15" version = "0.3.17"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a4c40298486cdf52cc00cd6d6987892ba502c7656a16a4192a9992b1ccedd121" checksum = "18e4a4b95cea4b4ccbcf1c5675ca7c4ee4e9e75eb79944d07defde18068f79bb"
dependencies = [ dependencies = [
"autocfg", "autocfg",
"proc-macro-hack", "proc-macro-hack",
@@ -337,21 +337,21 @@ dependencies = [
[[package]] [[package]]
name = "futures-sink" name = "futures-sink"
version = "0.3.15" version = "0.3.17"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a57bead0ceff0d6dde8f465ecd96c9338121bb7717d3e7b108059531870c4282" checksum = "36ea153c13024fe480590b3e3d4cad89a0cfacecc24577b68f86c6ced9c2bc11"
[[package]] [[package]]
name = "futures-task" name = "futures-task"
version = "0.3.15" version = "0.3.17"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8a16bef9fc1a4dddb5bee51c989e3fbba26569cbb0e31f5b303c184e3dd33dae" checksum = "1d3d00f4eddb73e498a54394f228cd55853bdf059259e8e7bc6e69d408892e99"
[[package]] [[package]]
name = "futures-util" name = "futures-util"
version = "0.3.15" version = "0.3.17"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "feb5c238d27e2bf94ffdfd27b2c29e3df4a68c4193bb6427384259e2bf191967" checksum = "36568465210a3a6ee45e1f165136d68671471a501e632e9a98d96872222b5481"
dependencies = [ dependencies = [
"autocfg", "autocfg",
"futures-channel", "futures-channel",
@@ -400,9 +400,9 @@ dependencies = [
[[package]] [[package]]
name = "hmac" name = "hmac"
version = "0.10.1" version = "0.11.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c1441c6b1e930e2817404b5046f1f989899143a12bf92de603b69f4e0aee1e15" checksum = "2a2a2320eb7ec0ebe8da8f744d7812d9fc4cb4d09344ac01898dbcb6a20ae69b"
dependencies = [ dependencies = [
"crypto-mac", "crypto-mac",
"digest", "digest",
@@ -451,34 +451,13 @@ dependencies = [
[[package]] [[package]]
name = "instant" name = "instant"
version = "0.1.10" version = "0.1.12"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bee0328b1209d157ef001c94dd85b4f8f64139adb0eac2659f4b08382b2f474d" checksum = "7a5bbe824c507c5da5956355e86a746d82e0e1464f65d862cc5e71da70e94b2c"
dependencies = [ dependencies = [
"cfg-if", "cfg-if",
] ]
[[package]]
name = "jemalloc-sys"
version = "0.3.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0d3b9f3f5c9b31aa0f5ed3260385ac205db665baa41d49bb8338008ae94ede45"
dependencies = [
"cc",
"fs_extra",
"libc",
]
[[package]]
name = "jemallocator"
version = "0.3.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "43ae63fcfc45e99ab3d1b29a46782ad679e98436c3169d15a167a1108a724b69"
dependencies = [
"jemalloc-sys",
"libc",
]
[[package]] [[package]]
name = "lazy_static" name = "lazy_static"
version = "1.4.0" version = "1.4.0"
@@ -487,15 +466,15 @@ checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646"
[[package]] [[package]]
name = "libc" name = "libc"
version = "0.2.98" version = "0.2.104"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "320cfe77175da3a483efed4bc0adc1968ca050b098ce4f2f1c13a56626128790" checksum = "7b2f96d100e1cf1929e7719b7edb3b90ab5298072638fccd77be9ce942ecdfce"
[[package]] [[package]]
name = "lock_api" name = "lock_api"
version = "0.4.4" version = "0.4.5"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0382880606dff6d15c9476c416d18690b72742aa7b605bb6dd6ec9030fbf07eb" checksum = "712a4d093c9976e24e7dbca41db895dabcbac38eb5f4045393d17a95bdfb1109"
dependencies = [ dependencies = [
"scopeguard", "scopeguard",
] ]
@@ -518,12 +497,6 @@ dependencies = [
"log", "log",
] ]
[[package]]
name = "matches"
version = "0.1.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7ffc5c5338469d4d3ea17d269fa8ea3512ad247247c30bd2df69e68309ed0a08"
[[package]] [[package]]
name = "md-5" name = "md-5"
version = "0.9.1" version = "0.9.1"
@@ -537,9 +510,9 @@ dependencies = [
[[package]] [[package]]
name = "memchr" name = "memchr"
version = "2.4.0" version = "2.4.1"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b16bd47d9e329435e309c58469fe0791c2d0d1ba96ec0954152a5ae2b04387dc" checksum = "308cc39be01b73d0d18f82a0e7b2a3df85245f84af96fdddc5d202d27e47b86a"
[[package]] [[package]]
name = "memoffset" name = "memoffset"
@@ -552,9 +525,9 @@ dependencies = [
[[package]] [[package]]
name = "mio" name = "mio"
version = "0.7.13" version = "0.7.14"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8c2bdb6314ec10835cd3293dd268473a835c02b7b352e788be788b3c6ca6bb16" checksum = "8067b404fe97c70829f082dec8bcf4f71225d7eaea1d8645349cb76fa06205cc"
dependencies = [ dependencies = [
"libc", "libc",
"log", "log",
@@ -617,9 +590,9 @@ checksum = "624a8340c38c1b80fd549087862da4ba43e08858af025b236e509b6649fc13d5"
[[package]] [[package]]
name = "openssl" name = "openssl"
version = "0.10.35" version = "0.10.36"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "549430950c79ae24e6d02e0b7404534ecf311d94cc9f861e9e4020187d13d885" checksum = "8d9facdb76fec0b73c406f125d44d86fdad818d66fef0531eec9233ca425ff4a"
dependencies = [ dependencies = [
"bitflags", "bitflags",
"cfg-if", "cfg-if",
@@ -631,9 +604,9 @@ dependencies = [
[[package]] [[package]]
name = "openssl-sys" name = "openssl-sys"
version = "0.9.65" version = "0.9.67"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7a7907e3bfa08bb85105209cdfcb6c63d109f8f6c1ed6ca318fff5c1853fbc1d" checksum = "69df2d8dfc6ce3aaf44b40dec6f487d5a886516cf6879c49e98e0710f310a058"
dependencies = [ dependencies = [
"autocfg", "autocfg",
"cc", "cc",
@@ -644,9 +617,9 @@ dependencies = [
[[package]] [[package]]
name = "parking_lot" name = "parking_lot"
version = "0.11.1" version = "0.11.2"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6d7744ac029df22dca6284efe4e898991d28e3085c706c972bcd7da4a27a15eb" checksum = "7d17b78036a60663b797adeaee46f5c9dfebb86948d1255007a1d6be0271ff99"
dependencies = [ dependencies = [
"instant", "instant",
"lock_api", "lock_api",
@@ -655,9 +628,9 @@ dependencies = [
[[package]] [[package]]
name = "parking_lot_core" name = "parking_lot_core"
version = "0.8.3" version = "0.8.5"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fa7a782938e745763fe6907fc6ba86946d72f49fe7e21de074e08128a99fb018" checksum = "d76e8e1493bcac0d2766c42737f34458f1c8c50c0d23bcb24ea953affb273216"
dependencies = [ dependencies = [
"cfg-if", "cfg-if",
"instant", "instant",
@@ -694,11 +667,11 @@ checksum = "d4fd5641d01c8f18a23da7b6fe29298ff4b55afcccdf78973b24cf3175fee32e"
[[package]] [[package]]
name = "phf" name = "phf"
version = "0.8.0" version = "0.10.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3dfb61232e34fcb633f43d12c58f83c1df82962dcdfa565a4e866ffc17dafe12" checksum = "b9fc3db1018c4b59d7d582a739436478b6035138b6aecbce989fc91c3e98409f"
dependencies = [ dependencies = [
"phf_shared", "phf_shared 0.10.0",
] ]
[[package]] [[package]]
@@ -710,6 +683,15 @@ dependencies = [
"siphasher", "siphasher",
] ]
[[package]]
name = "phf_shared"
version = "0.10.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b6796ad771acdc0123d2a88dc428b5e38ef24456743ddb1744ed628f9815c096"
dependencies = [
"siphasher",
]
[[package]] [[package]]
name = "pin-project-lite" name = "pin-project-lite"
version = "0.2.7" version = "0.2.7"
@@ -724,15 +706,15 @@ checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184"
[[package]] [[package]]
name = "pkg-config" name = "pkg-config"
version = "0.3.19" version = "0.3.20"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3831453b3449ceb48b6d9c7ad7c96d5ea673e9b470a1dc578c2ce6521230884c" checksum = "7c9b1041b4387893b91ee6746cddfc28516aff326a3519fb2adf820932c5e6cb"
[[package]] [[package]]
name = "postgres" name = "postgres"
version = "0.19.1" version = "0.19.2"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c7871ee579860d8183f542e387b176a25f2656b9fb5211e045397f745a68d1c2" checksum = "eb76d6535496f633fa799bb872ffb4790e9cbdedda9d35564ca0252f930c0dd5"
dependencies = [ dependencies = [
"bytes", "bytes",
"fallible-iterator", "fallible-iterator",
@@ -757,9 +739,9 @@ dependencies = [
[[package]] [[package]]
name = "postgres-protocol" name = "postgres-protocol"
version = "0.6.1" version = "0.6.2"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ff3e0f70d32e20923cabf2df02913be7c1842d4c772db8065c00fcfdd1d1bff3" checksum = "b145e6a4ed52cb316a27787fc20fe8a25221cb476479f61e4e0327c15b98d91a"
dependencies = [ dependencies = [
"base64", "base64",
"byteorder", "byteorder",
@@ -775,9 +757,9 @@ dependencies = [
[[package]] [[package]]
name = "postgres-types" name = "postgres-types"
version = "0.2.1" version = "0.2.2"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "430f4131e1b7657b0cd9a2b0c3408d77c9a43a042d300b8c77f981dffcc43a2f" checksum = "04619f94ba0cc80999f4fc7073607cb825bc739a883cb6d20900fc5e009d6b0d"
dependencies = [ dependencies = [
"bytes", "bytes",
"fallible-iterator", "fallible-iterator",
@@ -786,9 +768,9 @@ dependencies = [
[[package]] [[package]]
name = "ppv-lite86" name = "ppv-lite86"
version = "0.2.10" version = "0.2.14"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ac74c624d6b2d21f425f752262f42188365d7b8ff1aff74c82e45136510a4857" checksum = "c3ca011bd0129ff4ae15cd04c4eef202cadf6c51c21e47aba319b4e0501db741"
[[package]] [[package]]
name = "precomputed-hash" name = "precomputed-hash"
@@ -810,18 +792,18 @@ checksum = "bc881b2c22681370c6a780e47af9840ef841837bc98118431d4e1868bd0c1086"
[[package]] [[package]]
name = "proc-macro2" name = "proc-macro2"
version = "1.0.27" version = "1.0.30"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f0d8caf72986c1a598726adc988bb5984792ef84f5ee5aa50209145ee8077038" checksum = "edc3358ebc67bc8b7fa0c007f945b0b18226f78437d61bec735a9eb96b61ee70"
dependencies = [ dependencies = [
"unicode-xid", "unicode-xid",
] ]
[[package]] [[package]]
name = "pyo3" name = "pyo3"
version = "0.14.1" version = "0.14.5"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "338f7f3701e11fd7f76508c91fbcaabc982564bcaf4d1ca7e1574ff2b4778aec" checksum = "35100f9347670a566a67aa623369293703322bb9db77d99d7df7313b575ae0c8"
dependencies = [ dependencies = [
"cfg-if", "cfg-if",
"indoc", "indoc",
@@ -835,9 +817,9 @@ dependencies = [
[[package]] [[package]]
name = "pyo3-build-config" name = "pyo3-build-config"
version = "0.14.1" version = "0.14.5"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "dcb2e98cc9ccc83d4f7115c8f925e0057e88c8d324b1bc4c2db4a7270c06ac9d" checksum = "d12961738cacbd7f91b7c43bc25cfeeaa2698ad07a04b3be0aa88b950865738f"
dependencies = [ dependencies = [
"once_cell", "once_cell",
] ]
@@ -855,9 +837,9 @@ dependencies = [
[[package]] [[package]]
name = "pyo3-macros" name = "pyo3-macros"
version = "0.14.1" version = "0.14.5"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cfb8671a42d0ecc4bec8cc107ae96d49292ca20cd1968e09b98af4aafd516adf" checksum = "fc0bc5215d704824dfddddc03f93cb572e1155c68b6761c37005e1c288808ea8"
dependencies = [ dependencies = [
"pyo3-macros-backend", "pyo3-macros-backend",
"quote", "quote",
@@ -866,9 +848,9 @@ dependencies = [
[[package]] [[package]]
name = "pyo3-macros-backend" name = "pyo3-macros-backend"
version = "0.14.1" version = "0.14.5"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9addf6dc422f05d4949cc0990195ee74fa43e3c3780cc9a1972fe9e7b68a9f48" checksum = "71623fc593224afaab918aa3afcaf86ed2f43d34f6afde7f3922608f253240df"
dependencies = [ dependencies = [
"proc-macro2", "proc-macro2",
"pyo3-build-config", "pyo3-build-config",
@@ -878,9 +860,9 @@ dependencies = [
[[package]] [[package]]
name = "quote" name = "quote"
version = "1.0.9" version = "1.0.10"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c3d0b9745dc2debf507c8422de05d7226cc1f0644216dfdfead988f9b1ab32a7" checksum = "38bc8cc6a5f2e3655e0899c1b848643b2562f853f114bfec7be120678e3ace05"
dependencies = [ dependencies = [
"proc-macro2", "proc-macro2",
] ]
@@ -952,9 +934,9 @@ dependencies = [
[[package]] [[package]]
name = "redox_syscall" name = "redox_syscall"
version = "0.2.9" version = "0.2.10"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5ab49abadf3f9e1c4bc499e8845e152ad87d2ad2d30371841171169e9d75feee" checksum = "8383f39639269cde97d255a32bdb68c047337295414940c68bdd30c2e13203ff"
dependencies = [ dependencies = [
"bitflags", "bitflags",
] ]
@@ -984,9 +966,9 @@ checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd"
[[package]] [[package]]
name = "serde" name = "serde"
version = "1.0.126" version = "1.0.130"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ec7505abeacaec74ae4778d9d9328fe5a5d04253220a85c4ee022239fc996d03" checksum = "f12d06de37cf59146fbdecab66aa99f9fe4f78722e3607577a5375d66bd0c913"
[[package]] [[package]]
name = "serial_test" name = "serial_test"
@@ -1012,9 +994,9 @@ dependencies = [
[[package]] [[package]]
name = "sha2" name = "sha2"
version = "0.9.5" version = "0.9.8"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b362ae5752fd2137731f9fa25fd4d9058af34666ca1966fb969119cc35719f12" checksum = "b69f9a4c9740d74c5baa3fd2e547f9525fa8088a8a958e0ca2409a514e33f5fa"
dependencies = [ dependencies = [
"block-buffer", "block-buffer",
"cfg-if", "cfg-if",
@@ -1025,27 +1007,27 @@ dependencies = [
[[package]] [[package]]
name = "siphasher" name = "siphasher"
version = "0.3.5" version = "0.3.7"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cbce6d4507c7e4a3962091436e56e95290cb71fa302d0d270e32130b75fbff27" checksum = "533494a8f9b724d33625ab53c6c4800f7cc445895924a8ef649222dcb76e938b"
[[package]] [[package]]
name = "slab" name = "slab"
version = "0.4.3" version = "0.4.5"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f173ac3d1a7e3b28003f40de0b5ce7fe2710f9b9dc3fc38664cebee46b3b6527" checksum = "9def91fd1e018fe007022791f865d0ccc9b3a0d5001e01aabb8b40e46000afb5"
[[package]] [[package]]
name = "smallvec" name = "smallvec"
version = "1.6.1" version = "1.7.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fe0f37c9e8f3c5a4a66ad655a93c74daac4ad00c441533bf5c6e7990bb42604e" checksum = "1ecab6c735a6bb4139c0caafd0cc3635748bbb3acf4550e8138122099251f309"
[[package]] [[package]]
name = "socket2" name = "socket2"
version = "0.4.0" version = "0.4.2"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9e3dfc207c526015c632472a77be09cf1b6e46866581aecae5cc38fb4235dea2" checksum = "5dc90fe6c7be1a323296982db1836d1ea9e47b6839496dde9a541bc496df3516"
dependencies = [ dependencies = [
"libc", "libc",
"winapi", "winapi",
@@ -1058,13 +1040,14 @@ source = "git+https://github.com/matrix-org/rust-matrix-state-map#211343e8dd8d14
[[package]] [[package]]
name = "string_cache" name = "string_cache"
version = "0.8.1" version = "0.8.2"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8ddb1139b5353f96e429e1a5e19fbaf663bddedaa06d1dbd49f82e352601209a" checksum = "923f0f39b6267d37d23ce71ae7235602134b250ace715dd2c90421998ddac0c6"
dependencies = [ dependencies = [
"lazy_static", "lazy_static",
"new_debug_unreachable", "new_debug_unreachable",
"phf_shared", "parking_lot",
"phf_shared 0.8.0",
"precomputed-hash", "precomputed-hash",
"serde", "serde",
] ]
@@ -1087,15 +1070,15 @@ checksum = "8ea5119cdb4c55b55d432abb513a0429384878c15dde60cc77b1c99de1a95a6a"
[[package]] [[package]]
name = "subtle" name = "subtle"
version = "2.4.0" version = "2.4.1"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1e81da0851ada1f3e9d4312c704aa4f8806f0f9d69faaf8df2f3464b4a9437c2" checksum = "6bdef32e8150c2a081110b42772ffe7d7c9032b606bc226c8260fd97e0976601"
[[package]] [[package]]
name = "syn" name = "syn"
version = "1.0.73" version = "1.0.80"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f71489ff30030d2ae598524f61326b902466f72a0fb1a8564c001cc63425bcc7" checksum = "d010a1623fbd906d51d650a9916aaefc05ffa0e4053ff7fe601167f3e715d194"
dependencies = [ dependencies = [
"proc-macro2", "proc-macro2",
"quote", "quote",
@@ -1109,7 +1092,6 @@ dependencies = [
"anyhow", "anyhow",
"clap", "clap",
"env_logger", "env_logger",
"jemallocator",
"log", "log",
"log-panics", "log-panics",
"openssl", "openssl",
@@ -1120,6 +1102,7 @@ dependencies = [
"rand", "rand",
"serial_test", "serial_test",
"synapse_compress_state", "synapse_compress_state",
"tikv-jemallocator",
] ]
[[package]] [[package]]
@@ -1129,7 +1112,6 @@ dependencies = [
"clap", "clap",
"env_logger", "env_logger",
"indicatif", "indicatif",
"jemallocator",
"log", "log",
"log-panics", "log-panics",
"openssl", "openssl",
@@ -1141,6 +1123,7 @@ dependencies = [
"rayon", "rayon",
"state-map", "state-map",
"string_cache", "string_cache",
"tikv-jemallocator",
] ]
[[package]] [[package]]
@@ -1172,10 +1155,31 @@ dependencies = [
] ]
[[package]] [[package]]
name = "tinyvec" name = "tikv-jemalloc-sys"
version = "1.2.0" version = "0.4.2+5.2.1-patched.2"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5b5220f05bb7de7f3f53c7c065e1199b3172696fe2db9f9c4d8ad9b4ee74c342" checksum = "5844e429d797c62945a566f8da4e24c7fe3fbd5d6617fd8bf7a0b7dc1ee0f22e"
dependencies = [
"cc",
"fs_extra",
"libc",
]
[[package]]
name = "tikv-jemallocator"
version = "0.4.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3c14a5a604eb8715bc5785018a37d00739b180bcf609916ddf4393d33d49ccdf"
dependencies = [
"libc",
"tikv-jemalloc-sys",
]
[[package]]
name = "tinyvec"
version = "1.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f83b2a3d4d9091d0abd7eba4dc2710b1718583bd4d8992e2190720ea38f391f7"
dependencies = [ dependencies = [
"tinyvec_macros", "tinyvec_macros",
] ]
@@ -1188,9 +1192,9 @@ checksum = "cda74da7e1a664f795bb1f8a87ec406fb89a02522cf6e50620d016add6dbbf5c"
[[package]] [[package]]
name = "tokio" name = "tokio"
version = "1.8.1" version = "1.12.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "98c8b05dc14c75ea83d63dd391100353789f5f24b8b3866542a5e85c8be8e985" checksum = "c2c2416fdedca8443ae44b4527de1ea633af61d8f7169ffa6e72c5b53d24efcc"
dependencies = [ dependencies = [
"autocfg", "autocfg",
"bytes", "bytes",
@@ -1203,11 +1207,11 @@ dependencies = [
[[package]] [[package]]
name = "tokio-openssl" name = "tokio-openssl"
version = "0.6.2" version = "0.6.3"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f24cddc8445a4dc8359cdd9e91c19d544fc95f672e32afe8945852b9381a09fe" checksum = "c08f9ffb7809f1b20c1b398d92acf4cc719874b3b2b2d9ea2f09b4a80350878a"
dependencies = [ dependencies = [
"futures", "futures-util",
"openssl", "openssl",
"openssl-sys", "openssl-sys",
"tokio", "tokio",
@@ -1215,9 +1219,9 @@ dependencies = [
[[package]] [[package]]
name = "tokio-postgres" name = "tokio-postgres"
version = "0.7.2" version = "0.7.4"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2d2b1383c7e4fb9a09e292c7c6afb7da54418d53b045f1c1fac7a911411a2b8b" checksum = "b5c07a6ceeeb8515d53998ac4487788a21884e79d5651490bc31a7289f20a7d7"
dependencies = [ dependencies = [
"async-trait", "async-trait",
"byteorder", "byteorder",
@@ -1238,9 +1242,9 @@ dependencies = [
[[package]] [[package]]
name = "tokio-util" name = "tokio-util"
version = "0.6.7" version = "0.6.8"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1caa0b0c8d94a049db56b5acf8cba99dc0623aab1b26d5b5f5e2d945846b3592" checksum = "08d3725d3efa29485e87311c5b699de63cde14b00ed4d256b8318aa30ca452cd"
dependencies = [ dependencies = [
"bytes", "bytes",
"futures-core", "futures-core",
@@ -1252,18 +1256,15 @@ dependencies = [
[[package]] [[package]]
name = "typenum" name = "typenum"
version = "1.13.0" version = "1.14.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "879f6906492a7cd215bfa4cf595b600146ccfac0c79bcbd1f3000162af5e8b06" checksum = "b63708a265f51345575b27fe43f9500ad611579e764c79edbc2037b1121959ec"
[[package]] [[package]]
name = "unicode-bidi" name = "unicode-bidi"
version = "0.3.5" version = "0.3.7"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "eeb8be209bb1c96b7c177c7420d26e04eccacb0eeae6b980e35fcb74678107e0" checksum = "1a01404663e3db436ed2746d9fefef640d868edae3cceb81c3b8d5732fda678f"
dependencies = [
"matches",
]
[[package]] [[package]]
name = "unicode-normalization" name = "unicode-normalization"
@@ -1276,9 +1277,9 @@ dependencies = [
[[package]] [[package]]
name = "unicode-width" name = "unicode-width"
version = "0.1.8" version = "0.1.9"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9337591893a19b88d8d87f2cec1e73fad5cdfd10e5a6f349f498ad6ea2ffb1e3" checksum = "3ed742d4ea2bd1176e236172c8429aaf54486e7ac098db29ffe6529e0ce50973"
[[package]] [[package]]
name = "unicode-xid" name = "unicode-xid"

View File

@@ -11,7 +11,6 @@ edition = "2018"
[dependencies] [dependencies]
clap = "2.33.0" clap = "2.33.0"
indicatif = "0.16.0" indicatif = "0.16.0"
jemallocator = "0.3.2"
openssl = "0.10.32" openssl = "0.10.32"
postgres = "0.19.0" postgres = "0.19.0"
postgres-openssl = "0.5.0" postgres-openssl = "0.5.0"
@@ -34,7 +33,11 @@ crate-type = ["cdylib", "rlib"]
version = "0.14.1" version = "0.14.1"
features = ["extension-module","abi3-py36"] features = ["extension-module","abi3-py36"]
[dependencies.tikv-jemallocator]
version = "0.4.1"
optional = true
[features] [features]
default = ["jemalloc"] default = ["jemalloc"]
jemalloc = [] jemalloc = ["tikv-jemallocator"]
no-progress-bars = [] no-progress-bars = []

22
Dockerfile Normal file
View File

@@ -0,0 +1,22 @@
FROM rust:alpine AS builder
RUN apk add python3 musl-dev pkgconfig openssl-dev make
ENV RUSTFLAGS="-C target-feature=-crt-static"
WORKDIR /opt/synapse-compressor/
COPY . .
RUN cargo build
WORKDIR /opt/synapse-compressor/synapse_auto_compressor/
RUN cargo build
FROM alpine
RUN apk add --no-cache libgcc
COPY --from=builder /opt/synapse-compressor/target/debug/synapse_compress_state /usr/local/bin/synapse_compress_state
COPY --from=builder /opt/synapse-compressor/target/debug/synapse_auto_compressor /usr/local/bin/synapse_auto_compressor

View File

@@ -197,7 +197,7 @@ $ docker-compose down
# Using the synapse_compress_state library # Using the synapse_compress_state library
If you want to use the compressor in another project, it is recomended that you If you want to use the compressor in another project, it is recomended that you
use jemalloc `https://github.com/gnzlbg/jemallocator`. use jemalloc `https://github.com/tikv/jemallocator`.
To prevent the progress bars from being shown, use the `no-progress-bars` feature. To prevent the progress bars from being shown, use the `no-progress-bars` feature.
(See `synapse_auto_compressor/Cargo.toml` for an example) (See `synapse_auto_compressor/Cargo.toml` for an example)

View File

@@ -46,10 +46,11 @@ fn run_succeeds_without_crashing() {
let transactions = true; let transactions = true;
let graphs = false; let graphs = false;
let commit_changes = false; let commit_changes = false;
let verify = true;
let config = Config::new( let config = Config::new(
db_url.clone(), db_url,
room_id.clone(), room_id,
output_file, output_file,
min_state_group, min_state_group,
groups_to_compress, groups_to_compress,
@@ -59,6 +60,7 @@ fn run_succeeds_without_crashing() {
transactions, transactions,
graphs, graphs,
commit_changes, commit_changes,
verify,
) )
.unwrap(); .unwrap();
@@ -94,6 +96,7 @@ fn changes_commited_if_no_min_saved_rows() {
let transactions = true; let transactions = true;
let graphs = false; let graphs = false;
let commit_changes = true; let commit_changes = true;
let verify = true;
let config = Config::new( let config = Config::new(
db_url, db_url,
@@ -107,6 +110,7 @@ fn changes_commited_if_no_min_saved_rows() {
transactions, transactions,
graphs, graphs,
commit_changes, commit_changes,
verify,
) )
.unwrap(); .unwrap();
@@ -160,6 +164,7 @@ fn changes_commited_if_min_saved_rows_exceeded() {
let transactions = true; let transactions = true;
let graphs = false; let graphs = false;
let commit_changes = true; let commit_changes = true;
let verify = true;
let config = Config::new( let config = Config::new(
db_url, db_url,
@@ -173,6 +178,7 @@ fn changes_commited_if_min_saved_rows_exceeded() {
transactions, transactions,
graphs, graphs,
commit_changes, commit_changes,
verify,
) )
.unwrap(); .unwrap();
@@ -227,6 +233,7 @@ fn changes_not_commited_if_fewer_than_min_saved_rows() {
let transactions = true; let transactions = true;
let graphs = false; let graphs = false;
let commit_changes = true; let commit_changes = true;
let verify = true;
let config = Config::new( let config = Config::new(
db_url, db_url,
@@ -240,6 +247,7 @@ fn changes_not_commited_if_fewer_than_min_saved_rows() {
transactions, transactions,
graphs, graphs,
commit_changes, commit_changes,
verify,
) )
.unwrap(); .unwrap();
@@ -280,6 +288,7 @@ fn run_panics_if_invalid_db_url() {
let transactions = true; let transactions = true;
let graphs = false; let graphs = false;
let commit_changes = true; let commit_changes = true;
let verify = true;
let config = Config::new( let config = Config::new(
db_url, db_url,
@@ -293,6 +302,7 @@ fn run_panics_if_invalid_db_url() {
transactions, transactions,
graphs, graphs,
commit_changes, commit_changes,
verify,
) )
.unwrap(); .unwrap();
@@ -336,6 +346,7 @@ fn run_only_affects_given_room_id() {
let transactions = true; let transactions = true;
let graphs = false; let graphs = false;
let commit_changes = true; let commit_changes = true;
let verify = true;
let config = Config::new( let config = Config::new(
db_url, db_url,
@@ -349,6 +360,7 @@ fn run_only_affects_given_room_id() {
transactions, transactions,
graphs, graphs,
commit_changes, commit_changes,
verify,
) )
.unwrap(); .unwrap();
@@ -406,6 +418,7 @@ fn run_respects_groups_to_compress() {
let transactions = true; let transactions = true;
let graphs = false; let graphs = false;
let commit_changes = true; let commit_changes = true;
let verify = true;
let config = Config::new( let config = Config::new(
db_url, db_url,
@@ -419,6 +432,7 @@ fn run_respects_groups_to_compress() {
transactions, transactions,
graphs, graphs,
commit_changes, commit_changes,
verify,
) )
.unwrap(); .unwrap();
@@ -492,6 +506,7 @@ fn run_is_idempotent_when_run_on_whole_room() {
let transactions = true; let transactions = true;
let graphs = false; let graphs = false;
let commit_changes = true; let commit_changes = true;
let verify = true;
let config1 = Config::new( let config1 = Config::new(
db_url.clone(), db_url.clone(),
@@ -505,21 +520,23 @@ fn run_is_idempotent_when_run_on_whole_room() {
transactions, transactions,
graphs, graphs,
commit_changes, commit_changes,
verify,
) )
.unwrap(); .unwrap();
let config2 = Config::new( let config2 = Config::new(
db_url.clone(), db_url,
room_id.clone(), room_id,
output_file2, output_file2,
min_state_group, min_state_group,
groups_to_compress, groups_to_compress,
min_saved_rows, min_saved_rows,
max_state_group, max_state_group,
level_sizes.clone(), level_sizes,
transactions, transactions,
graphs, graphs,
commit_changes, commit_changes,
verify,
) )
.unwrap(); .unwrap();

View File

@@ -56,7 +56,7 @@ fn continue_run_called_twice_same_as_run() {
let start = Some(6); let start = Some(6);
let chunk_size = 7; let chunk_size = 7;
let level_info = chunk_stats_1.new_level_info.clone(); let level_info = chunk_stats_1.new_level_info;
// Run the compressor with those settings // Run the compressor with those settings
let chunk_stats_2 = continue_run(start, chunk_size, &db_url, &room_id, &level_info).unwrap(); let chunk_stats_2 = continue_run(start, chunk_size, &db_url, &room_id, &level_info).unwrap();

View File

@@ -181,12 +181,11 @@ impl<'a> Compressor<'a> {
panic!("Can only call `create_new_tree` once"); panic!("Can only call `create_new_tree` once");
} }
let pb: ProgressBar; let pb: ProgressBar = if cfg!(feature = "no-progress-bars") {
if cfg!(feature = "no-progress-bars") { ProgressBar::hidden()
pb = ProgressBar::hidden();
} else { } else {
pb = ProgressBar::new(self.original_state_map.len() as u64); ProgressBar::new(self.original_state_map.len() as u64)
} };
pb.set_style( pb.set_style(
ProgressStyle::default_bar().template("[{elapsed_precise}] {bar} {pos}/{len} {msg}"), ProgressStyle::default_bar().template("[{elapsed_precise}] {bar} {pos}/{len} {msg}"),
); );

View File

@@ -93,14 +93,7 @@ fn create_new_tree_does_nothing_if_already_compressed() {
for i in 0i64..=13i64 { for i in 0i64..=13i64 {
// edge from map // edge from map
let pred_group = initial_edges.get(&i); let prev = initial_edges.get(&i).copied();
// Need Option<i64> not Option<&i64>
let prev;
match pred_group {
Some(i) => prev = Some(*i),
None => prev = None,
}
// insert that edge into the initial map // insert that edge into the initial map
initial.insert( initial.insert(

View File

@@ -54,7 +54,7 @@ fn get_head_returns_head() {
#[test] #[test]
fn has_space_returns_true_if_empty() { fn has_space_returns_true_if_empty() {
let l = Level::new(15); let l = Level::new(15);
assert_eq!(l.has_space(), true); assert!(l.has_space());
} }
#[test] #[test]
@@ -65,7 +65,7 @@ fn has_space_returns_true_if_part_full() {
l.update(1, true); l.update(1, true);
l.update(143, true); l.update(143, true);
l.update(15, true); l.update(15, true);
assert_eq!(l.has_space(), true); assert!(l.has_space());
} }
#[test] #[test]
@@ -76,5 +76,5 @@ fn has_space_returns_false_if_full() {
l.update(3, true); l.update(3, true);
l.update(4, true); l.update(4, true);
l.update(5, true); l.update(5, true);
assert_eq!(l.has_space(), false); assert!(!l.has_space());
} }

View File

@@ -142,14 +142,7 @@ fn stats_correct_if_no_changes() {
for i in 0i64..=13i64 { for i in 0i64..=13i64 {
// edge from map // edge from map
let pred_group = initial_edges.get(&i); let prev = initial_edges.get(&i).copied();
// Need Option<i64> not Option<&i64>
let prev;
match pred_group {
Some(i) => prev = Some(*i),
None => prev = None,
}
// insert that edge into the initial map // insert that edge into the initial map
initial.insert( initial.insert(

View File

@@ -372,12 +372,11 @@ fn get_initial_data_from_db(
// Copy the data from the database into a map // Copy the data from the database into a map
let mut state_group_map: BTreeMap<i64, StateGroupEntry> = BTreeMap::new(); let mut state_group_map: BTreeMap<i64, StateGroupEntry> = BTreeMap::new();
let pb: ProgressBar; let pb: ProgressBar = if cfg!(feature = "no-progress-bars") {
if cfg!(feature = "no-progress-bars") { ProgressBar::hidden()
pb = ProgressBar::hidden();
} else { } else {
pb = ProgressBar::new_spinner(); ProgressBar::new_spinner()
} };
pb.set_style( pb.set_style(
ProgressStyle::default_spinner().template("{spinner} [{elapsed}] {pos} rows retrieved"), ProgressStyle::default_spinner().template("{spinner} [{elapsed}] {pos} rows retrieved"),
); );
@@ -537,12 +536,11 @@ pub fn send_changes_to_db(
debug!("Writing changes..."); debug!("Writing changes...");
// setup the progress bar // setup the progress bar
let pb: ProgressBar; let pb: ProgressBar = if cfg!(feature = "no-progress-bars") {
if cfg!(feature = "no-progress-bars") { ProgressBar::hidden()
pb = ProgressBar::hidden();
} else { } else {
pb = ProgressBar::new(old_map.len() as u64); ProgressBar::new(old_map.len() as u64)
} };
pb.set_style( pb.set_style(
ProgressStyle::default_bar().template("[{elapsed_precise}] {bar} {pos}/{len} {msg}"), ProgressStyle::default_bar().template("[{elapsed_precise}] {bar} {pos}/{len} {msg}"),
); );

View File

@@ -109,6 +109,9 @@ pub struct Config {
// Whether or not to commit changes to the database automatically // Whether or not to commit changes to the database automatically
// N.B. currently assumes transactions is true (to be on the safe side) // N.B. currently assumes transactions is true (to be on the safe side)
commit_changes: bool, commit_changes: bool,
// Whether to verify the correctness of the compressed state groups by
// comparing them to the original groups
verify: bool,
} }
impl Config { impl Config {
@@ -223,6 +226,13 @@ impl Config {
.long_help(concat!("If this flag is set then the changes the compressor makes will", .long_help(concat!("If this flag is set then the changes the compressor makes will",
" be committed to the database. This should be safe to use while synapse is running", " be committed to the database. This should be safe to use while synapse is running",
" as it assumes by default that the transactions flag is set")), " as it assumes by default that the transactions flag is set")),
).arg(
Arg::with_name("no_verify")
.short("N")
.help("Do not double-check that the compression was performed correctly")
.long_help(concat!("If this flag is set then the verification of the compressed",
" state groups, which compares them to the original groups, is skipped. This",
" saves time at the cost of potentially generating mismatched state.")),
).get_matches(); ).get_matches();
let db_url = matches let db_url = matches
@@ -262,6 +272,8 @@ impl Config {
let commit_changes = matches.is_present("commit_changes"); let commit_changes = matches.is_present("commit_changes");
let verify = !matches.is_present("no_verify");
Config { Config {
db_url: String::from(db_url), db_url: String::from(db_url),
output_file, output_file,
@@ -274,6 +286,7 @@ impl Config {
transactions, transactions,
graphs, graphs,
commit_changes, commit_changes,
verify,
} }
} }
} }
@@ -372,7 +385,9 @@ pub fn run(mut config: Config) {
} }
} }
check_that_maps_match(&state_group_map, new_state_group_map); if config.verify {
check_that_maps_match(&state_group_map, new_state_group_map);
}
// If we are given an output file, we output the changes as SQL. If the // If we are given an output file, we output the changes as SQL. If the
// `transactions` argument is set we wrap each change to a state group in a // `transactions` argument is set we wrap each change to a state group in a
@@ -492,12 +507,11 @@ fn output_sql(
info!("Writing changes..."); info!("Writing changes...");
let pb: ProgressBar; let pb: ProgressBar = if cfg!(feature = "no-progress-bars") {
if cfg!(feature = "no-progress-bars") { ProgressBar::hidden()
pb = ProgressBar::hidden();
} else { } else {
pb = ProgressBar::new(old_map.len() as u64); ProgressBar::new(old_map.len() as u64)
} };
pb.set_style( pb.set_style(
ProgressStyle::default_bar().template("[{elapsed_precise}] {bar} {pos}/{len} {msg}"), ProgressStyle::default_bar().template("[{elapsed_precise}] {bar} {pos}/{len} {msg}"),
); );
@@ -607,12 +621,11 @@ fn check_that_maps_match(
) { ) {
info!("Checking that state maps match..."); info!("Checking that state maps match...");
let pb: ProgressBar; let pb: ProgressBar = if cfg!(feature = "no-progress-bars") {
if cfg!(feature = "no-progress-bars") { ProgressBar::hidden()
pb = ProgressBar::hidden();
} else { } else {
pb = ProgressBar::new(old_map.len() as u64); ProgressBar::new(old_map.len() as u64)
} };
pb.set_style( pb.set_style(
ProgressStyle::default_bar().template("[{elapsed_precise}] {bar} {pos}/{len} {msg}"), ProgressStyle::default_bar().template("[{elapsed_precise}] {bar} {pos}/{len} {msg}"),
); );
@@ -695,6 +708,7 @@ impl Config {
transactions: bool, transactions: bool,
graphs: bool, graphs: bool,
commit_changes: bool, commit_changes: bool,
verify: bool,
) -> Result<Config, String> { ) -> Result<Config, String> {
let mut output: Option<File> = None; let mut output: Option<File> = None;
if let Some(file) = output_file { if let Some(file) = output_file {
@@ -722,6 +736,7 @@ impl Config {
transactions, transactions,
graphs, graphs,
commit_changes, commit_changes,
verify,
}) })
} }
} }
@@ -746,6 +761,7 @@ impl Config {
transactions = true, transactions = true,
graphs = false, graphs = false,
commit_changes = false, commit_changes = false,
verify = true,
)] )]
fn run_compression( fn run_compression(
db_url: String, db_url: String,
@@ -759,6 +775,7 @@ fn run_compression(
transactions: bool, transactions: bool,
graphs: bool, graphs: bool,
commit_changes: bool, commit_changes: bool,
verify: bool,
) -> PyResult<()> { ) -> PyResult<()> {
let config = Config::new( let config = Config::new(
db_url, db_url,
@@ -772,6 +789,7 @@ fn run_compression(
transactions, transactions,
graphs, graphs,
commit_changes, commit_changes,
verify,
); );
match config { match config {
Err(e) => Err(PyErr::new::<exceptions::PyException, _>(e)), Err(e) => Err(PyErr::new::<exceptions::PyException, _>(e)),
@@ -955,7 +973,6 @@ mod lib_tests {
#[test] #[test]
fn check_that_maps_match_returns_if_both_empty() { fn check_that_maps_match_returns_if_both_empty() {
check_that_maps_match(&BTreeMap::new(), &BTreeMap::new()); check_that_maps_match(&BTreeMap::new(), &BTreeMap::new());
assert!(true);
} }
#[test] #[test]
@@ -988,7 +1005,6 @@ mod lib_tests {
} }
check_that_maps_match(&old_map, &BTreeMap::new()); check_that_maps_match(&old_map, &BTreeMap::new());
assert!(true);
} }
#[test] #[test]
@@ -1024,7 +1040,6 @@ mod lib_tests {
} }
check_that_maps_match(&BTreeMap::new(), &new_map); check_that_maps_match(&BTreeMap::new(), &new_map);
assert!(true);
} }
#[test] #[test]
@@ -1056,7 +1071,6 @@ mod lib_tests {
} }
check_that_maps_match(&BTreeMap::new(), &old_map.clone()); check_that_maps_match(&BTreeMap::new(), &old_map.clone());
assert!(true);
} }
#[test] #[test]
@@ -1119,7 +1133,6 @@ mod lib_tests {
} }
check_that_maps_match(&old_map, &new_map); check_that_maps_match(&old_map, &new_map);
assert!(true);
} }
#[test] #[test]
@@ -1201,7 +1214,6 @@ mod lib_tests {
); );
check_that_maps_match(&old_map, &new_map); check_that_maps_match(&old_map, &new_map);
assert!(true);
} }
//TODO: tests for correct SQL code produced by output_sql //TODO: tests for correct SQL code produced by output_sql
@@ -1224,6 +1236,7 @@ mod pyo3_tests {
let transactions = false; let transactions = false;
let graphs = false; let graphs = false;
let commit_changes = false; let commit_changes = false;
let verify = true;
let config = Config::new( let config = Config::new(
db_url.clone(), db_url.clone(),
@@ -1237,6 +1250,7 @@ mod pyo3_tests {
transactions, transactions,
graphs, graphs,
commit_changes, commit_changes,
verify,
) )
.unwrap(); .unwrap();
@@ -1270,6 +1284,7 @@ mod pyo3_tests {
let transactions = true; let transactions = true;
let graphs = true; let graphs = true;
let commit_changes = true; let commit_changes = true;
let verify = true;
let config = Config::new( let config = Config::new(
db_url.clone(), db_url.clone(),
@@ -1283,11 +1298,12 @@ mod pyo3_tests {
transactions, transactions,
graphs, graphs,
commit_changes, commit_changes,
verify,
) )
.unwrap(); .unwrap();
assert_eq!(config.db_url, db_url); assert_eq!(config.db_url, db_url);
assert!(!config.output_file.is_none()); assert!(config.output_file.is_some());
assert_eq!(config.room_id, room_id); assert_eq!(config.room_id, room_id);
assert_eq!(config.min_state_group, Some(3225)); assert_eq!(config.min_state_group, Some(3225));
assert_eq!(config.groups_to_compress, Some(970)); assert_eq!(config.groups_to_compress, Some(970));

View File

@@ -18,7 +18,7 @@
#[cfg(feature = "jemalloc")] #[cfg(feature = "jemalloc")]
#[global_allocator] #[global_allocator]
static GLOBAL: jemallocator::Jemalloc = jemallocator::Jemalloc; static GLOBAL: tikv_jemallocator::Jemalloc = tikv_jemallocator::Jemalloc;
use log::LevelFilter; use log::LevelFilter;
use std::env; use std::env;

View File

@@ -17,7 +17,7 @@ clap = "2.33.0"
openssl = "0.10.32" openssl = "0.10.32"
postgres = "0.19.0" postgres = "0.19.0"
postgres-openssl = "0.5.0" postgres-openssl = "0.5.0"
jemallocator = "0.3.2" tikv-jemallocator = "0.4.1"
rand = "0.8.0" rand = "0.8.0"
serial_test = "0.5.1" serial_test = "0.5.1"
synapse_compress_state = { path = "../", features = ["no-progress-bars"] } synapse_compress_state = { path = "../", features = ["no-progress-bars"] }

View File

@@ -17,11 +17,11 @@
//! continue from where it left off. //! continue from where it left off.
#[global_allocator] #[global_allocator]
static GLOBAL: jemallocator::Jemalloc = jemallocator::Jemalloc; static GLOBAL: tikv_jemallocator::Jemalloc = tikv_jemallocator::Jemalloc;
use clap::{crate_authors, crate_description, crate_name, crate_version, value_t, App, Arg}; use clap::{crate_authors, crate_description, crate_name, crate_version, value_t, App, Arg};
use log::LevelFilter; use log::LevelFilter;
use std::{env, fs::OpenOptions}; use std::env;
use synapse_auto_compressor::{manager, state_saving, LevelInfo}; use synapse_auto_compressor::{manager, state_saving, LevelInfo};
/// Execution starts here /// Execution starts here
@@ -29,12 +29,6 @@ fn main() {
// setup the logger for the synapse_auto_compressor // setup the logger for the synapse_auto_compressor
// The default can be overwritten with RUST_LOG // The default can be overwritten with RUST_LOG
// see the README for more information // see the README for more information
let log_file = OpenOptions::new()
.append(true)
.create(true)
.open("synapse_auto_compressor.log")
.unwrap_or_else(|e| panic!("Error occured while opening the log file: {}", e));
if env::var("RUST_LOG").is_err() { if env::var("RUST_LOG").is_err() {
let mut log_builder = env_logger::builder(); let mut log_builder = env_logger::builder();
// Ensure panics still come through // Ensure panics still come through
@@ -47,7 +41,6 @@ fn main() {
} else { } else {
// If RUST_LOG was set then use that // If RUST_LOG was set then use that
let mut log_builder = env_logger::Builder::from_env("RUST_LOG"); let mut log_builder = env_logger::Builder::from_env("RUST_LOG");
log_builder.target(env_logger::Target::Pipe(Box::new(log_file)));
// Ensure panics still come through // Ensure panics still come through
log_builder.filter_module("panic", LevelFilter::Error); log_builder.filter_module("panic", LevelFilter::Error);
log_builder.init(); log_builder.init();