Compare commits
13 Commits
azren/log_
...
erikj/fix_
| Author | SHA1 | Date | |
|---|---|---|---|
|
29037e3ca1 | ||
|
04ac0529e1 | ||
|
d6df1ac5a4 | ||
|
32e43da3e8 | ||
|
4c7316311b | ||
|
|
83e8dedfa9 | ||
|
38d800a775 | ||
|
5272acedd2 | ||
|
|
9d642cfb67 | ||
|
|
0111079153 | ||
|
bf57e81f54 | ||
|
2fc2db2848 | ||
|
|
4f823fba78 |
3
.dockerignore
Normal file
3
.dockerignore
Normal file
@@ -0,0 +1,3 @@
|
||||
.git
|
||||
.github
|
||||
/target
|
||||
2
.github/CODEOWNERS
vendored
Normal file
2
.github/CODEOWNERS
vendored
Normal file
@@ -0,0 +1,2 @@
|
||||
# Automatically request reviews from the synapse-core team when a pull request comes in.
|
||||
* @matrix-org/synapse-core
|
||||
3
.vscode/settings.json
vendored
Normal file
3
.vscode/settings.json
vendored
Normal file
@@ -0,0 +1,3 @@
|
||||
{
|
||||
"rust-analyzer.checkOnSave.command": "clippy"
|
||||
}
|
||||
363
Cargo.lock
generated
363
Cargo.lock
generated
@@ -28,15 +28,15 @@ checksum = "61604a8f862e1d5c3229fdd78f8b02c68dcf73a4c4b05fd636d12240aaa242c1"
|
||||
|
||||
[[package]]
|
||||
name = "arc-swap"
|
||||
version = "1.3.2"
|
||||
version = "1.4.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b5ab7d9e73059c86c36473f459b52adbd99c3554a4fec492caef460806006f00"
|
||||
checksum = "e6df5aef5c5830360ce5218cecb8f018af3438af5686ae945094affc86fdec63"
|
||||
|
||||
[[package]]
|
||||
name = "async-trait"
|
||||
version = "0.1.50"
|
||||
version = "0.1.51"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0b98e84bbb4cbcdd97da190ba0c58a1bb0de2c1fdf67d159e192ed766aeca722"
|
||||
checksum = "44318e776df68115a881de9a8fd1b9e53368d7a4a5ce4cc48517da3393233a5e"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
@@ -54,26 +54,6 @@ dependencies = [
|
||||
"winapi",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "auto_compressor"
|
||||
version = "0.1.0"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"clap",
|
||||
"env_logger 0.9.0 (git+https://github.com/TilCreator/env_logger?branch=fix_pipe)",
|
||||
"jemallocator",
|
||||
"log",
|
||||
"log-panics",
|
||||
"openssl",
|
||||
"postgres",
|
||||
"postgres-openssl",
|
||||
"pyo3",
|
||||
"pyo3-log",
|
||||
"rand",
|
||||
"serial_test",
|
||||
"synapse_compress_state",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "autocfg"
|
||||
version = "1.0.1"
|
||||
@@ -88,9 +68,9 @@ checksum = "904dfeac50f3cdaba28fc6f57fdcddb75f49ed61346676a78c4ffe55877802fd"
|
||||
|
||||
[[package]]
|
||||
name = "bitflags"
|
||||
version = "1.2.1"
|
||||
version = "1.3.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "cf1de2fe8c75bc145a2f577add951f8134889b4795d47466a54a5c846d691693"
|
||||
checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a"
|
||||
|
||||
[[package]]
|
||||
name = "block-buffer"
|
||||
@@ -109,15 +89,15 @@ checksum = "14c189c53d098945499cdfa7ecc63567cf3886b3332b312a5b4585d8d3a6a610"
|
||||
|
||||
[[package]]
|
||||
name = "bytes"
|
||||
version = "1.0.1"
|
||||
version = "1.1.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b700ce4376041dcd0a327fd0097c41095743c4c8af8887265942faf1100bd040"
|
||||
checksum = "c4872d67bab6358e59559027aa3b9157c53d9358c51423c17554809a8858e0f8"
|
||||
|
||||
[[package]]
|
||||
name = "cc"
|
||||
version = "1.0.68"
|
||||
version = "1.0.71"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "4a72c244c1ff497a746a7e1fb3d14bd08420ecda70c8f25c7112f2781652d787"
|
||||
checksum = "79c2681d6594606957bbb8631c4b90a7fcaaa72cdb714743a437b156d6a7eedd"
|
||||
|
||||
[[package]]
|
||||
name = "cfg-if"
|
||||
@@ -144,8 +124,7 @@ dependencies = [
|
||||
name = "compressor_integration_tests"
|
||||
version = "0.1.0"
|
||||
dependencies = [
|
||||
"auto_compressor",
|
||||
"env_logger 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"env_logger",
|
||||
"log",
|
||||
"openssl",
|
||||
"postgres",
|
||||
@@ -154,27 +133,28 @@ dependencies = [
|
||||
"serial_test",
|
||||
"state-map",
|
||||
"string_cache",
|
||||
"synapse_auto_compressor",
|
||||
"synapse_compress_state",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "console"
|
||||
version = "0.14.1"
|
||||
version = "0.15.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3993e6445baa160675931ec041a5e03ca84b9c6e32a056150d3aa2bdda0a1f45"
|
||||
checksum = "a28b32d32ca44b70c3e4acd7db1babf555fa026e385fb95f18028f88848b3c31"
|
||||
dependencies = [
|
||||
"encode_unicode",
|
||||
"lazy_static",
|
||||
"libc",
|
||||
"once_cell",
|
||||
"terminal_size",
|
||||
"winapi",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "cpufeatures"
|
||||
version = "0.1.5"
|
||||
version = "0.2.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "66c99696f6c9dd7f35d486b9d04d7e6e202aa3e8c40d553f2fdf5e7e0c6a71ef"
|
||||
checksum = "95059428f66df56b63431fdb4e1947ed2190586af5c5a8a8b71122bdf5a7f469"
|
||||
dependencies = [
|
||||
"libc",
|
||||
]
|
||||
@@ -191,9 +171,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "crossbeam-deque"
|
||||
version = "0.8.0"
|
||||
version = "0.8.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "94af6efb46fef72616855b036a624cf27ba656ffc9be1b9a3c931cfc7749a9a9"
|
||||
checksum = "6455c0ca19f0d2fbf751b908d5c55c1f5cbc65e03c4225427254b46890bdde1e"
|
||||
dependencies = [
|
||||
"cfg-if",
|
||||
"crossbeam-epoch",
|
||||
@@ -225,9 +205,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "crypto-mac"
|
||||
version = "0.10.0"
|
||||
version = "0.11.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "4857fd85a0c34b3c3297875b747c1e02e06b6a0ea32dd892d8192b9ce0813ea6"
|
||||
checksum = "b1d1a86f49236c215f271d40892d5fc950490551400b02ef360692c29815c714"
|
||||
dependencies = [
|
||||
"generic-array",
|
||||
"subtle",
|
||||
@@ -267,18 +247,6 @@ dependencies = [
|
||||
"termcolor",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "env_logger"
|
||||
version = "0.9.0"
|
||||
source = "git+https://github.com/TilCreator/env_logger?branch=fix_pipe#3d09e0d824d9301cf1c0d4a9f148f8cfeb216329"
|
||||
dependencies = [
|
||||
"atty",
|
||||
"humantime",
|
||||
"log",
|
||||
"regex",
|
||||
"termcolor",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "fallible-iterator"
|
||||
version = "0.2.0"
|
||||
@@ -308,9 +276,9 @@ checksum = "2022715d62ab30faffd124d40b76f4134a550a87792276512b18d63272333394"
|
||||
|
||||
[[package]]
|
||||
name = "futures"
|
||||
version = "0.3.15"
|
||||
version = "0.3.17"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0e7e43a803dae2fa37c1f6a8fe121e1f7bf9548b4dfc0522a42f34145dadfc27"
|
||||
checksum = "a12aa0eb539080d55c3f2d45a67c3b58b6b0773c1a3ca2dfec66d58c97fd66ca"
|
||||
dependencies = [
|
||||
"futures-channel",
|
||||
"futures-core",
|
||||
@@ -323,9 +291,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "futures-channel"
|
||||
version = "0.3.15"
|
||||
version = "0.3.17"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e682a68b29a882df0545c143dc3646daefe80ba479bcdede94d5a703de2871e2"
|
||||
checksum = "5da6ba8c3bb3c165d3c7319fc1cc8304facf1fb8db99c5de877183c08a273888"
|
||||
dependencies = [
|
||||
"futures-core",
|
||||
"futures-sink",
|
||||
@@ -333,15 +301,15 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "futures-core"
|
||||
version = "0.3.15"
|
||||
version = "0.3.17"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0402f765d8a89a26043b889b26ce3c4679d268fa6bb22cd7c6aad98340e179d1"
|
||||
checksum = "88d1c26957f23603395cd326b0ffe64124b818f4449552f960d815cfba83a53d"
|
||||
|
||||
[[package]]
|
||||
name = "futures-executor"
|
||||
version = "0.3.15"
|
||||
version = "0.3.17"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "badaa6a909fac9e7236d0620a2f57f7664640c56575b71a7552fbd68deafab79"
|
||||
checksum = "45025be030969d763025784f7f355043dc6bc74093e4ecc5000ca4dc50d8745c"
|
||||
dependencies = [
|
||||
"futures-core",
|
||||
"futures-task",
|
||||
@@ -350,15 +318,15 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "futures-io"
|
||||
version = "0.3.15"
|
||||
version = "0.3.17"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "acc499defb3b348f8d8f3f66415835a9131856ff7714bf10dadfc4ec4bdb29a1"
|
||||
checksum = "522de2a0fe3e380f1bc577ba0474108faf3f6b18321dbf60b3b9c39a75073377"
|
||||
|
||||
[[package]]
|
||||
name = "futures-macro"
|
||||
version = "0.3.15"
|
||||
version = "0.3.17"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a4c40298486cdf52cc00cd6d6987892ba502c7656a16a4192a9992b1ccedd121"
|
||||
checksum = "18e4a4b95cea4b4ccbcf1c5675ca7c4ee4e9e75eb79944d07defde18068f79bb"
|
||||
dependencies = [
|
||||
"autocfg",
|
||||
"proc-macro-hack",
|
||||
@@ -369,21 +337,21 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "futures-sink"
|
||||
version = "0.3.15"
|
||||
version = "0.3.17"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a57bead0ceff0d6dde8f465ecd96c9338121bb7717d3e7b108059531870c4282"
|
||||
checksum = "36ea153c13024fe480590b3e3d4cad89a0cfacecc24577b68f86c6ced9c2bc11"
|
||||
|
||||
[[package]]
|
||||
name = "futures-task"
|
||||
version = "0.3.15"
|
||||
version = "0.3.17"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "8a16bef9fc1a4dddb5bee51c989e3fbba26569cbb0e31f5b303c184e3dd33dae"
|
||||
checksum = "1d3d00f4eddb73e498a54394f228cd55853bdf059259e8e7bc6e69d408892e99"
|
||||
|
||||
[[package]]
|
||||
name = "futures-util"
|
||||
version = "0.3.15"
|
||||
version = "0.3.17"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "feb5c238d27e2bf94ffdfd27b2c29e3df4a68c4193bb6427384259e2bf191967"
|
||||
checksum = "36568465210a3a6ee45e1f165136d68671471a501e632e9a98d96872222b5481"
|
||||
dependencies = [
|
||||
"autocfg",
|
||||
"futures-channel",
|
||||
@@ -432,9 +400,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "hmac"
|
||||
version = "0.10.1"
|
||||
version = "0.11.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c1441c6b1e930e2817404b5046f1f989899143a12bf92de603b69f4e0aee1e15"
|
||||
checksum = "2a2a2320eb7ec0ebe8da8f744d7812d9fc4cb4d09344ac01898dbcb6a20ae69b"
|
||||
dependencies = [
|
||||
"crypto-mac",
|
||||
"digest",
|
||||
@@ -483,34 +451,13 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "instant"
|
||||
version = "0.1.10"
|
||||
version = "0.1.12"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "bee0328b1209d157ef001c94dd85b4f8f64139adb0eac2659f4b08382b2f474d"
|
||||
checksum = "7a5bbe824c507c5da5956355e86a746d82e0e1464f65d862cc5e71da70e94b2c"
|
||||
dependencies = [
|
||||
"cfg-if",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "jemalloc-sys"
|
||||
version = "0.3.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0d3b9f3f5c9b31aa0f5ed3260385ac205db665baa41d49bb8338008ae94ede45"
|
||||
dependencies = [
|
||||
"cc",
|
||||
"fs_extra",
|
||||
"libc",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "jemallocator"
|
||||
version = "0.3.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "43ae63fcfc45e99ab3d1b29a46782ad679e98436c3169d15a167a1108a724b69"
|
||||
dependencies = [
|
||||
"jemalloc-sys",
|
||||
"libc",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "lazy_static"
|
||||
version = "1.4.0"
|
||||
@@ -519,15 +466,15 @@ checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646"
|
||||
|
||||
[[package]]
|
||||
name = "libc"
|
||||
version = "0.2.98"
|
||||
version = "0.2.104"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "320cfe77175da3a483efed4bc0adc1968ca050b098ce4f2f1c13a56626128790"
|
||||
checksum = "7b2f96d100e1cf1929e7719b7edb3b90ab5298072638fccd77be9ce942ecdfce"
|
||||
|
||||
[[package]]
|
||||
name = "lock_api"
|
||||
version = "0.4.4"
|
||||
version = "0.4.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0382880606dff6d15c9476c416d18690b72742aa7b605bb6dd6ec9030fbf07eb"
|
||||
checksum = "712a4d093c9976e24e7dbca41db895dabcbac38eb5f4045393d17a95bdfb1109"
|
||||
dependencies = [
|
||||
"scopeguard",
|
||||
]
|
||||
@@ -550,12 +497,6 @@ dependencies = [
|
||||
"log",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "matches"
|
||||
version = "0.1.8"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7ffc5c5338469d4d3ea17d269fa8ea3512ad247247c30bd2df69e68309ed0a08"
|
||||
|
||||
[[package]]
|
||||
name = "md-5"
|
||||
version = "0.9.1"
|
||||
@@ -569,9 +510,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "memchr"
|
||||
version = "2.4.0"
|
||||
version = "2.4.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b16bd47d9e329435e309c58469fe0791c2d0d1ba96ec0954152a5ae2b04387dc"
|
||||
checksum = "308cc39be01b73d0d18f82a0e7b2a3df85245f84af96fdddc5d202d27e47b86a"
|
||||
|
||||
[[package]]
|
||||
name = "memoffset"
|
||||
@@ -584,9 +525,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "mio"
|
||||
version = "0.7.13"
|
||||
version = "0.7.14"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "8c2bdb6314ec10835cd3293dd268473a835c02b7b352e788be788b3c6ca6bb16"
|
||||
checksum = "8067b404fe97c70829f082dec8bcf4f71225d7eaea1d8645349cb76fa06205cc"
|
||||
dependencies = [
|
||||
"libc",
|
||||
"log",
|
||||
@@ -649,9 +590,9 @@ checksum = "624a8340c38c1b80fd549087862da4ba43e08858af025b236e509b6649fc13d5"
|
||||
|
||||
[[package]]
|
||||
name = "openssl"
|
||||
version = "0.10.35"
|
||||
version = "0.10.36"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "549430950c79ae24e6d02e0b7404534ecf311d94cc9f861e9e4020187d13d885"
|
||||
checksum = "8d9facdb76fec0b73c406f125d44d86fdad818d66fef0531eec9233ca425ff4a"
|
||||
dependencies = [
|
||||
"bitflags",
|
||||
"cfg-if",
|
||||
@@ -663,9 +604,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "openssl-sys"
|
||||
version = "0.9.65"
|
||||
version = "0.9.67"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7a7907e3bfa08bb85105209cdfcb6c63d109f8f6c1ed6ca318fff5c1853fbc1d"
|
||||
checksum = "69df2d8dfc6ce3aaf44b40dec6f487d5a886516cf6879c49e98e0710f310a058"
|
||||
dependencies = [
|
||||
"autocfg",
|
||||
"cc",
|
||||
@@ -676,9 +617,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "parking_lot"
|
||||
version = "0.11.1"
|
||||
version = "0.11.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6d7744ac029df22dca6284efe4e898991d28e3085c706c972bcd7da4a27a15eb"
|
||||
checksum = "7d17b78036a60663b797adeaee46f5c9dfebb86948d1255007a1d6be0271ff99"
|
||||
dependencies = [
|
||||
"instant",
|
||||
"lock_api",
|
||||
@@ -687,9 +628,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "parking_lot_core"
|
||||
version = "0.8.3"
|
||||
version = "0.8.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "fa7a782938e745763fe6907fc6ba86946d72f49fe7e21de074e08128a99fb018"
|
||||
checksum = "d76e8e1493bcac0d2766c42737f34458f1c8c50c0d23bcb24ea953affb273216"
|
||||
dependencies = [
|
||||
"cfg-if",
|
||||
"instant",
|
||||
@@ -726,11 +667,11 @@ checksum = "d4fd5641d01c8f18a23da7b6fe29298ff4b55afcccdf78973b24cf3175fee32e"
|
||||
|
||||
[[package]]
|
||||
name = "phf"
|
||||
version = "0.8.0"
|
||||
version = "0.10.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3dfb61232e34fcb633f43d12c58f83c1df82962dcdfa565a4e866ffc17dafe12"
|
||||
checksum = "b9fc3db1018c4b59d7d582a739436478b6035138b6aecbce989fc91c3e98409f"
|
||||
dependencies = [
|
||||
"phf_shared",
|
||||
"phf_shared 0.10.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -742,6 +683,15 @@ dependencies = [
|
||||
"siphasher",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "phf_shared"
|
||||
version = "0.10.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b6796ad771acdc0123d2a88dc428b5e38ef24456743ddb1744ed628f9815c096"
|
||||
dependencies = [
|
||||
"siphasher",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "pin-project-lite"
|
||||
version = "0.2.7"
|
||||
@@ -756,15 +706,15 @@ checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184"
|
||||
|
||||
[[package]]
|
||||
name = "pkg-config"
|
||||
version = "0.3.19"
|
||||
version = "0.3.20"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3831453b3449ceb48b6d9c7ad7c96d5ea673e9b470a1dc578c2ce6521230884c"
|
||||
checksum = "7c9b1041b4387893b91ee6746cddfc28516aff326a3519fb2adf820932c5e6cb"
|
||||
|
||||
[[package]]
|
||||
name = "postgres"
|
||||
version = "0.19.1"
|
||||
version = "0.19.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c7871ee579860d8183f542e387b176a25f2656b9fb5211e045397f745a68d1c2"
|
||||
checksum = "eb76d6535496f633fa799bb872ffb4790e9cbdedda9d35564ca0252f930c0dd5"
|
||||
dependencies = [
|
||||
"bytes",
|
||||
"fallible-iterator",
|
||||
@@ -789,9 +739,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "postgres-protocol"
|
||||
version = "0.6.1"
|
||||
version = "0.6.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ff3e0f70d32e20923cabf2df02913be7c1842d4c772db8065c00fcfdd1d1bff3"
|
||||
checksum = "b145e6a4ed52cb316a27787fc20fe8a25221cb476479f61e4e0327c15b98d91a"
|
||||
dependencies = [
|
||||
"base64",
|
||||
"byteorder",
|
||||
@@ -807,9 +757,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "postgres-types"
|
||||
version = "0.2.1"
|
||||
version = "0.2.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "430f4131e1b7657b0cd9a2b0c3408d77c9a43a042d300b8c77f981dffcc43a2f"
|
||||
checksum = "04619f94ba0cc80999f4fc7073607cb825bc739a883cb6d20900fc5e009d6b0d"
|
||||
dependencies = [
|
||||
"bytes",
|
||||
"fallible-iterator",
|
||||
@@ -818,9 +768,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "ppv-lite86"
|
||||
version = "0.2.10"
|
||||
version = "0.2.14"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ac74c624d6b2d21f425f752262f42188365d7b8ff1aff74c82e45136510a4857"
|
||||
checksum = "c3ca011bd0129ff4ae15cd04c4eef202cadf6c51c21e47aba319b4e0501db741"
|
||||
|
||||
[[package]]
|
||||
name = "precomputed-hash"
|
||||
@@ -842,18 +792,18 @@ checksum = "bc881b2c22681370c6a780e47af9840ef841837bc98118431d4e1868bd0c1086"
|
||||
|
||||
[[package]]
|
||||
name = "proc-macro2"
|
||||
version = "1.0.27"
|
||||
version = "1.0.30"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f0d8caf72986c1a598726adc988bb5984792ef84f5ee5aa50209145ee8077038"
|
||||
checksum = "edc3358ebc67bc8b7fa0c007f945b0b18226f78437d61bec735a9eb96b61ee70"
|
||||
dependencies = [
|
||||
"unicode-xid",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "pyo3"
|
||||
version = "0.14.1"
|
||||
version = "0.14.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "338f7f3701e11fd7f76508c91fbcaabc982564bcaf4d1ca7e1574ff2b4778aec"
|
||||
checksum = "35100f9347670a566a67aa623369293703322bb9db77d99d7df7313b575ae0c8"
|
||||
dependencies = [
|
||||
"cfg-if",
|
||||
"indoc",
|
||||
@@ -867,9 +817,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "pyo3-build-config"
|
||||
version = "0.14.1"
|
||||
version = "0.14.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "dcb2e98cc9ccc83d4f7115c8f925e0057e88c8d324b1bc4c2db4a7270c06ac9d"
|
||||
checksum = "d12961738cacbd7f91b7c43bc25cfeeaa2698ad07a04b3be0aa88b950865738f"
|
||||
dependencies = [
|
||||
"once_cell",
|
||||
]
|
||||
@@ -887,9 +837,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "pyo3-macros"
|
||||
version = "0.14.1"
|
||||
version = "0.14.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "cfb8671a42d0ecc4bec8cc107ae96d49292ca20cd1968e09b98af4aafd516adf"
|
||||
checksum = "fc0bc5215d704824dfddddc03f93cb572e1155c68b6761c37005e1c288808ea8"
|
||||
dependencies = [
|
||||
"pyo3-macros-backend",
|
||||
"quote",
|
||||
@@ -898,9 +848,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "pyo3-macros-backend"
|
||||
version = "0.14.1"
|
||||
version = "0.14.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9addf6dc422f05d4949cc0990195ee74fa43e3c3780cc9a1972fe9e7b68a9f48"
|
||||
checksum = "71623fc593224afaab918aa3afcaf86ed2f43d34f6afde7f3922608f253240df"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"pyo3-build-config",
|
||||
@@ -910,9 +860,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "quote"
|
||||
version = "1.0.9"
|
||||
version = "1.0.10"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c3d0b9745dc2debf507c8422de05d7226cc1f0644216dfdfead988f9b1ab32a7"
|
||||
checksum = "38bc8cc6a5f2e3655e0899c1b848643b2562f853f114bfec7be120678e3ace05"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
]
|
||||
@@ -984,9 +934,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "redox_syscall"
|
||||
version = "0.2.9"
|
||||
version = "0.2.10"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5ab49abadf3f9e1c4bc499e8845e152ad87d2ad2d30371841171169e9d75feee"
|
||||
checksum = "8383f39639269cde97d255a32bdb68c047337295414940c68bdd30c2e13203ff"
|
||||
dependencies = [
|
||||
"bitflags",
|
||||
]
|
||||
@@ -1016,9 +966,9 @@ checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd"
|
||||
|
||||
[[package]]
|
||||
name = "serde"
|
||||
version = "1.0.126"
|
||||
version = "1.0.130"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ec7505abeacaec74ae4778d9d9328fe5a5d04253220a85c4ee022239fc996d03"
|
||||
checksum = "f12d06de37cf59146fbdecab66aa99f9fe4f78722e3607577a5375d66bd0c913"
|
||||
|
||||
[[package]]
|
||||
name = "serial_test"
|
||||
@@ -1044,9 +994,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "sha2"
|
||||
version = "0.9.5"
|
||||
version = "0.9.8"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b362ae5752fd2137731f9fa25fd4d9058af34666ca1966fb969119cc35719f12"
|
||||
checksum = "b69f9a4c9740d74c5baa3fd2e547f9525fa8088a8a958e0ca2409a514e33f5fa"
|
||||
dependencies = [
|
||||
"block-buffer",
|
||||
"cfg-if",
|
||||
@@ -1057,27 +1007,27 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "siphasher"
|
||||
version = "0.3.5"
|
||||
version = "0.3.7"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "cbce6d4507c7e4a3962091436e56e95290cb71fa302d0d270e32130b75fbff27"
|
||||
checksum = "533494a8f9b724d33625ab53c6c4800f7cc445895924a8ef649222dcb76e938b"
|
||||
|
||||
[[package]]
|
||||
name = "slab"
|
||||
version = "0.4.3"
|
||||
version = "0.4.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f173ac3d1a7e3b28003f40de0b5ce7fe2710f9b9dc3fc38664cebee46b3b6527"
|
||||
checksum = "9def91fd1e018fe007022791f865d0ccc9b3a0d5001e01aabb8b40e46000afb5"
|
||||
|
||||
[[package]]
|
||||
name = "smallvec"
|
||||
version = "1.6.1"
|
||||
version = "1.7.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "fe0f37c9e8f3c5a4a66ad655a93c74daac4ad00c441533bf5c6e7990bb42604e"
|
||||
checksum = "1ecab6c735a6bb4139c0caafd0cc3635748bbb3acf4550e8138122099251f309"
|
||||
|
||||
[[package]]
|
||||
name = "socket2"
|
||||
version = "0.4.0"
|
||||
version = "0.4.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9e3dfc207c526015c632472a77be09cf1b6e46866581aecae5cc38fb4235dea2"
|
||||
checksum = "5dc90fe6c7be1a323296982db1836d1ea9e47b6839496dde9a541bc496df3516"
|
||||
dependencies = [
|
||||
"libc",
|
||||
"winapi",
|
||||
@@ -1090,13 +1040,14 @@ source = "git+https://github.com/matrix-org/rust-matrix-state-map#211343e8dd8d14
|
||||
|
||||
[[package]]
|
||||
name = "string_cache"
|
||||
version = "0.8.1"
|
||||
version = "0.8.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "8ddb1139b5353f96e429e1a5e19fbaf663bddedaa06d1dbd49f82e352601209a"
|
||||
checksum = "923f0f39b6267d37d23ce71ae7235602134b250ace715dd2c90421998ddac0c6"
|
||||
dependencies = [
|
||||
"lazy_static",
|
||||
"new_debug_unreachable",
|
||||
"phf_shared",
|
||||
"parking_lot",
|
||||
"phf_shared 0.8.0",
|
||||
"precomputed-hash",
|
||||
"serde",
|
||||
]
|
||||
@@ -1119,29 +1070,48 @@ checksum = "8ea5119cdb4c55b55d432abb513a0429384878c15dde60cc77b1c99de1a95a6a"
|
||||
|
||||
[[package]]
|
||||
name = "subtle"
|
||||
version = "2.4.0"
|
||||
version = "2.4.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1e81da0851ada1f3e9d4312c704aa4f8806f0f9d69faaf8df2f3464b4a9437c2"
|
||||
checksum = "6bdef32e8150c2a081110b42772ffe7d7c9032b606bc226c8260fd97e0976601"
|
||||
|
||||
[[package]]
|
||||
name = "syn"
|
||||
version = "1.0.73"
|
||||
version = "1.0.80"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f71489ff30030d2ae598524f61326b902466f72a0fb1a8564c001cc63425bcc7"
|
||||
checksum = "d010a1623fbd906d51d650a9916aaefc05ffa0e4053ff7fe601167f3e715d194"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"unicode-xid",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "synapse_auto_compressor"
|
||||
version = "0.1.2"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"clap",
|
||||
"env_logger",
|
||||
"log",
|
||||
"log-panics",
|
||||
"openssl",
|
||||
"postgres",
|
||||
"postgres-openssl",
|
||||
"pyo3",
|
||||
"pyo3-log",
|
||||
"rand",
|
||||
"serial_test",
|
||||
"synapse_compress_state",
|
||||
"tikv-jemallocator",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "synapse_compress_state"
|
||||
version = "0.1.0"
|
||||
dependencies = [
|
||||
"clap",
|
||||
"env_logger 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"env_logger",
|
||||
"indicatif",
|
||||
"jemallocator",
|
||||
"log",
|
||||
"log-panics",
|
||||
"openssl",
|
||||
@@ -1153,6 +1123,7 @@ dependencies = [
|
||||
"rayon",
|
||||
"state-map",
|
||||
"string_cache",
|
||||
"tikv-jemallocator",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -1184,10 +1155,31 @@ dependencies = [
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "tinyvec"
|
||||
version = "1.2.0"
|
||||
name = "tikv-jemalloc-sys"
|
||||
version = "0.4.2+5.2.1-patched.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5b5220f05bb7de7f3f53c7c065e1199b3172696fe2db9f9c4d8ad9b4ee74c342"
|
||||
checksum = "5844e429d797c62945a566f8da4e24c7fe3fbd5d6617fd8bf7a0b7dc1ee0f22e"
|
||||
dependencies = [
|
||||
"cc",
|
||||
"fs_extra",
|
||||
"libc",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "tikv-jemallocator"
|
||||
version = "0.4.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3c14a5a604eb8715bc5785018a37d00739b180bcf609916ddf4393d33d49ccdf"
|
||||
dependencies = [
|
||||
"libc",
|
||||
"tikv-jemalloc-sys",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "tinyvec"
|
||||
version = "1.5.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f83b2a3d4d9091d0abd7eba4dc2710b1718583bd4d8992e2190720ea38f391f7"
|
||||
dependencies = [
|
||||
"tinyvec_macros",
|
||||
]
|
||||
@@ -1200,9 +1192,9 @@ checksum = "cda74da7e1a664f795bb1f8a87ec406fb89a02522cf6e50620d016add6dbbf5c"
|
||||
|
||||
[[package]]
|
||||
name = "tokio"
|
||||
version = "1.8.1"
|
||||
version = "1.12.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "98c8b05dc14c75ea83d63dd391100353789f5f24b8b3866542a5e85c8be8e985"
|
||||
checksum = "c2c2416fdedca8443ae44b4527de1ea633af61d8f7169ffa6e72c5b53d24efcc"
|
||||
dependencies = [
|
||||
"autocfg",
|
||||
"bytes",
|
||||
@@ -1215,11 +1207,11 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "tokio-openssl"
|
||||
version = "0.6.2"
|
||||
version = "0.6.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f24cddc8445a4dc8359cdd9e91c19d544fc95f672e32afe8945852b9381a09fe"
|
||||
checksum = "c08f9ffb7809f1b20c1b398d92acf4cc719874b3b2b2d9ea2f09b4a80350878a"
|
||||
dependencies = [
|
||||
"futures",
|
||||
"futures-util",
|
||||
"openssl",
|
||||
"openssl-sys",
|
||||
"tokio",
|
||||
@@ -1227,9 +1219,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "tokio-postgres"
|
||||
version = "0.7.2"
|
||||
version = "0.7.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2d2b1383c7e4fb9a09e292c7c6afb7da54418d53b045f1c1fac7a911411a2b8b"
|
||||
checksum = "b5c07a6ceeeb8515d53998ac4487788a21884e79d5651490bc31a7289f20a7d7"
|
||||
dependencies = [
|
||||
"async-trait",
|
||||
"byteorder",
|
||||
@@ -1250,9 +1242,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "tokio-util"
|
||||
version = "0.6.7"
|
||||
version = "0.6.8"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1caa0b0c8d94a049db56b5acf8cba99dc0623aab1b26d5b5f5e2d945846b3592"
|
||||
checksum = "08d3725d3efa29485e87311c5b699de63cde14b00ed4d256b8318aa30ca452cd"
|
||||
dependencies = [
|
||||
"bytes",
|
||||
"futures-core",
|
||||
@@ -1264,18 +1256,15 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "typenum"
|
||||
version = "1.13.0"
|
||||
version = "1.14.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "879f6906492a7cd215bfa4cf595b600146ccfac0c79bcbd1f3000162af5e8b06"
|
||||
checksum = "b63708a265f51345575b27fe43f9500ad611579e764c79edbc2037b1121959ec"
|
||||
|
||||
[[package]]
|
||||
name = "unicode-bidi"
|
||||
version = "0.3.5"
|
||||
version = "0.3.7"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "eeb8be209bb1c96b7c177c7420d26e04eccacb0eeae6b980e35fcb74678107e0"
|
||||
dependencies = [
|
||||
"matches",
|
||||
]
|
||||
checksum = "1a01404663e3db436ed2746d9fefef640d868edae3cceb81c3b8d5732fda678f"
|
||||
|
||||
[[package]]
|
||||
name = "unicode-normalization"
|
||||
@@ -1288,9 +1277,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "unicode-width"
|
||||
version = "0.1.8"
|
||||
version = "0.1.9"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9337591893a19b88d8d87f2cec1e73fad5cdfd10e5a6f349f498ad6ea2ffb1e3"
|
||||
checksum = "3ed742d4ea2bd1176e236172c8429aaf54486e7ac098db29ffe6529e0ce50973"
|
||||
|
||||
[[package]]
|
||||
name = "unicode-xid"
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
[workspace]
|
||||
members = ["auto_compressor", "compressor_integration_tests"]
|
||||
members = ["synapse_auto_compressor", "compressor_integration_tests"]
|
||||
|
||||
[package]
|
||||
authors = ["Erik Johnston"]
|
||||
@@ -11,7 +11,6 @@ edition = "2018"
|
||||
[dependencies]
|
||||
clap = "2.33.0"
|
||||
indicatif = "0.16.0"
|
||||
jemallocator = "0.3.2"
|
||||
openssl = "0.10.32"
|
||||
postgres = "0.19.0"
|
||||
postgres-openssl = "0.5.0"
|
||||
@@ -34,7 +33,11 @@ crate-type = ["cdylib", "rlib"]
|
||||
version = "0.14.1"
|
||||
features = ["extension-module","abi3-py36"]
|
||||
|
||||
[dependencies.tikv-jemallocator]
|
||||
version = "0.4.1"
|
||||
optional = true
|
||||
|
||||
[features]
|
||||
default = ["jemalloc"]
|
||||
jemalloc = []
|
||||
jemalloc = ["tikv-jemallocator"]
|
||||
no-progress-bars = []
|
||||
|
||||
22
Dockerfile
Normal file
22
Dockerfile
Normal file
@@ -0,0 +1,22 @@
|
||||
FROM rust:alpine AS builder
|
||||
|
||||
RUN apk add python3 musl-dev pkgconfig openssl-dev make
|
||||
|
||||
ENV RUSTFLAGS="-C target-feature=-crt-static"
|
||||
|
||||
WORKDIR /opt/synapse-compressor/
|
||||
|
||||
COPY . .
|
||||
|
||||
RUN cargo build
|
||||
|
||||
WORKDIR /opt/synapse-compressor/synapse_auto_compressor/
|
||||
|
||||
RUN cargo build
|
||||
|
||||
FROM alpine
|
||||
|
||||
RUN apk add --no-cache libgcc
|
||||
|
||||
COPY --from=builder /opt/synapse-compressor/target/debug/synapse_compress_state /usr/local/bin/synapse_compress_state
|
||||
COPY --from=builder /opt/synapse-compressor/target/debug/synapse_auto_compressor /usr/local/bin/synapse_auto_compressor
|
||||
28
README.md
28
README.md
@@ -3,7 +3,7 @@
|
||||
This workspace contains experimental tools that attempt to reduce the number of
|
||||
rows in the `state_groups_state` table inside of a Synapse Postgresql database.
|
||||
|
||||
# Automated tool: auto_compressor
|
||||
# Automated tool: synapse_auto_compressor
|
||||
|
||||
## Introduction:
|
||||
|
||||
@@ -11,7 +11,7 @@ This tool is significantly more simple to use than the manual tool (described be
|
||||
It scans through all of the rows in the `state_groups` database table from the start. When
|
||||
it finds a group that hasn't been compressed, it runs the compressor for a while on that
|
||||
group's room, saving where it got up to. After compressing a number of these chunks it stops,
|
||||
saving where it got up to for the next run of the `auto_compressor`.
|
||||
saving where it got up to for the next run of the `synapse_auto_compressor`.
|
||||
|
||||
It creates three extra tables in the database: `state_compressor_state` which stores the
|
||||
information needed to stop and start the compressor for each room, `state_compressor_progress`
|
||||
@@ -21,22 +21,20 @@ which stores how far through the `state_groups` table the compressor has scanned
|
||||
The tool can be run manually when you are running out of space, or be scheduled to run
|
||||
periodically.
|
||||
|
||||
The output from the auto_compressor will be sent to `auto_compressor.log` (in the directory
|
||||
that the compressor is run from).
|
||||
|
||||
## Building
|
||||
|
||||
This tool requires `cargo` to be installed. See https://www.rust-lang.org/tools/install
|
||||
for instructions on how to do this.
|
||||
|
||||
To build `auto_compressor`, clone this repository and navigate to the `autocompressor/`
|
||||
subdirectory. Then execute `cargo build`.
|
||||
To build `synapse_auto_compressor`, clone this repository and navigate to the
|
||||
`synapse_auto_compressor/` subdirectory. Then execute `cargo build`.
|
||||
|
||||
This will create an executable and store it in `auto_compressor/target/debug/auto_compressor`.
|
||||
This will create an executable and store it in
|
||||
`synapse_auto_compressor/target/debug/synapse_auto_compressor`.
|
||||
|
||||
## Example usage
|
||||
```
|
||||
$ auto_compressor -p postgresql://user:pass@localhost/synapse -c 500 -n 100
|
||||
$ synapse_auto_compressor -p postgresql://user:pass@localhost/synapse -c 500 -n 100
|
||||
```
|
||||
## Running Options
|
||||
|
||||
@@ -199,10 +197,10 @@ $ docker-compose down
|
||||
# Using the synapse_compress_state library
|
||||
|
||||
If you want to use the compressor in another project, it is recomended that you
|
||||
use jemalloc `https://github.com/gnzlbg/jemallocator`.
|
||||
use jemalloc `https://github.com/tikv/jemallocator`.
|
||||
|
||||
To prevent the progress bars from being shown, use the `no-progress-bars` feature.
|
||||
(See `auto_compressor/Cargo.toml` for an example)
|
||||
(See `synapse_auto_compressor/Cargo.toml` for an example)
|
||||
|
||||
# Troubleshooting
|
||||
|
||||
@@ -228,17 +226,17 @@ setting in [`postgresql.conf`](https://www.postgresql.org/docs/current/runtime-c
|
||||
The amount of output the tools produce can be altered by setting the RUST_LOG
|
||||
environment variable to something.
|
||||
|
||||
To get more logs when running the auto_compressor tool try the following:
|
||||
To get more logs when running the synapse_auto_compressor tool try the following:
|
||||
|
||||
```
|
||||
$ RUST_LOG=debug auto_compressor -p postgresql://user:pass@localhost/synapse -c 50 -n 100
|
||||
$ RUST_LOG=debug synapse_auto_compressor -p postgresql://user:pass@localhost/synapse -c 50 -n 100
|
||||
```
|
||||
|
||||
If you want to suppress all the debugging info you are getting from the
|
||||
Postgres client then try:
|
||||
|
||||
```
|
||||
RUST_LOG=auto_compressor=debug,synapse_compress_state=debug auto_compressor [etc.]
|
||||
RUST_LOG=synapse_auto_compressor=debug,synapse_compress_state=debug synapse_auto_compressor [etc.]
|
||||
```
|
||||
|
||||
This will only print the debugging information from those two packages. For more info see
|
||||
@@ -268,7 +266,7 @@ be a large problem.
|
||||
|
||||
## Compressor is trying to increase the number of rows
|
||||
|
||||
Backfilling can lead to issues with compression. The auto_compressor will
|
||||
Backfilling can lead to issues with compression. The synapse_auto_compressor will
|
||||
skip chunks it can't reduce the size of and so this should help jump over the backfilled
|
||||
state_groups. Lots of state resolution might also impact the ability to use the compressor.
|
||||
|
||||
|
||||
@@ -13,7 +13,7 @@ postgres = "0.19.0"
|
||||
postgres-openssl = "0.5.0"
|
||||
rand = "0.8.0"
|
||||
synapse_compress_state = { path = "../", features = ["no-progress-bars"] }
|
||||
auto_compressor = { path = "../auto_compressor/" }
|
||||
synapse_auto_compressor = { path = "../synapse_auto_compressor/" }
|
||||
env_logger = "0.9.0"
|
||||
log = "0.4.14"
|
||||
|
||||
|
||||
@@ -356,7 +356,7 @@ fn functions_are_self_consistent() {
|
||||
}
|
||||
|
||||
pub fn setup_logger() {
|
||||
// setup the logger for the auto_compressor
|
||||
// setup the logger for the synapse_auto_compressor
|
||||
// The default can be overwritten with RUST_LOG
|
||||
// see the README for more information
|
||||
if env::var("RUST_LOG").is_err() {
|
||||
@@ -366,7 +366,7 @@ pub fn setup_logger() {
|
||||
// default to printing the debug information for both packages being tested
|
||||
// (Note that just setting the global level to debug will log every sql transaction)
|
||||
log_builder.filter_module("synapse_compress_state", LevelFilter::Debug);
|
||||
log_builder.filter_module("auto_compressor", LevelFilter::Debug);
|
||||
log_builder.filter_module("synapse_auto_compressor", LevelFilter::Debug);
|
||||
// use try_init() incase the logger has been setup by some previous test
|
||||
let _ = log_builder.try_init();
|
||||
} else {
|
||||
|
||||
@@ -1,9 +1,5 @@
|
||||
use std::collections::BTreeMap;
|
||||
|
||||
use auto_compressor::{
|
||||
manager::{compress_chunks_of_database, run_compressor_on_room_chunk},
|
||||
state_saving::{connect_to_database, create_tables_if_needed},
|
||||
};
|
||||
use compressor_integration_tests::{
|
||||
add_contents_to_database, clear_compressor_state, database_collapsed_states_match_map,
|
||||
database_structure_matches_map, empty_database,
|
||||
@@ -14,6 +10,10 @@ use compressor_integration_tests::{
|
||||
setup_logger, DB_URL,
|
||||
};
|
||||
use serial_test::serial;
|
||||
use synapse_auto_compressor::{
|
||||
manager::{compress_chunks_of_database, run_compressor_on_room_chunk},
|
||||
state_saving::{connect_to_database, create_tables_if_needed},
|
||||
};
|
||||
use synapse_compress_state::Level;
|
||||
|
||||
#[test]
|
||||
|
||||
@@ -1,9 +1,9 @@
|
||||
use auto_compressor::state_saving::{
|
||||
use compressor_integration_tests::{clear_compressor_state, setup_logger, DB_URL};
|
||||
use serial_test::serial;
|
||||
use synapse_auto_compressor::state_saving::{
|
||||
connect_to_database, create_tables_if_needed, read_room_compressor_state,
|
||||
write_room_compressor_state,
|
||||
};
|
||||
use compressor_integration_tests::{clear_compressor_state, setup_logger, DB_URL};
|
||||
use serial_test::serial;
|
||||
use synapse_compress_state::Level;
|
||||
|
||||
#[test]
|
||||
|
||||
@@ -46,10 +46,11 @@ fn run_succeeds_without_crashing() {
|
||||
let transactions = true;
|
||||
let graphs = false;
|
||||
let commit_changes = false;
|
||||
let verify = true;
|
||||
|
||||
let config = Config::new(
|
||||
db_url.clone(),
|
||||
room_id.clone(),
|
||||
db_url,
|
||||
room_id,
|
||||
output_file,
|
||||
min_state_group,
|
||||
groups_to_compress,
|
||||
@@ -59,6 +60,7 @@ fn run_succeeds_without_crashing() {
|
||||
transactions,
|
||||
graphs,
|
||||
commit_changes,
|
||||
verify,
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
@@ -94,6 +96,7 @@ fn changes_commited_if_no_min_saved_rows() {
|
||||
let transactions = true;
|
||||
let graphs = false;
|
||||
let commit_changes = true;
|
||||
let verify = true;
|
||||
|
||||
let config = Config::new(
|
||||
db_url,
|
||||
@@ -107,6 +110,7 @@ fn changes_commited_if_no_min_saved_rows() {
|
||||
transactions,
|
||||
graphs,
|
||||
commit_changes,
|
||||
verify,
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
@@ -160,6 +164,7 @@ fn changes_commited_if_min_saved_rows_exceeded() {
|
||||
let transactions = true;
|
||||
let graphs = false;
|
||||
let commit_changes = true;
|
||||
let verify = true;
|
||||
|
||||
let config = Config::new(
|
||||
db_url,
|
||||
@@ -173,6 +178,7 @@ fn changes_commited_if_min_saved_rows_exceeded() {
|
||||
transactions,
|
||||
graphs,
|
||||
commit_changes,
|
||||
verify,
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
@@ -227,6 +233,7 @@ fn changes_not_commited_if_fewer_than_min_saved_rows() {
|
||||
let transactions = true;
|
||||
let graphs = false;
|
||||
let commit_changes = true;
|
||||
let verify = true;
|
||||
|
||||
let config = Config::new(
|
||||
db_url,
|
||||
@@ -240,6 +247,7 @@ fn changes_not_commited_if_fewer_than_min_saved_rows() {
|
||||
transactions,
|
||||
graphs,
|
||||
commit_changes,
|
||||
verify,
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
@@ -280,6 +288,7 @@ fn run_panics_if_invalid_db_url() {
|
||||
let transactions = true;
|
||||
let graphs = false;
|
||||
let commit_changes = true;
|
||||
let verify = true;
|
||||
|
||||
let config = Config::new(
|
||||
db_url,
|
||||
@@ -293,6 +302,7 @@ fn run_panics_if_invalid_db_url() {
|
||||
transactions,
|
||||
graphs,
|
||||
commit_changes,
|
||||
verify,
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
@@ -336,6 +346,7 @@ fn run_only_affects_given_room_id() {
|
||||
let transactions = true;
|
||||
let graphs = false;
|
||||
let commit_changes = true;
|
||||
let verify = true;
|
||||
|
||||
let config = Config::new(
|
||||
db_url,
|
||||
@@ -349,6 +360,7 @@ fn run_only_affects_given_room_id() {
|
||||
transactions,
|
||||
graphs,
|
||||
commit_changes,
|
||||
verify,
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
@@ -406,6 +418,7 @@ fn run_respects_groups_to_compress() {
|
||||
let transactions = true;
|
||||
let graphs = false;
|
||||
let commit_changes = true;
|
||||
let verify = true;
|
||||
|
||||
let config = Config::new(
|
||||
db_url,
|
||||
@@ -419,6 +432,7 @@ fn run_respects_groups_to_compress() {
|
||||
transactions,
|
||||
graphs,
|
||||
commit_changes,
|
||||
verify,
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
@@ -492,6 +506,7 @@ fn run_is_idempotent_when_run_on_whole_room() {
|
||||
let transactions = true;
|
||||
let graphs = false;
|
||||
let commit_changes = true;
|
||||
let verify = true;
|
||||
|
||||
let config1 = Config::new(
|
||||
db_url.clone(),
|
||||
@@ -505,21 +520,23 @@ fn run_is_idempotent_when_run_on_whole_room() {
|
||||
transactions,
|
||||
graphs,
|
||||
commit_changes,
|
||||
verify,
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
let config2 = Config::new(
|
||||
db_url.clone(),
|
||||
room_id.clone(),
|
||||
db_url,
|
||||
room_id,
|
||||
output_file2,
|
||||
min_state_group,
|
||||
groups_to_compress,
|
||||
min_saved_rows,
|
||||
max_state_group,
|
||||
level_sizes.clone(),
|
||||
level_sizes,
|
||||
transactions,
|
||||
graphs,
|
||||
commit_changes,
|
||||
verify,
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
|
||||
@@ -56,7 +56,7 @@ fn continue_run_called_twice_same_as_run() {
|
||||
|
||||
let start = Some(6);
|
||||
let chunk_size = 7;
|
||||
let level_info = chunk_stats_1.new_level_info.clone();
|
||||
let level_info = chunk_stats_1.new_level_info;
|
||||
|
||||
// Run the compressor with those settings
|
||||
let chunk_stats_2 = continue_run(start, chunk_size, &db_url, &room_id, &level_info).unwrap();
|
||||
|
||||
@@ -18,7 +18,7 @@ the compressor is run.
|
||||
|
||||
3. Navigate to the correct location
|
||||
For the automatic tool:
|
||||
`$ cd /home/synapse/rust-synapse-compress-state/auto_compressor`
|
||||
`$ cd /home/synapse/rust-synapse-compress-state/synpase_auto_compressor`
|
||||
For the manual tool:
|
||||
`$ cd /home/synapse/rust-synapse-compress-state`
|
||||
|
||||
@@ -30,9 +30,9 @@ This will install the relevant compressor tool into the activated virtual enviro
|
||||
## Automatic tool example:
|
||||
|
||||
```python
|
||||
import auto_compressor
|
||||
import synapse_auto_compressor
|
||||
|
||||
auto_compressor.compress_state_events_table(
|
||||
synapse_auto_compressor.compress_state_events_table(
|
||||
db_url="postgresql://localhost/synapse",
|
||||
chunk_size=500,
|
||||
default_levels="100,50,25",
|
||||
|
||||
@@ -181,12 +181,11 @@ impl<'a> Compressor<'a> {
|
||||
panic!("Can only call `create_new_tree` once");
|
||||
}
|
||||
|
||||
let pb: ProgressBar;
|
||||
if cfg!(feature = "no-progress-bars") {
|
||||
pb = ProgressBar::hidden();
|
||||
let pb: ProgressBar = if cfg!(feature = "no-progress-bars") {
|
||||
ProgressBar::hidden()
|
||||
} else {
|
||||
pb = ProgressBar::new(self.original_state_map.len() as u64);
|
||||
}
|
||||
ProgressBar::new(self.original_state_map.len() as u64)
|
||||
};
|
||||
pb.set_style(
|
||||
ProgressStyle::default_bar().template("[{elapsed_precise}] {bar} {pos}/{len} {msg}"),
|
||||
);
|
||||
|
||||
@@ -93,14 +93,7 @@ fn create_new_tree_does_nothing_if_already_compressed() {
|
||||
|
||||
for i in 0i64..=13i64 {
|
||||
// edge from map
|
||||
let pred_group = initial_edges.get(&i);
|
||||
|
||||
// Need Option<i64> not Option<&i64>
|
||||
let prev;
|
||||
match pred_group {
|
||||
Some(i) => prev = Some(*i),
|
||||
None => prev = None,
|
||||
}
|
||||
let prev = initial_edges.get(&i).copied();
|
||||
|
||||
// insert that edge into the initial map
|
||||
initial.insert(
|
||||
|
||||
@@ -54,7 +54,7 @@ fn get_head_returns_head() {
|
||||
#[test]
|
||||
fn has_space_returns_true_if_empty() {
|
||||
let l = Level::new(15);
|
||||
assert_eq!(l.has_space(), true);
|
||||
assert!(l.has_space());
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -65,7 +65,7 @@ fn has_space_returns_true_if_part_full() {
|
||||
l.update(1, true);
|
||||
l.update(143, true);
|
||||
l.update(15, true);
|
||||
assert_eq!(l.has_space(), true);
|
||||
assert!(l.has_space());
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -76,5 +76,5 @@ fn has_space_returns_false_if_full() {
|
||||
l.update(3, true);
|
||||
l.update(4, true);
|
||||
l.update(5, true);
|
||||
assert_eq!(l.has_space(), false);
|
||||
assert!(!l.has_space());
|
||||
}
|
||||
|
||||
@@ -142,14 +142,7 @@ fn stats_correct_if_no_changes() {
|
||||
|
||||
for i in 0i64..=13i64 {
|
||||
// edge from map
|
||||
let pred_group = initial_edges.get(&i);
|
||||
|
||||
// Need Option<i64> not Option<&i64>
|
||||
let prev;
|
||||
match pred_group {
|
||||
Some(i) => prev = Some(*i),
|
||||
None => prev = None,
|
||||
}
|
||||
let prev = initial_edges.get(&i).copied();
|
||||
|
||||
// insert that edge into the initial map
|
||||
initial.insert(
|
||||
|
||||
@@ -372,12 +372,11 @@ fn get_initial_data_from_db(
|
||||
// Copy the data from the database into a map
|
||||
let mut state_group_map: BTreeMap<i64, StateGroupEntry> = BTreeMap::new();
|
||||
|
||||
let pb: ProgressBar;
|
||||
if cfg!(feature = "no-progress-bars") {
|
||||
pb = ProgressBar::hidden();
|
||||
let pb: ProgressBar = if cfg!(feature = "no-progress-bars") {
|
||||
ProgressBar::hidden()
|
||||
} else {
|
||||
pb = ProgressBar::new_spinner();
|
||||
}
|
||||
ProgressBar::new_spinner()
|
||||
};
|
||||
pb.set_style(
|
||||
ProgressStyle::default_spinner().template("{spinner} [{elapsed}] {pos} rows retrieved"),
|
||||
);
|
||||
@@ -537,12 +536,11 @@ pub fn send_changes_to_db(
|
||||
debug!("Writing changes...");
|
||||
|
||||
// setup the progress bar
|
||||
let pb: ProgressBar;
|
||||
if cfg!(feature = "no-progress-bars") {
|
||||
pb = ProgressBar::hidden();
|
||||
let pb: ProgressBar = if cfg!(feature = "no-progress-bars") {
|
||||
ProgressBar::hidden()
|
||||
} else {
|
||||
pb = ProgressBar::new(old_map.len() as u64);
|
||||
}
|
||||
ProgressBar::new(old_map.len() as u64)
|
||||
};
|
||||
pb.set_style(
|
||||
ProgressStyle::default_bar().template("[{elapsed_precise}] {bar} {pos}/{len} {msg}"),
|
||||
);
|
||||
|
||||
58
src/lib.rs
58
src/lib.rs
@@ -27,7 +27,7 @@ use clap::{crate_authors, crate_description, crate_name, crate_version, value_t,
|
||||
use indicatif::{ProgressBar, ProgressStyle};
|
||||
use rayon::prelude::*;
|
||||
use state_map::StateMap;
|
||||
use std::{collections::BTreeMap, fs::File, io::Write, str::FromStr};
|
||||
use std::{collections::BTreeMap, convert::TryInto, fs::File, io::Write, str::FromStr};
|
||||
use string_cache::DefaultAtom as Atom;
|
||||
|
||||
mod compressor;
|
||||
@@ -109,6 +109,9 @@ pub struct Config {
|
||||
// Whether or not to commit changes to the database automatically
|
||||
// N.B. currently assumes transactions is true (to be on the safe side)
|
||||
commit_changes: bool,
|
||||
// Whether to verify the correctness of the compressed state groups by
|
||||
// comparing them to the original groups
|
||||
verify: bool,
|
||||
}
|
||||
|
||||
impl Config {
|
||||
@@ -223,6 +226,13 @@ impl Config {
|
||||
.long_help(concat!("If this flag is set then the changes the compressor makes will",
|
||||
" be committed to the database. This should be safe to use while synapse is running",
|
||||
" as it assumes by default that the transactions flag is set")),
|
||||
).arg(
|
||||
Arg::with_name("no_verify")
|
||||
.short("N")
|
||||
.help("Do not double-check that the compression was performed correctly")
|
||||
.long_help(concat!("If this flag is set then the verification of the compressed",
|
||||
" state groups, which compares them to the original groups, is skipped. This",
|
||||
" saves time at the cost of potentially generating mismatched state.")),
|
||||
).get_matches();
|
||||
|
||||
let db_url = matches
|
||||
@@ -262,6 +272,8 @@ impl Config {
|
||||
|
||||
let commit_changes = matches.is_present("commit_changes");
|
||||
|
||||
let verify = !matches.is_present("no_verify");
|
||||
|
||||
Config {
|
||||
db_url: String::from(db_url),
|
||||
output_file,
|
||||
@@ -274,6 +286,7 @@ impl Config {
|
||||
transactions,
|
||||
graphs,
|
||||
commit_changes,
|
||||
verify,
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -362,8 +375,8 @@ pub fn run(mut config: Config) {
|
||||
}
|
||||
|
||||
if let Some(min) = config.min_saved_rows {
|
||||
let saving = (original_summed_size - compressed_summed_size) as i32;
|
||||
if saving < min {
|
||||
let saving = original_summed_size.saturating_sub(compressed_summed_size);
|
||||
if saving < min.try_into().unwrap_or(0) {
|
||||
warn!(
|
||||
"Only {} rows would be saved by this compression. Skipping output.",
|
||||
saving
|
||||
@@ -372,7 +385,9 @@ pub fn run(mut config: Config) {
|
||||
}
|
||||
}
|
||||
|
||||
if config.verify {
|
||||
check_that_maps_match(&state_group_map, new_state_group_map);
|
||||
}
|
||||
|
||||
// If we are given an output file, we output the changes as SQL. If the
|
||||
// `transactions` argument is set we wrap each change to a state group in a
|
||||
@@ -492,12 +507,11 @@ fn output_sql(
|
||||
|
||||
info!("Writing changes...");
|
||||
|
||||
let pb: ProgressBar;
|
||||
if cfg!(feature = "no-progress-bars") {
|
||||
pb = ProgressBar::hidden();
|
||||
let pb: ProgressBar = if cfg!(feature = "no-progress-bars") {
|
||||
ProgressBar::hidden()
|
||||
} else {
|
||||
pb = ProgressBar::new(old_map.len() as u64);
|
||||
}
|
||||
ProgressBar::new(old_map.len() as u64)
|
||||
};
|
||||
pb.set_style(
|
||||
ProgressStyle::default_bar().template("[{elapsed_precise}] {bar} {pos}/{len} {msg}"),
|
||||
);
|
||||
@@ -607,12 +621,11 @@ fn check_that_maps_match(
|
||||
) {
|
||||
info!("Checking that state maps match...");
|
||||
|
||||
let pb: ProgressBar;
|
||||
if cfg!(feature = "no-progress-bars") {
|
||||
pb = ProgressBar::hidden();
|
||||
let pb: ProgressBar = if cfg!(feature = "no-progress-bars") {
|
||||
ProgressBar::hidden()
|
||||
} else {
|
||||
pb = ProgressBar::new(old_map.len() as u64);
|
||||
}
|
||||
ProgressBar::new(old_map.len() as u64)
|
||||
};
|
||||
pb.set_style(
|
||||
ProgressStyle::default_bar().template("[{elapsed_precise}] {bar} {pos}/{len} {msg}"),
|
||||
);
|
||||
@@ -695,6 +708,7 @@ impl Config {
|
||||
transactions: bool,
|
||||
graphs: bool,
|
||||
commit_changes: bool,
|
||||
verify: bool,
|
||||
) -> Result<Config, String> {
|
||||
let mut output: Option<File> = None;
|
||||
if let Some(file) = output_file {
|
||||
@@ -722,6 +736,7 @@ impl Config {
|
||||
transactions,
|
||||
graphs,
|
||||
commit_changes,
|
||||
verify,
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -746,6 +761,7 @@ impl Config {
|
||||
transactions = true,
|
||||
graphs = false,
|
||||
commit_changes = false,
|
||||
verify = true,
|
||||
)]
|
||||
fn run_compression(
|
||||
db_url: String,
|
||||
@@ -759,6 +775,7 @@ fn run_compression(
|
||||
transactions: bool,
|
||||
graphs: bool,
|
||||
commit_changes: bool,
|
||||
verify: bool,
|
||||
) -> PyResult<()> {
|
||||
let config = Config::new(
|
||||
db_url,
|
||||
@@ -772,6 +789,7 @@ fn run_compression(
|
||||
transactions,
|
||||
graphs,
|
||||
commit_changes,
|
||||
verify,
|
||||
);
|
||||
match config {
|
||||
Err(e) => Err(PyErr::new::<exceptions::PyException, _>(e)),
|
||||
@@ -788,7 +806,7 @@ fn synapse_compress_state(_py: Python, m: &PyModule) -> PyResult<()> {
|
||||
let _ = pyo3_log::Logger::default()
|
||||
// don't send out anything lower than a warning from other crates
|
||||
.filter(LevelFilter::Warn)
|
||||
// don't log warnings from synapse_compress_state, the auto_compressor handles these
|
||||
// don't log warnings from synapse_compress_state, the synapse_auto_compressor handles these
|
||||
// situations and provides better log messages
|
||||
.filter_target("synapse_compress_state".to_owned(), LevelFilter::Debug)
|
||||
.install();
|
||||
@@ -955,7 +973,6 @@ mod lib_tests {
|
||||
#[test]
|
||||
fn check_that_maps_match_returns_if_both_empty() {
|
||||
check_that_maps_match(&BTreeMap::new(), &BTreeMap::new());
|
||||
assert!(true);
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -988,7 +1005,6 @@ mod lib_tests {
|
||||
}
|
||||
|
||||
check_that_maps_match(&old_map, &BTreeMap::new());
|
||||
assert!(true);
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -1024,7 +1040,6 @@ mod lib_tests {
|
||||
}
|
||||
|
||||
check_that_maps_match(&BTreeMap::new(), &new_map);
|
||||
assert!(true);
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -1056,7 +1071,6 @@ mod lib_tests {
|
||||
}
|
||||
|
||||
check_that_maps_match(&BTreeMap::new(), &old_map.clone());
|
||||
assert!(true);
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -1119,7 +1133,6 @@ mod lib_tests {
|
||||
}
|
||||
|
||||
check_that_maps_match(&old_map, &new_map);
|
||||
assert!(true);
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -1201,7 +1214,6 @@ mod lib_tests {
|
||||
);
|
||||
|
||||
check_that_maps_match(&old_map, &new_map);
|
||||
assert!(true);
|
||||
}
|
||||
|
||||
//TODO: tests for correct SQL code produced by output_sql
|
||||
@@ -1224,6 +1236,7 @@ mod pyo3_tests {
|
||||
let transactions = false;
|
||||
let graphs = false;
|
||||
let commit_changes = false;
|
||||
let verify = true;
|
||||
|
||||
let config = Config::new(
|
||||
db_url.clone(),
|
||||
@@ -1237,6 +1250,7 @@ mod pyo3_tests {
|
||||
transactions,
|
||||
graphs,
|
||||
commit_changes,
|
||||
verify,
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
@@ -1270,6 +1284,7 @@ mod pyo3_tests {
|
||||
let transactions = true;
|
||||
let graphs = true;
|
||||
let commit_changes = true;
|
||||
let verify = true;
|
||||
|
||||
let config = Config::new(
|
||||
db_url.clone(),
|
||||
@@ -1283,11 +1298,12 @@ mod pyo3_tests {
|
||||
transactions,
|
||||
graphs,
|
||||
commit_changes,
|
||||
verify,
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(config.db_url, db_url);
|
||||
assert!(!config.output_file.is_none());
|
||||
assert!(config.output_file.is_some());
|
||||
assert_eq!(config.room_id, room_id);
|
||||
assert_eq!(config.min_state_group, Some(3225));
|
||||
assert_eq!(config.groups_to_compress, Some(970));
|
||||
|
||||
@@ -18,7 +18,7 @@
|
||||
|
||||
#[cfg(feature = "jemalloc")]
|
||||
#[global_allocator]
|
||||
static GLOBAL: jemallocator::Jemalloc = jemallocator::Jemalloc;
|
||||
static GLOBAL: tikv_jemallocator::Jemalloc = tikv_jemallocator::Jemalloc;
|
||||
|
||||
use log::LevelFilter;
|
||||
use std::env;
|
||||
|
||||
@@ -1,21 +1,27 @@
|
||||
[package]
|
||||
name = "auto_compressor"
|
||||
name = "synapse_auto_compressor"
|
||||
authors = ["William Ashton"]
|
||||
version = "0.1.0"
|
||||
version = "0.1.2"
|
||||
edition = "2018"
|
||||
|
||||
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
|
||||
[package.metadata.maturin]
|
||||
requires-python = ">=3.6"
|
||||
project-url = {Source = "https://github.com/matrix-org/rust-synapse-compress-state"}
|
||||
classifier = [
|
||||
"Development Status :: 4 - Beta",
|
||||
"Programming Language :: Rust",
|
||||
]
|
||||
|
||||
[dependencies]
|
||||
clap = "2.33.0"
|
||||
openssl = "0.10.32"
|
||||
postgres = "0.19.0"
|
||||
postgres-openssl = "0.5.0"
|
||||
jemallocator = "0.3.2"
|
||||
tikv-jemallocator = "0.4.1"
|
||||
rand = "0.8.0"
|
||||
serial_test = "0.5.1"
|
||||
synapse_compress_state = { path = "../", features = ["no-progress-bars"] }
|
||||
env_logger = { version = "0.9.0", git = "https://github.com/TilCreator/env_logger", branch = "fix_pipe" }
|
||||
env_logger = "0.9.0"
|
||||
log = "0.4.14"
|
||||
log-panics = "2.0.0"
|
||||
anyhow = "1.0.42"
|
||||
12
synapse_auto_compressor/README.md
Normal file
12
synapse_auto_compressor/README.md
Normal file
@@ -0,0 +1,12 @@
|
||||
# Auto Compressor
|
||||
|
||||
See the top level readme for information.
|
||||
|
||||
|
||||
## Publishing to PyPI
|
||||
|
||||
Bump the version number and run from the root directory of the repo:
|
||||
|
||||
```
|
||||
docker run -it --rm -v $(pwd):/io -e OPENSSL_STATIC=1 konstin2/maturin publish -m synapse_auto_compressor/Cargo.toml --cargo-extra-args "\--features='openssl/vendored'"
|
||||
```
|
||||
@@ -57,15 +57,16 @@ impl FromStr for LevelInfo {
|
||||
|
||||
// PyO3 INTERFACE STARTS HERE
|
||||
#[pymodule]
|
||||
fn auto_compressor(_py: Python, m: &PyModule) -> PyResult<()> {
|
||||
fn synapse_auto_compressor(_py: Python, m: &PyModule) -> PyResult<()> {
|
||||
let _ = pyo3_log::Logger::default()
|
||||
// don't send out anything lower than a warning from other crates
|
||||
.filter(LevelFilter::Warn)
|
||||
// don't log warnings from synapse_compress_state, the auto_compressor handles these
|
||||
// situations and provides better log messages
|
||||
// don't log warnings from synapse_compress_state, the
|
||||
// synapse_auto_compressor handles these situations and provides better
|
||||
// log messages
|
||||
.filter_target("synapse_compress_state".to_owned(), LevelFilter::Error)
|
||||
// log info and above for the auto_compressor
|
||||
.filter_target("auto_compressor".to_owned(), LevelFilter::Debug)
|
||||
// log info and above for the synapse_auto_compressor
|
||||
.filter_target("synapse_auto_compressor".to_owned(), LevelFilter::Debug)
|
||||
.install();
|
||||
// ensure any panics produce error messages in the log
|
||||
log_panics::init();
|
||||
@@ -92,7 +93,7 @@ fn auto_compressor(_py: Python, m: &PyModule) -> PyResult<()> {
|
||||
number_of_chunks: i64,
|
||||
) -> PyResult<()> {
|
||||
// Announce the start of the program to the logs
|
||||
log::info!("auto_compressor started");
|
||||
log::info!("synapse_auto_compressor started");
|
||||
|
||||
// Parse the default_level string into a LevelInfo struct
|
||||
let default_levels: LevelInfo = match default_levels.parse() {
|
||||
@@ -120,7 +121,7 @@ fn auto_compressor(_py: Python, m: &PyModule) -> PyResult<()> {
|
||||
return Err(PyErr::new::<PyRuntimeError, _>(format!("{:?}", e)));
|
||||
}
|
||||
|
||||
log::info!("auto_compressor finished");
|
||||
log::info!("synapse_auto_compressor finished");
|
||||
Ok(())
|
||||
}
|
||||
Ok(())
|
||||
@@ -17,45 +17,37 @@
|
||||
//! continue from where it left off.
|
||||
|
||||
#[global_allocator]
|
||||
static GLOBAL: jemallocator::Jemalloc = jemallocator::Jemalloc;
|
||||
static GLOBAL: tikv_jemallocator::Jemalloc = tikv_jemallocator::Jemalloc;
|
||||
|
||||
use auto_compressor::{manager, state_saving, LevelInfo};
|
||||
use clap::{crate_authors, crate_description, crate_name, crate_version, value_t, App, Arg};
|
||||
use log::LevelFilter;
|
||||
use std::{env, fs::OpenOptions};
|
||||
use std::env;
|
||||
use synapse_auto_compressor::{manager, state_saving, LevelInfo};
|
||||
|
||||
/// Execution starts here
|
||||
fn main() {
|
||||
// setup the logger for the auto_compressor
|
||||
// setup the logger for the synapse_auto_compressor
|
||||
// The default can be overwritten with RUST_LOG
|
||||
// see the README for more information
|
||||
let log_file = OpenOptions::new()
|
||||
.append(true)
|
||||
.create(true)
|
||||
.open("auto_compressor.log")
|
||||
.unwrap_or_else(|e| panic!("Error occured while opening the log file: {}", e));
|
||||
|
||||
if env::var("RUST_LOG").is_err() {
|
||||
let mut log_builder = env_logger::builder();
|
||||
log_builder.target(env_logger::Target::Pipe(Box::new(log_file)));
|
||||
// Ensure panics still come through
|
||||
log_builder.filter_module("panic", LevelFilter::Error);
|
||||
// Only output errors from the synapse_compress state library
|
||||
log_builder.filter_module("synapse_compress_state", LevelFilter::Error);
|
||||
// Output log levels info and above from auto_compressor
|
||||
log_builder.filter_module("auto_compressor", LevelFilter::Info);
|
||||
// Output log levels info and above from synapse_auto_compressor
|
||||
log_builder.filter_module("synapse_auto_compressor", LevelFilter::Info);
|
||||
log_builder.init();
|
||||
} else {
|
||||
// If RUST_LOG was set then use that
|
||||
let mut log_builder = env_logger::Builder::from_env("RUST_LOG");
|
||||
log_builder.target(env_logger::Target::Pipe(Box::new(log_file)));
|
||||
// Ensure panics still come through
|
||||
log_builder.filter_module("panic", LevelFilter::Error);
|
||||
log_builder.init();
|
||||
}
|
||||
log_panics::init();
|
||||
// Announce the start of the program to the logs
|
||||
log::info!("auto_compressor started");
|
||||
log::info!("synapse_auto_compressor started");
|
||||
|
||||
// parse the command line arguments using the clap crate
|
||||
let arguments = App::new(crate_name!())
|
||||
@@ -156,5 +148,5 @@ fn main() {
|
||||
manager::compress_chunks_of_database(db_url, chunk_size, &default_levels.0, number_of_chunks)
|
||||
.unwrap();
|
||||
|
||||
log::info!("auto_compressor finished");
|
||||
log::info!("synapse_auto_compressor finished");
|
||||
}
|
||||
Reference in New Issue
Block a user