Fix clippy lints

PR #74 (Log to stderr not to a file) did not remove all references to
log_file, the file was still created, but remained empty.

The synapse_auto_compressor failed if run in a read only environment.

Signed-off-by: saces <saces@c-base.org>

.dockerignore

@@ -0,0 +1,3 @@
+.git
+.github
+/target

.github/CODEOWNERS

@@ -0,0 +1,2 @@
+# Automatically request reviews from the synapse-core team when a pull request comes in.
+* @matrix-org/synapse-core

.vscode/settings.json

@@ -0,0 +1,3 @@
+{
+    "rust-analyzer.checkOnSave.command": "clippy"
+}

Cargo.lock

@@ -28,15 +28,15 @@ checksum = "61604a8f862e1d5c3229fdd78f8b02c68dcf73a4c4b05fd636d12240aaa242c1"
 
 [[package]]
 name = "arc-swap"
-version = "1.3.2"
+version = "1.4.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b5ab7d9e73059c86c36473f459b52adbd99c3554a4fec492caef460806006f00"
+checksum = "e6df5aef5c5830360ce5218cecb8f018af3438af5686ae945094affc86fdec63"
 
 [[package]]
 name = "async-trait"
-version = "0.1.50"
+version = "0.1.51"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0b98e84bbb4cbcdd97da190ba0c58a1bb0de2c1fdf67d159e192ed766aeca722"
+checksum = "44318e776df68115a881de9a8fd1b9e53368d7a4a5ce4cc48517da3393233a5e"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -54,26 +54,6 @@ dependencies = [
  "winapi",
 ]
 
-[[package]]
-name = "auto_compressor"
-version = "0.1.0"
-dependencies = [
- "anyhow",
- "clap",
- "env_logger 0.9.0 (git+https://github.com/TilCreator/env_logger?branch=fix_pipe)",
- "jemallocator",
- "log",
- "log-panics",
- "openssl",
- "postgres",
- "postgres-openssl",
- "pyo3",
- "pyo3-log",
- "rand",
- "serial_test",
- "synapse_compress_state",
-]
-
 [[package]]
 name = "autocfg"
 version = "1.0.1"
@@ -88,9 +68,9 @@ checksum = "904dfeac50f3cdaba28fc6f57fdcddb75f49ed61346676a78c4ffe55877802fd"
 
 [[package]]
 name = "bitflags"
-version = "1.2.1"
+version = "1.3.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "cf1de2fe8c75bc145a2f577add951f8134889b4795d47466a54a5c846d691693"
+checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a"
 
 [[package]]
 name = "block-buffer"
@@ -109,15 +89,15 @@ checksum = "14c189c53d098945499cdfa7ecc63567cf3886b3332b312a5b4585d8d3a6a610"
 
 [[package]]
 name = "bytes"
-version = "1.0.1"
+version = "1.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b700ce4376041dcd0a327fd0097c41095743c4c8af8887265942faf1100bd040"
+checksum = "c4872d67bab6358e59559027aa3b9157c53d9358c51423c17554809a8858e0f8"
 
 [[package]]
 name = "cc"
-version = "1.0.68"
+version = "1.0.71"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4a72c244c1ff497a746a7e1fb3d14bd08420ecda70c8f25c7112f2781652d787"
+checksum = "79c2681d6594606957bbb8631c4b90a7fcaaa72cdb714743a437b156d6a7eedd"
 
 [[package]]
 name = "cfg-if"
@@ -144,8 +124,7 @@ dependencies = [
 name = "compressor_integration_tests"
 version = "0.1.0"
 dependencies = [
- "auto_compressor",
+ "env_logger",
- "env_logger 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "log",
  "openssl",
  "postgres",
@@ -154,27 +133,28 @@ dependencies = [
  "serial_test",
  "state-map",
  "string_cache",
+ "synapse_auto_compressor",
  "synapse_compress_state",
 ]
 
 [[package]]
 name = "console"
-version = "0.14.1"
+version = "0.15.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3993e6445baa160675931ec041a5e03ca84b9c6e32a056150d3aa2bdda0a1f45"
+checksum = "a28b32d32ca44b70c3e4acd7db1babf555fa026e385fb95f18028f88848b3c31"
 dependencies = [
  "encode_unicode",
- "lazy_static",
  "libc",
+ "once_cell",
  "terminal_size",
  "winapi",
 ]
 
 [[package]]
 name = "cpufeatures"
-version = "0.1.5"
+version = "0.2.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "66c99696f6c9dd7f35d486b9d04d7e6e202aa3e8c40d553f2fdf5e7e0c6a71ef"
+checksum = "95059428f66df56b63431fdb4e1947ed2190586af5c5a8a8b71122bdf5a7f469"
 dependencies = [
  "libc",
 ]
@@ -191,9 +171,9 @@ dependencies = [
 
 [[package]]
 name = "crossbeam-deque"
-version = "0.8.0"
+version = "0.8.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "94af6efb46fef72616855b036a624cf27ba656ffc9be1b9a3c931cfc7749a9a9"
+checksum = "6455c0ca19f0d2fbf751b908d5c55c1f5cbc65e03c4225427254b46890bdde1e"
 dependencies = [
  "cfg-if",
  "crossbeam-epoch",
@@ -225,9 +205,9 @@ dependencies = [
 
 [[package]]
 name = "crypto-mac"
-version = "0.10.0"
+version = "0.11.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4857fd85a0c34b3c3297875b747c1e02e06b6a0ea32dd892d8192b9ce0813ea6"
+checksum = "b1d1a86f49236c215f271d40892d5fc950490551400b02ef360692c29815c714"
 dependencies = [
  "generic-array",
  "subtle",
@@ -267,18 +247,6 @@ dependencies = [
  "termcolor",
 ]
 
-[[package]]
-name = "env_logger"
-version = "0.9.0"
-source = "git+https://github.com/TilCreator/env_logger?branch=fix_pipe#3d09e0d824d9301cf1c0d4a9f148f8cfeb216329"
-dependencies = [
- "atty",
- "humantime",
- "log",
- "regex",
- "termcolor",
-]
-
 [[package]]
 name = "fallible-iterator"
 version = "0.2.0"
@@ -308,9 +276,9 @@ checksum = "2022715d62ab30faffd124d40b76f4134a550a87792276512b18d63272333394"
 
 [[package]]
 name = "futures"
-version = "0.3.15"
+version = "0.3.17"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0e7e43a803dae2fa37c1f6a8fe121e1f7bf9548b4dfc0522a42f34145dadfc27"
+checksum = "a12aa0eb539080d55c3f2d45a67c3b58b6b0773c1a3ca2dfec66d58c97fd66ca"
 dependencies = [
  "futures-channel",
  "futures-core",
@@ -323,9 +291,9 @@ dependencies = [
 
 [[package]]
 name = "futures-channel"
-version = "0.3.15"
+version = "0.3.17"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e682a68b29a882df0545c143dc3646daefe80ba479bcdede94d5a703de2871e2"
+checksum = "5da6ba8c3bb3c165d3c7319fc1cc8304facf1fb8db99c5de877183c08a273888"
 dependencies = [
  "futures-core",
  "futures-sink",
@@ -333,15 +301,15 @@ dependencies = [
 
 [[package]]
 name = "futures-core"
-version = "0.3.15"
+version = "0.3.17"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0402f765d8a89a26043b889b26ce3c4679d268fa6bb22cd7c6aad98340e179d1"
+checksum = "88d1c26957f23603395cd326b0ffe64124b818f4449552f960d815cfba83a53d"
 
 [[package]]
 name = "futures-executor"
-version = "0.3.15"
+version = "0.3.17"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "badaa6a909fac9e7236d0620a2f57f7664640c56575b71a7552fbd68deafab79"
+checksum = "45025be030969d763025784f7f355043dc6bc74093e4ecc5000ca4dc50d8745c"
 dependencies = [
  "futures-core",
  "futures-task",
@@ -350,15 +318,15 @@ dependencies = [
 
 [[package]]
 name = "futures-io"
-version = "0.3.15"
+version = "0.3.17"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "acc499defb3b348f8d8f3f66415835a9131856ff7714bf10dadfc4ec4bdb29a1"
+checksum = "522de2a0fe3e380f1bc577ba0474108faf3f6b18321dbf60b3b9c39a75073377"
 
 [[package]]
 name = "futures-macro"
-version = "0.3.15"
+version = "0.3.17"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a4c40298486cdf52cc00cd6d6987892ba502c7656a16a4192a9992b1ccedd121"
+checksum = "18e4a4b95cea4b4ccbcf1c5675ca7c4ee4e9e75eb79944d07defde18068f79bb"
 dependencies = [
  "autocfg",
  "proc-macro-hack",
@@ -369,21 +337,21 @@ dependencies = [
 
 [[package]]
 name = "futures-sink"
-version = "0.3.15"
+version = "0.3.17"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a57bead0ceff0d6dde8f465ecd96c9338121bb7717d3e7b108059531870c4282"
+checksum = "36ea153c13024fe480590b3e3d4cad89a0cfacecc24577b68f86c6ced9c2bc11"
 
 [[package]]
 name = "futures-task"
-version = "0.3.15"
+version = "0.3.17"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8a16bef9fc1a4dddb5bee51c989e3fbba26569cbb0e31f5b303c184e3dd33dae"
+checksum = "1d3d00f4eddb73e498a54394f228cd55853bdf059259e8e7bc6e69d408892e99"
 
 [[package]]
 name = "futures-util"
-version = "0.3.15"
+version = "0.3.17"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "feb5c238d27e2bf94ffdfd27b2c29e3df4a68c4193bb6427384259e2bf191967"
+checksum = "36568465210a3a6ee45e1f165136d68671471a501e632e9a98d96872222b5481"
 dependencies = [
  "autocfg",
  "futures-channel",
@@ -432,9 +400,9 @@ dependencies = [
 
 [[package]]
 name = "hmac"
-version = "0.10.1"
+version = "0.11.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c1441c6b1e930e2817404b5046f1f989899143a12bf92de603b69f4e0aee1e15"
+checksum = "2a2a2320eb7ec0ebe8da8f744d7812d9fc4cb4d09344ac01898dbcb6a20ae69b"
 dependencies = [
  "crypto-mac",
  "digest",
@@ -483,34 +451,13 @@ dependencies = [
 
 [[package]]
 name = "instant"
-version = "0.1.10"
+version = "0.1.12"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bee0328b1209d157ef001c94dd85b4f8f64139adb0eac2659f4b08382b2f474d"
+checksum = "7a5bbe824c507c5da5956355e86a746d82e0e1464f65d862cc5e71da70e94b2c"
 dependencies = [
  "cfg-if",
 ]
 
-[[package]]
-name = "jemalloc-sys"
-version = "0.3.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0d3b9f3f5c9b31aa0f5ed3260385ac205db665baa41d49bb8338008ae94ede45"
-dependencies = [
- "cc",
- "fs_extra",
- "libc",
-]
-
-[[package]]
-name = "jemallocator"
-version = "0.3.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "43ae63fcfc45e99ab3d1b29a46782ad679e98436c3169d15a167a1108a724b69"
-dependencies = [
- "jemalloc-sys",
- "libc",
-]
-
 [[package]]
 name = "lazy_static"
 version = "1.4.0"
@@ -519,15 +466,15 @@ checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646"
 
 [[package]]
 name = "libc"
-version = "0.2.98"
+version = "0.2.104"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "320cfe77175da3a483efed4bc0adc1968ca050b098ce4f2f1c13a56626128790"
+checksum = "7b2f96d100e1cf1929e7719b7edb3b90ab5298072638fccd77be9ce942ecdfce"
 
 [[package]]
 name = "lock_api"
-version = "0.4.4"
+version = "0.4.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0382880606dff6d15c9476c416d18690b72742aa7b605bb6dd6ec9030fbf07eb"
+checksum = "712a4d093c9976e24e7dbca41db895dabcbac38eb5f4045393d17a95bdfb1109"
 dependencies = [
  "scopeguard",
 ]
@@ -550,12 +497,6 @@ dependencies = [
  "log",
 ]
 
-[[package]]
-name = "matches"
-version = "0.1.8"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7ffc5c5338469d4d3ea17d269fa8ea3512ad247247c30bd2df69e68309ed0a08"
-
 [[package]]
 name = "md-5"
 version = "0.9.1"
@@ -569,9 +510,9 @@ dependencies = [
 
 [[package]]
 name = "memchr"
-version = "2.4.0"
+version = "2.4.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b16bd47d9e329435e309c58469fe0791c2d0d1ba96ec0954152a5ae2b04387dc"
+checksum = "308cc39be01b73d0d18f82a0e7b2a3df85245f84af96fdddc5d202d27e47b86a"
 
 [[package]]
 name = "memoffset"
@@ -584,9 +525,9 @@ dependencies = [
 
 [[package]]
 name = "mio"
-version = "0.7.13"
+version = "0.7.14"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8c2bdb6314ec10835cd3293dd268473a835c02b7b352e788be788b3c6ca6bb16"
+checksum = "8067b404fe97c70829f082dec8bcf4f71225d7eaea1d8645349cb76fa06205cc"
 dependencies = [
  "libc",
  "log",
@@ -649,9 +590,9 @@ checksum = "624a8340c38c1b80fd549087862da4ba43e08858af025b236e509b6649fc13d5"
 
 [[package]]
 name = "openssl"
-version = "0.10.35"
+version = "0.10.36"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "549430950c79ae24e6d02e0b7404534ecf311d94cc9f861e9e4020187d13d885"
+checksum = "8d9facdb76fec0b73c406f125d44d86fdad818d66fef0531eec9233ca425ff4a"
 dependencies = [
  "bitflags",
  "cfg-if",
@@ -663,9 +604,9 @@ dependencies = [
 
 [[package]]
 name = "openssl-sys"
-version = "0.9.65"
+version = "0.9.67"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7a7907e3bfa08bb85105209cdfcb6c63d109f8f6c1ed6ca318fff5c1853fbc1d"
+checksum = "69df2d8dfc6ce3aaf44b40dec6f487d5a886516cf6879c49e98e0710f310a058"
 dependencies = [
  "autocfg",
  "cc",
@@ -676,9 +617,9 @@ dependencies = [
 
 [[package]]
 name = "parking_lot"
-version = "0.11.1"
+version = "0.11.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6d7744ac029df22dca6284efe4e898991d28e3085c706c972bcd7da4a27a15eb"
+checksum = "7d17b78036a60663b797adeaee46f5c9dfebb86948d1255007a1d6be0271ff99"
 dependencies = [
  "instant",
  "lock_api",
@@ -687,9 +628,9 @@ dependencies = [
 
 [[package]]
 name = "parking_lot_core"
-version = "0.8.3"
+version = "0.8.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fa7a782938e745763fe6907fc6ba86946d72f49fe7e21de074e08128a99fb018"
+checksum = "d76e8e1493bcac0d2766c42737f34458f1c8c50c0d23bcb24ea953affb273216"
 dependencies = [
  "cfg-if",
  "instant",
@@ -726,11 +667,11 @@ checksum = "d4fd5641d01c8f18a23da7b6fe29298ff4b55afcccdf78973b24cf3175fee32e"
 
 [[package]]
 name = "phf"
-version = "0.8.0"
+version = "0.10.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3dfb61232e34fcb633f43d12c58f83c1df82962dcdfa565a4e866ffc17dafe12"
+checksum = "b9fc3db1018c4b59d7d582a739436478b6035138b6aecbce989fc91c3e98409f"
 dependencies = [
- "phf_shared",
+ "phf_shared 0.10.0",
 ]
 
 [[package]]
@@ -742,6 +683,15 @@ dependencies = [
  "siphasher",
 ]
 
+[[package]]
+name = "phf_shared"
+version = "0.10.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b6796ad771acdc0123d2a88dc428b5e38ef24456743ddb1744ed628f9815c096"
+dependencies = [
+ "siphasher",
+]
+
 [[package]]
 name = "pin-project-lite"
 version = "0.2.7"
@@ -756,15 +706,15 @@ checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184"
 
 [[package]]
 name = "pkg-config"
-version = "0.3.19"
+version = "0.3.20"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3831453b3449ceb48b6d9c7ad7c96d5ea673e9b470a1dc578c2ce6521230884c"
+checksum = "7c9b1041b4387893b91ee6746cddfc28516aff326a3519fb2adf820932c5e6cb"
 
 [[package]]
 name = "postgres"
-version = "0.19.1"
+version = "0.19.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c7871ee579860d8183f542e387b176a25f2656b9fb5211e045397f745a68d1c2"
+checksum = "eb76d6535496f633fa799bb872ffb4790e9cbdedda9d35564ca0252f930c0dd5"
 dependencies = [
  "bytes",
  "fallible-iterator",
@@ -789,9 +739,9 @@ dependencies = [
 
 [[package]]
 name = "postgres-protocol"
-version = "0.6.1"
+version = "0.6.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ff3e0f70d32e20923cabf2df02913be7c1842d4c772db8065c00fcfdd1d1bff3"
+checksum = "b145e6a4ed52cb316a27787fc20fe8a25221cb476479f61e4e0327c15b98d91a"
 dependencies = [
  "base64",
  "byteorder",
@@ -807,9 +757,9 @@ dependencies = [
 
 [[package]]
 name = "postgres-types"
-version = "0.2.1"
+version = "0.2.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "430f4131e1b7657b0cd9a2b0c3408d77c9a43a042d300b8c77f981dffcc43a2f"
+checksum = "04619f94ba0cc80999f4fc7073607cb825bc739a883cb6d20900fc5e009d6b0d"
 dependencies = [
  "bytes",
  "fallible-iterator",
@@ -818,9 +768,9 @@ dependencies = [
 
 [[package]]
 name = "ppv-lite86"
-version = "0.2.10"
+version = "0.2.14"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ac74c624d6b2d21f425f752262f42188365d7b8ff1aff74c82e45136510a4857"
+checksum = "c3ca011bd0129ff4ae15cd04c4eef202cadf6c51c21e47aba319b4e0501db741"
 
 [[package]]
 name = "precomputed-hash"
@@ -842,18 +792,18 @@ checksum = "bc881b2c22681370c6a780e47af9840ef841837bc98118431d4e1868bd0c1086"
 
 [[package]]
 name = "proc-macro2"
-version = "1.0.27"
+version = "1.0.30"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f0d8caf72986c1a598726adc988bb5984792ef84f5ee5aa50209145ee8077038"
+checksum = "edc3358ebc67bc8b7fa0c007f945b0b18226f78437d61bec735a9eb96b61ee70"
 dependencies = [
  "unicode-xid",
 ]
 
 [[package]]
 name = "pyo3"
-version = "0.14.1"
+version = "0.14.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "338f7f3701e11fd7f76508c91fbcaabc982564bcaf4d1ca7e1574ff2b4778aec"
+checksum = "35100f9347670a566a67aa623369293703322bb9db77d99d7df7313b575ae0c8"
 dependencies = [
  "cfg-if",
  "indoc",
@@ -867,9 +817,9 @@ dependencies = [
 
 [[package]]
 name = "pyo3-build-config"
-version = "0.14.1"
+version = "0.14.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "dcb2e98cc9ccc83d4f7115c8f925e0057e88c8d324b1bc4c2db4a7270c06ac9d"
+checksum = "d12961738cacbd7f91b7c43bc25cfeeaa2698ad07a04b3be0aa88b950865738f"
 dependencies = [
  "once_cell",
 ]
@@ -887,9 +837,9 @@ dependencies = [
 
 [[package]]
 name = "pyo3-macros"
-version = "0.14.1"
+version = "0.14.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "cfb8671a42d0ecc4bec8cc107ae96d49292ca20cd1968e09b98af4aafd516adf"
+checksum = "fc0bc5215d704824dfddddc03f93cb572e1155c68b6761c37005e1c288808ea8"
 dependencies = [
  "pyo3-macros-backend",
  "quote",
@@ -898,9 +848,9 @@ dependencies = [
 
 [[package]]
 name = "pyo3-macros-backend"
-version = "0.14.1"
+version = "0.14.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9addf6dc422f05d4949cc0990195ee74fa43e3c3780cc9a1972fe9e7b68a9f48"
+checksum = "71623fc593224afaab918aa3afcaf86ed2f43d34f6afde7f3922608f253240df"
 dependencies = [
  "proc-macro2",
  "pyo3-build-config",
@@ -910,9 +860,9 @@ dependencies = [
 
 [[package]]
 name = "quote"
-version = "1.0.9"
+version = "1.0.10"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c3d0b9745dc2debf507c8422de05d7226cc1f0644216dfdfead988f9b1ab32a7"
+checksum = "38bc8cc6a5f2e3655e0899c1b848643b2562f853f114bfec7be120678e3ace05"
 dependencies = [
  "proc-macro2",
 ]
@@ -984,9 +934,9 @@ dependencies = [
 
 [[package]]
 name = "redox_syscall"
-version = "0.2.9"
+version = "0.2.10"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5ab49abadf3f9e1c4bc499e8845e152ad87d2ad2d30371841171169e9d75feee"
+checksum = "8383f39639269cde97d255a32bdb68c047337295414940c68bdd30c2e13203ff"
 dependencies = [
  "bitflags",
 ]
@@ -1016,9 +966,9 @@ checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd"
 
 [[package]]
 name = "serde"
-version = "1.0.126"
+version = "1.0.130"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ec7505abeacaec74ae4778d9d9328fe5a5d04253220a85c4ee022239fc996d03"
+checksum = "f12d06de37cf59146fbdecab66aa99f9fe4f78722e3607577a5375d66bd0c913"
 
 [[package]]
 name = "serial_test"
@@ -1044,9 +994,9 @@ dependencies = [
 
 [[package]]
 name = "sha2"
-version = "0.9.5"
+version = "0.9.8"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b362ae5752fd2137731f9fa25fd4d9058af34666ca1966fb969119cc35719f12"
+checksum = "b69f9a4c9740d74c5baa3fd2e547f9525fa8088a8a958e0ca2409a514e33f5fa"
 dependencies = [
  "block-buffer",
  "cfg-if",
@@ -1057,27 +1007,27 @@ dependencies = [
 
 [[package]]
 name = "siphasher"
-version = "0.3.5"
+version = "0.3.7"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "cbce6d4507c7e4a3962091436e56e95290cb71fa302d0d270e32130b75fbff27"
+checksum = "533494a8f9b724d33625ab53c6c4800f7cc445895924a8ef649222dcb76e938b"
 
 [[package]]
 name = "slab"
-version = "0.4.3"
+version = "0.4.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f173ac3d1a7e3b28003f40de0b5ce7fe2710f9b9dc3fc38664cebee46b3b6527"
+checksum = "9def91fd1e018fe007022791f865d0ccc9b3a0d5001e01aabb8b40e46000afb5"
 
 [[package]]
 name = "smallvec"
-version = "1.6.1"
+version = "1.7.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fe0f37c9e8f3c5a4a66ad655a93c74daac4ad00c441533bf5c6e7990bb42604e"
+checksum = "1ecab6c735a6bb4139c0caafd0cc3635748bbb3acf4550e8138122099251f309"
 
 [[package]]
 name = "socket2"
-version = "0.4.0"
+version = "0.4.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9e3dfc207c526015c632472a77be09cf1b6e46866581aecae5cc38fb4235dea2"
+checksum = "5dc90fe6c7be1a323296982db1836d1ea9e47b6839496dde9a541bc496df3516"
 dependencies = [
  "libc",
  "winapi",
@@ -1090,13 +1040,14 @@ source = "git+https://github.com/matrix-org/rust-matrix-state-map#211343e8dd8d14
 
 [[package]]
 name = "string_cache"
-version = "0.8.1"
+version = "0.8.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8ddb1139b5353f96e429e1a5e19fbaf663bddedaa06d1dbd49f82e352601209a"
+checksum = "923f0f39b6267d37d23ce71ae7235602134b250ace715dd2c90421998ddac0c6"
 dependencies = [
  "lazy_static",
  "new_debug_unreachable",
- "phf_shared",
+ "parking_lot",
+ "phf_shared 0.8.0",
  "precomputed-hash",
  "serde",
 ]
@@ -1119,29 +1070,48 @@ checksum = "8ea5119cdb4c55b55d432abb513a0429384878c15dde60cc77b1c99de1a95a6a"
 
 [[package]]
 name = "subtle"
-version = "2.4.0"
+version = "2.4.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1e81da0851ada1f3e9d4312c704aa4f8806f0f9d69faaf8df2f3464b4a9437c2"
+checksum = "6bdef32e8150c2a081110b42772ffe7d7c9032b606bc226c8260fd97e0976601"
 
 [[package]]
 name = "syn"
-version = "1.0.73"
+version = "1.0.80"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f71489ff30030d2ae598524f61326b902466f72a0fb1a8564c001cc63425bcc7"
+checksum = "d010a1623fbd906d51d650a9916aaefc05ffa0e4053ff7fe601167f3e715d194"
 dependencies = [
  "proc-macro2",
  "quote",
  "unicode-xid",
 ]
 
+[[package]]
+name = "synapse_auto_compressor"
+version = "0.1.2"
+dependencies = [
+ "anyhow",
+ "clap",
+ "env_logger",
+ "log",
+ "log-panics",
+ "openssl",
+ "postgres",
+ "postgres-openssl",
+ "pyo3",
+ "pyo3-log",
+ "rand",
+ "serial_test",
+ "synapse_compress_state",
+ "tikv-jemallocator",
+]
+
 [[package]]
 name = "synapse_compress_state"
 version = "0.1.0"
 dependencies = [
  "clap",
- "env_logger 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)",
+ "env_logger",
  "indicatif",
- "jemallocator",
  "log",
  "log-panics",
  "openssl",
@@ -1153,6 +1123,7 @@ dependencies = [
  "rayon",
  "state-map",
  "string_cache",
+ "tikv-jemallocator",
 ]
 
 [[package]]
@@ -1184,10 +1155,31 @@ dependencies = [
 ]
 
 [[package]]
-name = "tinyvec"
+name = "tikv-jemalloc-sys"
-version = "1.2.0"
+version = "0.4.2+5.2.1-patched.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5b5220f05bb7de7f3f53c7c065e1199b3172696fe2db9f9c4d8ad9b4ee74c342"
+checksum = "5844e429d797c62945a566f8da4e24c7fe3fbd5d6617fd8bf7a0b7dc1ee0f22e"
+dependencies = [
+ "cc",
+ "fs_extra",
+ "libc",
+]
+
+[[package]]
+name = "tikv-jemallocator"
+version = "0.4.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3c14a5a604eb8715bc5785018a37d00739b180bcf609916ddf4393d33d49ccdf"
+dependencies = [
+ "libc",
+ "tikv-jemalloc-sys",
+]
+
+[[package]]
+name = "tinyvec"
+version = "1.5.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f83b2a3d4d9091d0abd7eba4dc2710b1718583bd4d8992e2190720ea38f391f7"
 dependencies = [
  "tinyvec_macros",
 ]
@@ -1200,9 +1192,9 @@ checksum = "cda74da7e1a664f795bb1f8a87ec406fb89a02522cf6e50620d016add6dbbf5c"
 
 [[package]]
 name = "tokio"
-version = "1.8.1"
+version = "1.12.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "98c8b05dc14c75ea83d63dd391100353789f5f24b8b3866542a5e85c8be8e985"
+checksum = "c2c2416fdedca8443ae44b4527de1ea633af61d8f7169ffa6e72c5b53d24efcc"
 dependencies = [
  "autocfg",
  "bytes",
@@ -1215,11 +1207,11 @@ dependencies = [
 
 [[package]]
 name = "tokio-openssl"
-version = "0.6.2"
+version = "0.6.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f24cddc8445a4dc8359cdd9e91c19d544fc95f672e32afe8945852b9381a09fe"
+checksum = "c08f9ffb7809f1b20c1b398d92acf4cc719874b3b2b2d9ea2f09b4a80350878a"
 dependencies = [
- "futures",
+ "futures-util",
  "openssl",
  "openssl-sys",
  "tokio",
@@ -1227,9 +1219,9 @@ dependencies = [
 
 [[package]]
 name = "tokio-postgres"
-version = "0.7.2"
+version = "0.7.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2d2b1383c7e4fb9a09e292c7c6afb7da54418d53b045f1c1fac7a911411a2b8b"
+checksum = "b5c07a6ceeeb8515d53998ac4487788a21884e79d5651490bc31a7289f20a7d7"
 dependencies = [
  "async-trait",
  "byteorder",
@@ -1250,9 +1242,9 @@ dependencies = [
 
 [[package]]
 name = "tokio-util"
-version = "0.6.7"
+version = "0.6.8"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1caa0b0c8d94a049db56b5acf8cba99dc0623aab1b26d5b5f5e2d945846b3592"
+checksum = "08d3725d3efa29485e87311c5b699de63cde14b00ed4d256b8318aa30ca452cd"
 dependencies = [
  "bytes",
  "futures-core",
@@ -1264,18 +1256,15 @@ dependencies = [
 
 [[package]]
 name = "typenum"
-version = "1.13.0"
+version = "1.14.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "879f6906492a7cd215bfa4cf595b600146ccfac0c79bcbd1f3000162af5e8b06"
+checksum = "b63708a265f51345575b27fe43f9500ad611579e764c79edbc2037b1121959ec"
 
 [[package]]
 name = "unicode-bidi"
-version = "0.3.5"
+version = "0.3.7"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "eeb8be209bb1c96b7c177c7420d26e04eccacb0eeae6b980e35fcb74678107e0"
+checksum = "1a01404663e3db436ed2746d9fefef640d868edae3cceb81c3b8d5732fda678f"
-dependencies = [
- "matches",
-]
 
 [[package]]
 name = "unicode-normalization"
@@ -1288,9 +1277,9 @@ dependencies = [
 
 [[package]]
 name = "unicode-width"
-version = "0.1.8"
+version = "0.1.9"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9337591893a19b88d8d87f2cec1e73fad5cdfd10e5a6f349f498ad6ea2ffb1e3"
+checksum = "3ed742d4ea2bd1176e236172c8429aaf54486e7ac098db29ffe6529e0ce50973"
 
 [[package]]
 name = "unicode-xid"

Cargo.toml

@@ -1,5 +1,5 @@
 [workspace]
-members = ["auto_compressor", "compressor_integration_tests"]
+members = ["synapse_auto_compressor", "compressor_integration_tests"]
 
 [package]
 authors = ["Erik Johnston"]
@@ -11,7 +11,6 @@ edition = "2018"
 [dependencies]
 clap = "2.33.0"
 indicatif = "0.16.0"
-jemallocator = "0.3.2"
 openssl = "0.10.32"
 postgres = "0.19.0"
 postgres-openssl = "0.5.0"
@@ -34,7 +33,11 @@ crate-type = ["cdylib", "rlib"]
 version = "0.14.1"
 features = ["extension-module","abi3-py36"]
 
+[dependencies.tikv-jemallocator]
+version = "0.4.1"
+optional = true
+
 [features]
 default = ["jemalloc"]
-jemalloc = []
+jemalloc = ["tikv-jemallocator"]
 no-progress-bars = []

Dockerfile

@@ -0,0 +1,22 @@
+FROM rust:alpine AS builder
+
+RUN apk add python3 musl-dev pkgconfig openssl-dev make
+
+ENV RUSTFLAGS="-C target-feature=-crt-static"
+
+WORKDIR /opt/synapse-compressor/
+
+COPY . .
+
+RUN cargo build
+
+WORKDIR /opt/synapse-compressor/synapse_auto_compressor/
+
+RUN cargo build
+
+FROM alpine
+
+RUN apk add --no-cache libgcc
+
+COPY --from=builder /opt/synapse-compressor/target/debug/synapse_compress_state /usr/local/bin/synapse_compress_state
+COPY --from=builder /opt/synapse-compressor/target/debug/synapse_auto_compressor /usr/local/bin/synapse_auto_compressor

README.md

@@ -3,7 +3,7 @@
 This workspace contains experimental tools that attempt to reduce the number of
 rows in the `state_groups_state` table inside of a Synapse Postgresql database.
 
-# Automated tool: auto_compressor
+# Automated tool: synapse_auto_compressor
 
 ## Introduction:
 
@@ -11,7 +11,7 @@ This tool is significantly more simple to use than the manual tool (described be
 It scans through all of the rows in the `state_groups` database table from the start. When
 it finds a group that hasn't been compressed, it runs the compressor for a while on that
 group's room, saving where it got up to. After compressing a number of these chunks it stops,
-saving where it got up to for the next run of the `auto_compressor`.
+saving where it got up to for the next run of the `synapse_auto_compressor`.
 
 It creates three extra tables in the database: `state_compressor_state` which stores the
 information needed to stop and start the compressor for each room, `state_compressor_progress`
@@ -21,22 +21,20 @@ which stores how far through the `state_groups` table the compressor has scanned
 The tool can be run manually when you are running out of space, or be scheduled to run
 periodically.
 
-The output from the auto_compressor will be sent to `auto_compressor.log` (in the directory
-that the compressor is run from).
-
 ## Building
 
 This tool requires `cargo` to be installed. See https://www.rust-lang.org/tools/install
 for instructions on how to do this.
 
-To build `auto_compressor`, clone this repository and navigate to the `autocompressor/` 
+To build `synapse_auto_compressor`, clone this repository and navigate to the
-subdirectory. Then execute `cargo build`.
+`synapse_auto_compressor/` subdirectory. Then execute `cargo build`.
 
-This will create an executable and store it in `auto_compressor/target/debug/auto_compressor`.
+This will create an executable and store it in
+`synapse_auto_compressor/target/debug/synapse_auto_compressor`.
 
 ## Example usage
 ```
-$ auto_compressor -p postgresql://user:pass@localhost/synapse -c 500 -n 100
+$ synapse_auto_compressor -p postgresql://user:pass@localhost/synapse -c 500 -n 100
 ```
 ## Running Options
 
@@ -199,10 +197,10 @@ $ docker-compose down
 # Using the synapse_compress_state library
 
 If you want to use the compressor in another project, it is recomended that you
-use jemalloc `https://github.com/gnzlbg/jemallocator`. 
+use jemalloc `https://github.com/tikv/jemallocator`.
 
 To prevent the progress bars from being shown, use the `no-progress-bars` feature.
-(See `auto_compressor/Cargo.toml` for an example)
+(See `synapse_auto_compressor/Cargo.toml` for an example)
 
 # Troubleshooting
 
@@ -228,17 +226,17 @@ setting in [`postgresql.conf`](https://www.postgresql.org/docs/current/runtime-c
 The amount of output the tools produce can be altered by setting the RUST_LOG
 environment variable to something.
 
-To get more logs when running the auto_compressor tool try the following:
+To get more logs when running the synapse_auto_compressor tool try the following:
 
 ```
-$ RUST_LOG=debug auto_compressor -p postgresql://user:pass@localhost/synapse -c 50 -n 100
+$ RUST_LOG=debug synapse_auto_compressor -p postgresql://user:pass@localhost/synapse -c 50 -n 100
 ```
 
 If you want to suppress all the debugging info you are getting from the
 Postgres client then try:
 
 ```
-RUST_LOG=auto_compressor=debug,synapse_compress_state=debug auto_compressor [etc.]
+RUST_LOG=synapse_auto_compressor=debug,synapse_compress_state=debug synapse_auto_compressor [etc.]
 ```
 
 This will only print the debugging information from those two packages. For more info see
@@ -268,7 +266,7 @@ be a large problem.
 
 ## Compressor is trying to increase the number of rows
 
-Backfilling can lead to issues with compression. The auto_compressor will
+Backfilling can lead to issues with compression. The synapse_auto_compressor will
 skip chunks it can't reduce the size of and so this should help jump over the backfilled
 state_groups. Lots of state resolution might also impact the ability to use the compressor.
 

compressor_integration_tests/Cargo.toml

@@ -13,7 +13,7 @@ postgres = "0.19.0"
 postgres-openssl = "0.5.0"
 rand = "0.8.0"
 synapse_compress_state = { path = "../", features = ["no-progress-bars"] }
-auto_compressor = { path = "../auto_compressor/" }
+synapse_auto_compressor = { path = "../synapse_auto_compressor/" }
 env_logger = "0.9.0"
 log = "0.4.14"
 

compressor_integration_tests/src/lib.rs

@@ -356,7 +356,7 @@ fn functions_are_self_consistent() {
 }
 
 pub fn setup_logger() {
-    // setup the logger for the auto_compressor
+    // setup the logger for the synapse_auto_compressor
     // The default can be overwritten with RUST_LOG
     // see the README for more information
     if env::var("RUST_LOG").is_err() {
@@ -366,7 +366,7 @@ pub fn setup_logger() {
         // default to printing the debug information for both packages being tested
         // (Note that just setting the global level to debug will log every sql transaction)
         log_builder.filter_module("synapse_compress_state", LevelFilter::Debug);
-        log_builder.filter_module("auto_compressor", LevelFilter::Debug);
+        log_builder.filter_module("synapse_auto_compressor", LevelFilter::Debug);
         // use try_init() incase the logger has been setup by some previous test
         let _ = log_builder.try_init();
     } else {

compressor_integration_tests/tests/auto_compressor_manager_tests.rs

@@ -1,9 +1,5 @@
 use std::collections::BTreeMap;
 
-use auto_compressor::{
-    manager::{compress_chunks_of_database, run_compressor_on_room_chunk},
-    state_saving::{connect_to_database, create_tables_if_needed},
-};
 use compressor_integration_tests::{
     add_contents_to_database, clear_compressor_state, database_collapsed_states_match_map,
     database_structure_matches_map, empty_database,
@@ -14,6 +10,10 @@ use compressor_integration_tests::{
     setup_logger, DB_URL,
 };
 use serial_test::serial;
+use synapse_auto_compressor::{
+    manager::{compress_chunks_of_database, run_compressor_on_room_chunk},
+    state_saving::{connect_to_database, create_tables_if_needed},
+};
 use synapse_compress_state::Level;
 
 #[test]

compressor_integration_tests/tests/auto_compressor_state_saving_tests.rs

@@ -1,9 +1,9 @@
-use auto_compressor::state_saving::{
+use compressor_integration_tests::{clear_compressor_state, setup_logger, DB_URL};
+use serial_test::serial;
+use synapse_auto_compressor::state_saving::{
     connect_to_database, create_tables_if_needed, read_room_compressor_state,
     write_room_compressor_state,
 };
-use compressor_integration_tests::{clear_compressor_state, setup_logger, DB_URL};
-use serial_test::serial;
 use synapse_compress_state::Level;
 
 #[test]

compressor_integration_tests/tests/compressor_config_tests.rs

@@ -46,10 +46,11 @@ fn run_succeeds_without_crashing() {
     let transactions = true;
     let graphs = false;
     let commit_changes = false;
+    let verify = true;
 
     let config = Config::new(
-        db_url.clone(),
+        db_url,
-        room_id.clone(),
+        room_id,
         output_file,
         min_state_group,
         groups_to_compress,
@@ -59,6 +60,7 @@ fn run_succeeds_without_crashing() {
         transactions,
         graphs,
         commit_changes,
+        verify,
     )
     .unwrap();
 
@@ -94,6 +96,7 @@ fn changes_commited_if_no_min_saved_rows() {
     let transactions = true;
     let graphs = false;
     let commit_changes = true;
+    let verify = true;
 
     let config = Config::new(
         db_url,
@@ -107,6 +110,7 @@ fn changes_commited_if_no_min_saved_rows() {
         transactions,
         graphs,
         commit_changes,
+        verify,
     )
     .unwrap();
 
@@ -160,6 +164,7 @@ fn changes_commited_if_min_saved_rows_exceeded() {
     let transactions = true;
     let graphs = false;
     let commit_changes = true;
+    let verify = true;
 
     let config = Config::new(
         db_url,
@@ -173,6 +178,7 @@ fn changes_commited_if_min_saved_rows_exceeded() {
         transactions,
         graphs,
         commit_changes,
+        verify,
     )
     .unwrap();
 
@@ -227,6 +233,7 @@ fn changes_not_commited_if_fewer_than_min_saved_rows() {
     let transactions = true;
     let graphs = false;
     let commit_changes = true;
+    let verify = true;
 
     let config = Config::new(
         db_url,
@@ -240,6 +247,7 @@ fn changes_not_commited_if_fewer_than_min_saved_rows() {
         transactions,
         graphs,
         commit_changes,
+        verify,
     )
     .unwrap();
 
@@ -280,6 +288,7 @@ fn run_panics_if_invalid_db_url() {
     let transactions = true;
     let graphs = false;
     let commit_changes = true;
+    let verify = true;
 
     let config = Config::new(
         db_url,
@@ -293,6 +302,7 @@ fn run_panics_if_invalid_db_url() {
         transactions,
         graphs,
         commit_changes,
+        verify,
     )
     .unwrap();
 
@@ -336,6 +346,7 @@ fn run_only_affects_given_room_id() {
     let transactions = true;
     let graphs = false;
     let commit_changes = true;
+    let verify = true;
 
     let config = Config::new(
         db_url,
@@ -349,6 +360,7 @@ fn run_only_affects_given_room_id() {
         transactions,
         graphs,
         commit_changes,
+        verify,
     )
     .unwrap();
 
@@ -406,6 +418,7 @@ fn run_respects_groups_to_compress() {
     let transactions = true;
     let graphs = false;
     let commit_changes = true;
+    let verify = true;
 
     let config = Config::new(
         db_url,
@@ -419,6 +432,7 @@ fn run_respects_groups_to_compress() {
         transactions,
         graphs,
         commit_changes,
+        verify,
     )
     .unwrap();
 
@@ -492,6 +506,7 @@ fn run_is_idempotent_when_run_on_whole_room() {
     let transactions = true;
     let graphs = false;
     let commit_changes = true;
+    let verify = true;
 
     let config1 = Config::new(
         db_url.clone(),
@@ -505,21 +520,23 @@ fn run_is_idempotent_when_run_on_whole_room() {
         transactions,
         graphs,
         commit_changes,
+        verify,
     )
     .unwrap();
 
     let config2 = Config::new(
-        db_url.clone(),
+        db_url,
-        room_id.clone(),
+        room_id,
         output_file2,
         min_state_group,
         groups_to_compress,
         min_saved_rows,
         max_state_group,
-        level_sizes.clone(),
+        level_sizes,
         transactions,
         graphs,
         commit_changes,
+        verify,
     )
     .unwrap();
 

compressor_integration_tests/tests/compressor_continue_run_tests.rs

@@ -56,7 +56,7 @@ fn continue_run_called_twice_same_as_run() {
 
     let start = Some(6);
     let chunk_size = 7;
-    let level_info = chunk_stats_1.new_level_info.clone();
+    let level_info = chunk_stats_1.new_level_info;
 
     // Run the compressor with those settings
     let chunk_stats_2 = continue_run(start, chunk_size, &db_url, &room_id, &level_info).unwrap();

docs/python.md

@@ -18,7 +18,7 @@ the compressor is run.
 
 3. Navigate to the correct location  
 For the automatic tool:  
-`$ cd /home/synapse/rust-synapse-compress-state/auto_compressor`   
+`$ cd /home/synapse/rust-synapse-compress-state/synpase_auto_compressor`   
 For the manual tool:  
 `$ cd /home/synapse/rust-synapse-compress-state`   
 
@@ -30,9 +30,9 @@ This will install the relevant compressor tool into the activated virtual enviro
 ## Automatic tool example:
 
 ```python
-import auto_compressor
+import synapse_auto_compressor
 
-auto_compressor.compress_state_events_table(
+synapse_auto_compressor.compress_state_events_table(
   db_url="postgresql://localhost/synapse",
   chunk_size=500,
   default_levels="100,50,25",

src/compressor.rs

@@ -181,12 +181,11 @@ impl<'a> Compressor<'a> {
             panic!("Can only call `create_new_tree` once");
         }
 
-        let pb: ProgressBar;
+        let pb: ProgressBar = if cfg!(feature = "no-progress-bars") {
-        if cfg!(feature = "no-progress-bars") {
+            ProgressBar::hidden()
-            pb = ProgressBar::hidden();
         } else {
-            pb = ProgressBar::new(self.original_state_map.len() as u64);
+            ProgressBar::new(self.original_state_map.len() as u64)
-        }
+        };
         pb.set_style(
             ProgressStyle::default_bar().template("[{elapsed_precise}] {bar} {pos}/{len} {msg}"),
         );

src/compressor/compressor_tests.rs

@@ -93,14 +93,7 @@ fn create_new_tree_does_nothing_if_already_compressed() {
 
     for i in 0i64..=13i64 {
         // edge from map
-        let pred_group = initial_edges.get(&i);
+        let prev = initial_edges.get(&i).copied();
-
-        // Need Option<i64> not Option<&i64>
-        let prev;
-        match pred_group {
-            Some(i) => prev = Some(*i),
-            None => prev = None,
-        }
 
         // insert that edge into the initial map
         initial.insert(

src/compressor/level_tests.rs

@@ -54,7 +54,7 @@ fn get_head_returns_head() {
 #[test]
 fn has_space_returns_true_if_empty() {
     let l = Level::new(15);
-    assert_eq!(l.has_space(), true);
+    assert!(l.has_space());
 }
 
 #[test]
@@ -65,7 +65,7 @@ fn has_space_returns_true_if_part_full() {
     l.update(1, true);
     l.update(143, true);
     l.update(15, true);
-    assert_eq!(l.has_space(), true);
+    assert!(l.has_space());
 }
 
 #[test]
@@ -76,5 +76,5 @@ fn has_space_returns_false_if_full() {
     l.update(3, true);
     l.update(4, true);
     l.update(5, true);
-    assert_eq!(l.has_space(), false);
+    assert!(!l.has_space());
 }

src/compressor/stats_tests.rs

@@ -142,14 +142,7 @@ fn stats_correct_if_no_changes() {
 
     for i in 0i64..=13i64 {
         // edge from map
-        let pred_group = initial_edges.get(&i);
+        let prev = initial_edges.get(&i).copied();
-
-        // Need Option<i64> not Option<&i64>
-        let prev;
-        match pred_group {
-            Some(i) => prev = Some(*i),
-            None => prev = None,
-        }
 
         // insert that edge into the initial map
         initial.insert(

src/database.rs

@@ -372,12 +372,11 @@ fn get_initial_data_from_db(
     // Copy the data from the database into a map
     let mut state_group_map: BTreeMap<i64, StateGroupEntry> = BTreeMap::new();
 
-    let pb: ProgressBar;
+    let pb: ProgressBar = if cfg!(feature = "no-progress-bars") {
-    if cfg!(feature = "no-progress-bars") {
+        ProgressBar::hidden()
-        pb = ProgressBar::hidden();
     } else {
-        pb = ProgressBar::new_spinner();
+        ProgressBar::new_spinner()
-    }
+    };
     pb.set_style(
         ProgressStyle::default_spinner().template("{spinner} [{elapsed}] {pos} rows retrieved"),
     );
@@ -537,12 +536,11 @@ pub fn send_changes_to_db(
     debug!("Writing changes...");
 
     // setup the progress bar
-    let pb: ProgressBar;
+    let pb: ProgressBar = if cfg!(feature = "no-progress-bars") {
-    if cfg!(feature = "no-progress-bars") {
+        ProgressBar::hidden()
-        pb = ProgressBar::hidden();
     } else {
-        pb = ProgressBar::new(old_map.len() as u64);
+        ProgressBar::new(old_map.len() as u64)
-    }
+    };
     pb.set_style(
         ProgressStyle::default_bar().template("[{elapsed_precise}] {bar} {pos}/{len} {msg}"),
     );

src/lib.rs

@@ -27,7 +27,7 @@ use clap::{crate_authors, crate_description, crate_name, crate_version, value_t,
 use indicatif::{ProgressBar, ProgressStyle};
 use rayon::prelude::*;
 use state_map::StateMap;
-use std::{collections::BTreeMap, fs::File, io::Write, str::FromStr};
+use std::{collections::BTreeMap, convert::TryInto, fs::File, io::Write, str::FromStr};
 use string_cache::DefaultAtom as Atom;
 
 mod compressor;
@@ -109,6 +109,9 @@ pub struct Config {
     // Whether or not to commit changes to the database automatically
     // N.B. currently assumes transactions is true (to be on the safe side)
     commit_changes: bool,
+    // Whether to verify the correctness of the compressed state groups by
+    // comparing them to the original groups
+    verify: bool,
 }
 
 impl Config {
@@ -223,6 +226,13 @@ impl Config {
                 .long_help(concat!("If this flag is set then the changes the compressor makes will",
                     " be committed to the database. This should be safe to use while synapse is running",
                     " as it assumes by default that the transactions flag is set")),
+        ).arg(
+            Arg::with_name("no_verify")
+                .short("N")
+                .help("Do not double-check that the compression was performed correctly")
+                .long_help(concat!("If this flag is set then the verification of the compressed",
+                    " state groups, which compares them to the original groups, is skipped. This",
+                    " saves time at the cost of potentially generating mismatched state.")),
         ).get_matches();
 
         let db_url = matches
@@ -262,6 +272,8 @@ impl Config {
 
         let commit_changes = matches.is_present("commit_changes");
 
+        let verify = !matches.is_present("no_verify");
+
         Config {
             db_url: String::from(db_url),
             output_file,
@@ -274,6 +286,7 @@ impl Config {
             transactions,
             graphs,
             commit_changes,
+            verify,
         }
     }
 }
@@ -362,8 +375,8 @@ pub fn run(mut config: Config) {
     }
 
     if let Some(min) = config.min_saved_rows {
-        let saving = (original_summed_size - compressed_summed_size) as i32;
+        let saving = original_summed_size.saturating_sub(compressed_summed_size);
-        if saving < min {
+        if saving < min.try_into().unwrap_or(0) {
             warn!(
                 "Only {} rows would be saved by this compression. Skipping output.",
                 saving
@@ -372,7 +385,9 @@ pub fn run(mut config: Config) {
         }
     }
 
-    check_that_maps_match(&state_group_map, new_state_group_map);
+    if config.verify {
+        check_that_maps_match(&state_group_map, new_state_group_map);
+    }
 
     // If we are given an output file, we output the changes as SQL. If the
     // `transactions` argument is set we wrap each change to a state group in a
@@ -492,12 +507,11 @@ fn output_sql(
 
     info!("Writing changes...");
 
-    let pb: ProgressBar;
+    let pb: ProgressBar = if cfg!(feature = "no-progress-bars") {
-    if cfg!(feature = "no-progress-bars") {
+        ProgressBar::hidden()
-        pb = ProgressBar::hidden();
     } else {
-        pb = ProgressBar::new(old_map.len() as u64);
+        ProgressBar::new(old_map.len() as u64)
-    }
+    };
     pb.set_style(
         ProgressStyle::default_bar().template("[{elapsed_precise}] {bar} {pos}/{len} {msg}"),
     );
@@ -607,12 +621,11 @@ fn check_that_maps_match(
 ) {
     info!("Checking that state maps match...");
 
-    let pb: ProgressBar;
+    let pb: ProgressBar = if cfg!(feature = "no-progress-bars") {
-    if cfg!(feature = "no-progress-bars") {
+        ProgressBar::hidden()
-        pb = ProgressBar::hidden();
     } else {
-        pb = ProgressBar::new(old_map.len() as u64);
+        ProgressBar::new(old_map.len() as u64)
-    }
+    };
     pb.set_style(
         ProgressStyle::default_bar().template("[{elapsed_precise}] {bar} {pos}/{len} {msg}"),
     );
@@ -695,6 +708,7 @@ impl Config {
         transactions: bool,
         graphs: bool,
         commit_changes: bool,
+        verify: bool,
     ) -> Result<Config, String> {
         let mut output: Option<File> = None;
         if let Some(file) = output_file {
@@ -722,6 +736,7 @@ impl Config {
             transactions,
             graphs,
             commit_changes,
+            verify,
         })
     }
 }
@@ -746,6 +761,7 @@ impl Config {
     transactions = true,
     graphs = false,
     commit_changes = false,
+    verify = true,
 )]
 fn run_compression(
     db_url: String,
@@ -759,6 +775,7 @@ fn run_compression(
     transactions: bool,
     graphs: bool,
     commit_changes: bool,
+    verify: bool,
 ) -> PyResult<()> {
     let config = Config::new(
         db_url,
@@ -772,6 +789,7 @@ fn run_compression(
         transactions,
         graphs,
         commit_changes,
+        verify,
     );
     match config {
         Err(e) => Err(PyErr::new::<exceptions::PyException, _>(e)),
@@ -788,7 +806,7 @@ fn synapse_compress_state(_py: Python, m: &PyModule) -> PyResult<()> {
     let _ = pyo3_log::Logger::default()
         // don't send out anything lower than a warning from other crates
         .filter(LevelFilter::Warn)
-        // don't log warnings from synapse_compress_state, the auto_compressor handles these
+        // don't log warnings from synapse_compress_state, the synapse_auto_compressor handles these
         // situations and provides better log messages
         .filter_target("synapse_compress_state".to_owned(), LevelFilter::Debug)
         .install();
@@ -955,7 +973,6 @@ mod lib_tests {
     #[test]
     fn check_that_maps_match_returns_if_both_empty() {
         check_that_maps_match(&BTreeMap::new(), &BTreeMap::new());
-        assert!(true);
     }
 
     #[test]
@@ -988,7 +1005,6 @@ mod lib_tests {
         }
 
         check_that_maps_match(&old_map, &BTreeMap::new());
-        assert!(true);
     }
 
     #[test]
@@ -1024,7 +1040,6 @@ mod lib_tests {
         }
 
         check_that_maps_match(&BTreeMap::new(), &new_map);
-        assert!(true);
     }
 
     #[test]
@@ -1056,7 +1071,6 @@ mod lib_tests {
         }
 
         check_that_maps_match(&BTreeMap::new(), &old_map.clone());
-        assert!(true);
     }
 
     #[test]
@@ -1119,7 +1133,6 @@ mod lib_tests {
         }
 
         check_that_maps_match(&old_map, &new_map);
-        assert!(true);
     }
 
     #[test]
@@ -1201,7 +1214,6 @@ mod lib_tests {
         );
 
         check_that_maps_match(&old_map, &new_map);
-        assert!(true);
     }
 
     //TODO: tests for correct SQL code produced by output_sql
@@ -1224,6 +1236,7 @@ mod pyo3_tests {
         let transactions = false;
         let graphs = false;
         let commit_changes = false;
+        let verify = true;
 
         let config = Config::new(
             db_url.clone(),
@@ -1237,6 +1250,7 @@ mod pyo3_tests {
             transactions,
             graphs,
             commit_changes,
+            verify,
         )
         .unwrap();
 
@@ -1270,6 +1284,7 @@ mod pyo3_tests {
         let transactions = true;
         let graphs = true;
         let commit_changes = true;
+        let verify = true;
 
         let config = Config::new(
             db_url.clone(),
@@ -1283,11 +1298,12 @@ mod pyo3_tests {
             transactions,
             graphs,
             commit_changes,
+            verify,
         )
         .unwrap();
 
         assert_eq!(config.db_url, db_url);
-        assert!(!config.output_file.is_none());
+        assert!(config.output_file.is_some());
         assert_eq!(config.room_id, room_id);
         assert_eq!(config.min_state_group, Some(3225));
         assert_eq!(config.groups_to_compress, Some(970));

src/main.rs

@@ -18,7 +18,7 @@
 
 #[cfg(feature = "jemalloc")]
 #[global_allocator]
-static GLOBAL: jemallocator::Jemalloc = jemallocator::Jemalloc;
+static GLOBAL: tikv_jemallocator::Jemalloc = tikv_jemallocator::Jemalloc;
 
 use log::LevelFilter;
 use std::env;

synapse_auto_compressor/Cargo.toml

@@ -1,21 +1,27 @@
 [package]
-name = "auto_compressor"
+name = "synapse_auto_compressor"
 authors = ["William Ashton"]
-version = "0.1.0"
+version = "0.1.2"
 edition = "2018"
 
-# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+[package.metadata.maturin]
+requires-python = ">=3.6"
+project-url = {Source = "https://github.com/matrix-org/rust-synapse-compress-state"}
+classifier = [
+    "Development Status :: 4 - Beta",
+    "Programming Language :: Rust",
+]
 
 [dependencies]
 clap = "2.33.0"
 openssl = "0.10.32"
 postgres = "0.19.0"
 postgres-openssl = "0.5.0"
-jemallocator = "0.3.2"
+tikv-jemallocator = "0.4.1"
 rand = "0.8.0"
 serial_test = "0.5.1"
 synapse_compress_state = { path = "../", features = ["no-progress-bars"] }
-env_logger = { version = "0.9.0", git = "https://github.com/TilCreator/env_logger", branch = "fix_pipe" }
+env_logger = "0.9.0"
 log = "0.4.14"
 log-panics = "2.0.0"
 anyhow = "1.0.42"

synapse_auto_compressor/README.md

@@ -0,0 +1,12 @@
+# Auto Compressor
+
+See the top level readme for information.
+
+
+## Publishing to PyPI
+
+Bump the version number and run from the root directory of the repo:
+
+```
+docker run -it --rm -v $(pwd):/io -e OPENSSL_STATIC=1 konstin2/maturin publish -m synapse_auto_compressor/Cargo.toml --cargo-extra-args "\--features='openssl/vendored'"
+```

synapse_auto_compressor/src/lib.rs

@@ -57,15 +57,16 @@ impl FromStr for LevelInfo {
 
 // PyO3 INTERFACE STARTS HERE
 #[pymodule]
-fn auto_compressor(_py: Python, m: &PyModule) -> PyResult<()> {
+fn synapse_auto_compressor(_py: Python, m: &PyModule) -> PyResult<()> {
     let _ = pyo3_log::Logger::default()
         // don't send out anything lower than a warning from other crates
         .filter(LevelFilter::Warn)
-        // don't log warnings from synapse_compress_state, the auto_compressor handles these
+        // don't log warnings from synapse_compress_state, the
-        // situations and provides better log messages
+        // synapse_auto_compressor handles these situations and provides better
+        // log messages
         .filter_target("synapse_compress_state".to_owned(), LevelFilter::Error)
-        // log info and above for the auto_compressor
+        // log info and above for the synapse_auto_compressor
-        .filter_target("auto_compressor".to_owned(), LevelFilter::Debug)
+        .filter_target("synapse_auto_compressor".to_owned(), LevelFilter::Debug)
         .install();
     // ensure any panics produce error messages in the log
     log_panics::init();
@@ -92,7 +93,7 @@ fn auto_compressor(_py: Python, m: &PyModule) -> PyResult<()> {
         number_of_chunks: i64,
     ) -> PyResult<()> {
         // Announce the start of the program to the logs
-        log::info!("auto_compressor started");
+        log::info!("synapse_auto_compressor started");
 
         // Parse the default_level string into a LevelInfo struct
         let default_levels: LevelInfo = match default_levels.parse() {
@@ -120,7 +121,7 @@ fn auto_compressor(_py: Python, m: &PyModule) -> PyResult<()> {
             return Err(PyErr::new::<PyRuntimeError, _>(format!("{:?}", e)));
         }
 
-        log::info!("auto_compressor finished");
+        log::info!("synapse_auto_compressor finished");
         Ok(())
     }
     Ok(())

synapse_auto_compressor/src/main.rs

@@ -17,45 +17,37 @@
 //! continue from where it left off.
 
 #[global_allocator]
-static GLOBAL: jemallocator::Jemalloc = jemallocator::Jemalloc;
+static GLOBAL: tikv_jemallocator::Jemalloc = tikv_jemallocator::Jemalloc;
 
-use auto_compressor::{manager, state_saving, LevelInfo};
 use clap::{crate_authors, crate_description, crate_name, crate_version, value_t, App, Arg};
 use log::LevelFilter;
-use std::{env, fs::OpenOptions};
+use std::env;
+use synapse_auto_compressor::{manager, state_saving, LevelInfo};
 
 /// Execution starts here
 fn main() {
-    // setup the logger for the auto_compressor
+    // setup the logger for the synapse_auto_compressor
     // The default can be overwritten with RUST_LOG
     // see the README for more information
-    let log_file = OpenOptions::new()
-        .append(true)
-        .create(true)
-        .open("auto_compressor.log")
-        .unwrap_or_else(|e| panic!("Error occured while opening the log file: {}", e));
-
     if env::var("RUST_LOG").is_err() {
         let mut log_builder = env_logger::builder();
-        log_builder.target(env_logger::Target::Pipe(Box::new(log_file)));
         // Ensure panics still come through
         log_builder.filter_module("panic", LevelFilter::Error);
         // Only output errors from the synapse_compress state library
         log_builder.filter_module("synapse_compress_state", LevelFilter::Error);
-        // Output log levels info and above from auto_compressor
+        // Output log levels info and above from synapse_auto_compressor
-        log_builder.filter_module("auto_compressor", LevelFilter::Info);
+        log_builder.filter_module("synapse_auto_compressor", LevelFilter::Info);
         log_builder.init();
     } else {
         // If RUST_LOG was set then use that
         let mut log_builder = env_logger::Builder::from_env("RUST_LOG");
-        log_builder.target(env_logger::Target::Pipe(Box::new(log_file)));
         // Ensure panics still come through
         log_builder.filter_module("panic", LevelFilter::Error);
         log_builder.init();
     }
     log_panics::init();
     // Announce the start of the program to the logs
-    log::info!("auto_compressor started");
+    log::info!("synapse_auto_compressor started");
 
     // parse the command line arguments using the clap crate
     let arguments = App::new(crate_name!())
@@ -156,5 +148,5 @@ fn main() {
     manager::compress_chunks_of_database(db_url, chunk_size, &default_levels.0, number_of_chunks)
         .unwrap();
 
-    log::info!("auto_compressor finished");
+    log::info!("synapse_auto_compressor finished");
 }