From 08d312b66d81498b818109434790c45bb5a8da63 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 28 Sep 2021 16:05:38 +0100 Subject: [PATCH] Move synapse_auto_compressor --- Cargo.lock | 42 +++++++++---------- Cargo.toml | 2 +- README.md | 23 +++++----- compressor_integration_tests/Cargo.toml | 4 +- .../auto_compressor_state_saving_tests.rs | 6 +-- docs/python.md | 8 ++-- src/lib.rs | 2 +- synapse_auto_compressor/Cargo.toml | 2 +- synapse_auto_compressor/src/main.rs | 2 +- 9 files changed, 46 insertions(+), 45 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 0f55e8d..a39b9b8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -54,26 +54,6 @@ dependencies = [ "winapi", ] -[[package]] -name = "auto_compressor" -version = "0.1.0" -dependencies = [ - "anyhow", - "clap", - "env_logger", - "jemallocator", - "log", - "log-panics", - "openssl", - "postgres", - "postgres-openssl", - "pyo3", - "pyo3-log", - "rand", - "serial_test", - "synapse_compress_state", -] - [[package]] name = "autocfg" version = "1.0.1" @@ -144,7 +124,6 @@ dependencies = [ name = "compressor_integration_tests" version = "0.1.0" dependencies = [ - "auto_compressor", "env_logger", "log", "openssl", @@ -154,6 +133,7 @@ dependencies = [ "serial_test", "state-map", "string_cache", + "synapse_auto_compressor", "synapse_compress_state", ] @@ -1122,6 +1102,26 @@ dependencies = [ "unicode-xid", ] +[[package]] +name = "synapse_auto_compressor" +version = "0.1.0" +dependencies = [ + "anyhow", + "clap", + "env_logger", + "jemallocator", + "log", + "log-panics", + "openssl", + "postgres", + "postgres-openssl", + "pyo3", + "pyo3-log", + "rand", + "serial_test", + "synapse_compress_state", +] + [[package]] name = "synapse_compress_state" version = "0.1.0" diff --git a/Cargo.toml b/Cargo.toml index bfa9ec7..9c867c8 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,5 +1,5 @@ [workspace] -members = ["auto_compressor", "compressor_integration_tests"] +members = ["synapse_auto_compressor", "compressor_integration_tests"] [package] authors = ["Erik Johnston"] diff --git a/README.md b/README.md index 60995dd..d55a7ef 100644 --- a/README.md +++ b/README.md @@ -3,7 +3,7 @@ This workspace contains experimental tools that attempt to reduce the number of rows in the `state_groups_state` table inside of a Synapse Postgresql database. -# Automated tool: auto_compressor +# Automated tool: synapse_auto_compressor ## Introduction: @@ -11,7 +11,7 @@ This tool is significantly more simple to use than the manual tool (described be It scans through all of the rows in the `state_groups` database table from the start. When it finds a group that hasn't been compressed, it runs the compressor for a while on that group's room, saving where it got up to. After compressing a number of these chunks it stops, -saving where it got up to for the next run of the `auto_compressor`. +saving where it got up to for the next run of the `synapse_auto_compressor`. It creates three extra tables in the database: `state_compressor_state` which stores the information needed to stop and start the compressor for each room, `state_compressor_progress` @@ -26,14 +26,15 @@ periodically. This tool requires `cargo` to be installed. See https://www.rust-lang.org/tools/install for instructions on how to do this. -To build `auto_compressor`, clone this repository and navigate to the `autocompressor/` -subdirectory. Then execute `cargo build`. +To build `synapse_auto_compressor`, clone this repository and navigate to the +`synapse_auto_compressor/` subdirectory. Then execute `cargo build`. -This will create an executable and store it in `auto_compressor/target/debug/auto_compressor`. +This will create an executable and store it in +`synapse_auto_compressor/target/debug/auto_compressor`. ## Example usage ``` -$ auto_compressor -p postgresql://user:pass@localhost/synapse -c 500 -n 100 +$ synapse_auto_compressor -p postgresql://user:pass@localhost/synapse -c 500 -n 100 ``` ## Running Options @@ -199,7 +200,7 @@ If you want to use the compressor in another project, it is recomended that you use jemalloc `https://github.com/gnzlbg/jemallocator`. To prevent the progress bars from being shown, use the `no-progress-bars` feature. -(See `auto_compressor/Cargo.toml` for an example) +(See `synapse_auto_compressor/Cargo.toml` for an example) # Troubleshooting @@ -225,17 +226,17 @@ setting in [`postgresql.conf`](https://www.postgresql.org/docs/current/runtime-c The amount of output the tools produce can be altered by setting the RUST_LOG environment variable to something. -To get more logs when running the auto_compressor tool try the following: +To get more logs when running the synapse_auto_compressor tool try the following: ``` -$ RUST_LOG=debug auto_compressor -p postgresql://user:pass@localhost/synapse -c 50 -n 100 +$ RUST_LOG=debug synapse_auto_compressor -p postgresql://user:pass@localhost/synapse -c 50 -n 100 ``` If you want to suppress all the debugging info you are getting from the Postgres client then try: ``` -RUST_LOG=auto_compressor=debug,synapse_compress_state=debug auto_compressor [etc.] +RUST_LOG=synapse_auto_compressor=debug,synapse_compress_state=debug synapse_auto_compressor [etc.] ``` This will only print the debugging information from those two packages. For more info see @@ -265,7 +266,7 @@ be a large problem. ## Compressor is trying to increase the number of rows -Backfilling can lead to issues with compression. The auto_compressor will +Backfilling can lead to issues with compression. The synapse_auto_compressor will skip chunks it can't reduce the size of and so this should help jump over the backfilled state_groups. Lots of state resolution might also impact the ability to use the compressor. diff --git a/compressor_integration_tests/Cargo.toml b/compressor_integration_tests/Cargo.toml index a03fe2a..e3142a0 100644 --- a/compressor_integration_tests/Cargo.toml +++ b/compressor_integration_tests/Cargo.toml @@ -13,9 +13,9 @@ postgres = "0.19.0" postgres-openssl = "0.5.0" rand = "0.8.0" synapse_compress_state = { path = "../", features = ["no-progress-bars"] } -auto_compressor = { path = "../auto_compressor/" } +synapse_auto_compressor = { path = "../synapse_auto_compressor/" } env_logger = "0.9.0" log = "0.4.14" [dependencies.state-map] -git = "https://github.com/matrix-org/rust-matrix-state-map" \ No newline at end of file +git = "https://github.com/matrix-org/rust-matrix-state-map" diff --git a/compressor_integration_tests/tests/auto_compressor_state_saving_tests.rs b/compressor_integration_tests/tests/auto_compressor_state_saving_tests.rs index 849fcde..a854e7b 100644 --- a/compressor_integration_tests/tests/auto_compressor_state_saving_tests.rs +++ b/compressor_integration_tests/tests/auto_compressor_state_saving_tests.rs @@ -1,9 +1,9 @@ -use auto_compressor::state_saving::{ +use compressor_integration_tests::{clear_compressor_state, setup_logger, DB_URL}; +use serial_test::serial; +use synapse_auto_compressor::state_saving::{ connect_to_database, create_tables_if_needed, read_room_compressor_state, write_room_compressor_state, }; -use compressor_integration_tests::{clear_compressor_state, setup_logger, DB_URL}; -use serial_test::serial; use synapse_compress_state::Level; #[test] diff --git a/docs/python.md b/docs/python.md index 3f7cd26..11ff955 100644 --- a/docs/python.md +++ b/docs/python.md @@ -18,7 +18,7 @@ the compressor is run. 3. Navigate to the correct location For the automatic tool: -`$ cd /home/synapse/rust-synapse-compress-state/auto_compressor` +`$ cd /home/synapse/rust-synapse-compress-state/synpase_auto_compressor` For the manual tool: `$ cd /home/synapse/rust-synapse-compress-state` @@ -30,9 +30,9 @@ This will install the relevant compressor tool into the activated virtual enviro ## Automatic tool example: ```python -import auto_compressor +import synapse_auto_compressor -auto_compressor.compress_state_events_table( +synapse_auto_compressor.compress_state_events_table( db_url="postgresql://localhost/synapse", chunk_size=500, default_levels="100,50,25", @@ -51,4 +51,4 @@ synapse_compress_state.run_compression( output_file="out.sql", transactions=True ) -``` \ No newline at end of file +``` diff --git a/src/lib.rs b/src/lib.rs index 0433619..da11af8 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -788,7 +788,7 @@ fn synapse_compress_state(_py: Python, m: &PyModule) -> PyResult<()> { let _ = pyo3_log::Logger::default() // don't send out anything lower than a warning from other crates .filter(LevelFilter::Warn) - // don't log warnings from synapse_compress_state, the auto_compressor handles these + // don't log warnings from synapse_compress_state, the synapse_auto_compressor handles these // situations and provides better log messages .filter_target("synapse_compress_state".to_owned(), LevelFilter::Debug) .install(); diff --git a/synapse_auto_compressor/Cargo.toml b/synapse_auto_compressor/Cargo.toml index 1f3a031..4839809 100644 --- a/synapse_auto_compressor/Cargo.toml +++ b/synapse_auto_compressor/Cargo.toml @@ -1,5 +1,5 @@ [package] -name = "auto_compressor" +name = "synapse_auto_compressor" authors = ["William Ashton"] version = "0.1.0" edition = "2018" diff --git a/synapse_auto_compressor/src/main.rs b/synapse_auto_compressor/src/main.rs index 4cb06e2..34a19b2 100644 --- a/synapse_auto_compressor/src/main.rs +++ b/synapse_auto_compressor/src/main.rs @@ -19,10 +19,10 @@ #[global_allocator] static GLOBAL: jemallocator::Jemalloc = jemallocator::Jemalloc; -use auto_compressor::{manager, state_saving, LevelInfo}; use clap::{crate_authors, crate_description, crate_name, crate_version, value_t, App, Arg}; use log::LevelFilter; use std::{env, fs::OpenOptions}; +use synapse_auto_compressor::{manager, state_saving, LevelInfo}; /// Execution starts here fn main() {