Make the auto compressor uploadable to pypi (#75)

Cargo.lock

@@ -54,26 +54,6 @@ dependencies = [
  "winapi",
 ]
 
-[[package]]
-name = "auto_compressor"
-version = "0.1.0"
-dependencies = [
- "anyhow",
- "clap",
- "env_logger",
- "jemallocator",
- "log",
- "log-panics",
- "openssl",
- "postgres",
- "postgres-openssl",
- "pyo3",
- "pyo3-log",
- "rand",
- "serial_test",
- "synapse_compress_state",
-]
-
 [[package]]
 name = "autocfg"
 version = "1.0.1"
@@ -144,7 +124,6 @@ dependencies = [
 name = "compressor_integration_tests"
 version = "0.1.0"
 dependencies = [
- "auto_compressor",
  "env_logger",
  "log",
  "openssl",
@@ -154,6 +133,7 @@ dependencies = [
  "serial_test",
  "state-map",
  "string_cache",
+ "synapse_auto_compressor",
  "synapse_compress_state",
 ]
 
@@ -1122,6 +1102,26 @@ dependencies = [
  "unicode-xid",
 ]
 
+[[package]]
+name = "synapse_auto_compressor"
+version = "0.1.1"
+dependencies = [
+ "anyhow",
+ "clap",
+ "env_logger",
+ "jemallocator",
+ "log",
+ "log-panics",
+ "openssl",
+ "postgres",
+ "postgres-openssl",
+ "pyo3",
+ "pyo3-log",
+ "rand",
+ "serial_test",
+ "synapse_compress_state",
+]
+
 [[package]]
 name = "synapse_compress_state"
 version = "0.1.0"

Cargo.toml

@@ -1,5 +1,5 @@
 [workspace]
-members = ["auto_compressor", "compressor_integration_tests"]
+members = ["synapse_auto_compressor", "compressor_integration_tests"]
 
 [package]
 authors = ["Erik Johnston"]

README.md

@@ -3,7 +3,7 @@
 This workspace contains experimental tools that attempt to reduce the number of
 rows in the `state_groups_state` table inside of a Synapse Postgresql database.
 
-# Automated tool: auto_compressor
+# Automated tool: synapse_auto_compressor
 
 ## Introduction:
 
@@ -11,7 +11,7 @@ This tool is significantly more simple to use than the manual tool (described be
 It scans through all of the rows in the `state_groups` database table from the start. When
 it finds a group that hasn't been compressed, it runs the compressor for a while on that
 group's room, saving where it got up to. After compressing a number of these chunks it stops,
-saving where it got up to for the next run of the `auto_compressor`.
+saving where it got up to for the next run of the `synapse_auto_compressor`.
 
 It creates three extra tables in the database: `state_compressor_state` which stores the
 information needed to stop and start the compressor for each room, `state_compressor_progress`
@@ -26,14 +26,15 @@ periodically.
 This tool requires `cargo` to be installed. See https://www.rust-lang.org/tools/install
 for instructions on how to do this.
 
-To build `auto_compressor`, clone this repository and navigate to the `autocompressor/` 
+To build `synapse_auto_compressor`, clone this repository and navigate to the
-subdirectory. Then execute `cargo build`.
+`synapse_auto_compressor/` subdirectory. Then execute `cargo build`.
 
-This will create an executable and store it in `auto_compressor/target/debug/auto_compressor`.
+This will create an executable and store it in
+`synapse_auto_compressor/target/debug/synapse_auto_compressor`.
 
 ## Example usage
 ```
-$ auto_compressor -p postgresql://user:pass@localhost/synapse -c 500 -n 100
+$ synapse_auto_compressor -p postgresql://user:pass@localhost/synapse -c 500 -n 100
 ```
 ## Running Options
 
@@ -199,7 +200,7 @@ If you want to use the compressor in another project, it is recomended that you
 use jemalloc `https://github.com/gnzlbg/jemallocator`.
 
 To prevent the progress bars from being shown, use the `no-progress-bars` feature.
-(See `auto_compressor/Cargo.toml` for an example)
+(See `synapse_auto_compressor/Cargo.toml` for an example)
 
 # Troubleshooting
 
@@ -225,17 +226,17 @@ setting in [`postgresql.conf`](https://www.postgresql.org/docs/current/runtime-c
 The amount of output the tools produce can be altered by setting the RUST_LOG
 environment variable to something.
 
-To get more logs when running the auto_compressor tool try the following:
+To get more logs when running the synapse_auto_compressor tool try the following:
 
 ```
-$ RUST_LOG=debug auto_compressor -p postgresql://user:pass@localhost/synapse -c 50 -n 100
+$ RUST_LOG=debug synapse_auto_compressor -p postgresql://user:pass@localhost/synapse -c 50 -n 100
 ```
 
 If you want to suppress all the debugging info you are getting from the
 Postgres client then try:
 
 ```
-RUST_LOG=auto_compressor=debug,synapse_compress_state=debug auto_compressor [etc.]
+RUST_LOG=synapse_auto_compressor=debug,synapse_compress_state=debug synapse_auto_compressor [etc.]
 ```
 
 This will only print the debugging information from those two packages. For more info see
@@ -265,7 +266,7 @@ be a large problem.
 
 ## Compressor is trying to increase the number of rows
 
-Backfilling can lead to issues with compression. The auto_compressor will
+Backfilling can lead to issues with compression. The synapse_auto_compressor will
 skip chunks it can't reduce the size of and so this should help jump over the backfilled
 state_groups. Lots of state resolution might also impact the ability to use the compressor.
 

compressor_integration_tests/Cargo.toml

@@ -13,7 +13,7 @@ postgres = "0.19.0"
 postgres-openssl = "0.5.0"
 rand = "0.8.0"
 synapse_compress_state = { path = "../", features = ["no-progress-bars"] }
-auto_compressor = { path = "../auto_compressor/" }
+synapse_auto_compressor = { path = "../synapse_auto_compressor/" }
 env_logger = "0.9.0"
 log = "0.4.14"
 

compressor_integration_tests/src/lib.rs

@@ -356,7 +356,7 @@ fn functions_are_self_consistent() {
 }
 
 pub fn setup_logger() {
-    // setup the logger for the auto_compressor
+    // setup the logger for the synapse_auto_compressor
     // The default can be overwritten with RUST_LOG
     // see the README for more information
     if env::var("RUST_LOG").is_err() {
@@ -366,7 +366,7 @@ pub fn setup_logger() {
         // default to printing the debug information for both packages being tested
         // (Note that just setting the global level to debug will log every sql transaction)
         log_builder.filter_module("synapse_compress_state", LevelFilter::Debug);
-        log_builder.filter_module("auto_compressor", LevelFilter::Debug);
+        log_builder.filter_module("synapse_auto_compressor", LevelFilter::Debug);
         // use try_init() incase the logger has been setup by some previous test
         let _ = log_builder.try_init();
     } else {

compressor_integration_tests/tests/auto_compressor_manager_tests.rs

@@ -1,9 +1,5 @@
 use std::collections::BTreeMap;
 
-use auto_compressor::{
-    manager::{compress_chunks_of_database, run_compressor_on_room_chunk},
-    state_saving::{connect_to_database, create_tables_if_needed},
-};
 use compressor_integration_tests::{
     add_contents_to_database, clear_compressor_state, database_collapsed_states_match_map,
     database_structure_matches_map, empty_database,
@@ -14,6 +10,10 @@ use compressor_integration_tests::{
     setup_logger, DB_URL,
 };
 use serial_test::serial;
+use synapse_auto_compressor::{
+    manager::{compress_chunks_of_database, run_compressor_on_room_chunk},
+    state_saving::{connect_to_database, create_tables_if_needed},
+};
 use synapse_compress_state::Level;
 
 #[test]

compressor_integration_tests/tests/auto_compressor_state_saving_tests.rs

@@ -1,9 +1,9 @@
-use auto_compressor::state_saving::{
+use compressor_integration_tests::{clear_compressor_state, setup_logger, DB_URL};
+use serial_test::serial;
+use synapse_auto_compressor::state_saving::{
     connect_to_database, create_tables_if_needed, read_room_compressor_state,
     write_room_compressor_state,
 };
-use compressor_integration_tests::{clear_compressor_state, setup_logger, DB_URL};
-use serial_test::serial;
 use synapse_compress_state::Level;
 
 #[test]

docs/python.md

@@ -18,7 +18,7 @@ the compressor is run.
 
 3. Navigate to the correct location  
 For the automatic tool:  
-`$ cd /home/synapse/rust-synapse-compress-state/auto_compressor`   
+`$ cd /home/synapse/rust-synapse-compress-state/synpase_auto_compressor`   
 For the manual tool:  
 `$ cd /home/synapse/rust-synapse-compress-state`   
 
@@ -30,9 +30,9 @@ This will install the relevant compressor tool into the activated virtual enviro
 ## Automatic tool example:
 
 ```python
-import auto_compressor
+import synapse_auto_compressor
 
-auto_compressor.compress_state_events_table(
+synapse_auto_compressor.compress_state_events_table(
   db_url="postgresql://localhost/synapse",
   chunk_size=500,
   default_levels="100,50,25",

src/lib.rs

@@ -788,7 +788,7 @@ fn synapse_compress_state(_py: Python, m: &PyModule) -> PyResult<()> {
     let _ = pyo3_log::Logger::default()
         // don't send out anything lower than a warning from other crates
         .filter(LevelFilter::Warn)
-        // don't log warnings from synapse_compress_state, the auto_compressor handles these
+        // don't log warnings from synapse_compress_state, the synapse_auto_compressor handles these
         // situations and provides better log messages
         .filter_target("synapse_compress_state".to_owned(), LevelFilter::Debug)
         .install();

synapse_auto_compressor/Cargo.toml

@@ -1,10 +1,16 @@
 [package]
-name = "auto_compressor"
+name = "synapse_auto_compressor"
 authors = ["William Ashton"]
-version = "0.1.0"
+version = "0.1.1"
 edition = "2018"
 
-# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+[package.metadata.maturin]
+requires-python = ">=3.6"
+project-url = {Source = "https://github.com/matrix-org/rust-synapse-compress-state"}
+classifier = [
+    "Development Status :: 4 - Beta",
+    "Programming Language :: Rust",
+]
 
 [dependencies]
 clap = "2.33.0"

synapse_auto_compressor/README.md

@@ -0,0 +1,12 @@
+# Auto Compressor
+
+See the top level readme for information.
+
+
+## Publishing to PyPI
+
+Bump the version number and run from the root directory of the repo:
+
+```
+docker run -it --rm -v $(pwd):/io -e OPENSSL_STATIC=1 konstin2/maturin publish -m synapse_auto_compressor/Cargo.toml --cargo-extra-args "\--features='openssl/vendored'"
+```

synapse_auto_compressor/src/lib.rs

@@ -57,15 +57,16 @@ impl FromStr for LevelInfo {
 
 // PyO3 INTERFACE STARTS HERE
 #[pymodule]
-fn auto_compressor(_py: Python, m: &PyModule) -> PyResult<()> {
+fn synapse_auto_compressor(_py: Python, m: &PyModule) -> PyResult<()> {
     let _ = pyo3_log::Logger::default()
         // don't send out anything lower than a warning from other crates
         .filter(LevelFilter::Warn)
-        // don't log warnings from synapse_compress_state, the auto_compressor handles these
+        // don't log warnings from synapse_compress_state, the
-        // situations and provides better log messages
+        // synapse_auto_compressor handles these situations and provides better
+        // log messages
         .filter_target("synapse_compress_state".to_owned(), LevelFilter::Error)
-        // log info and above for the auto_compressor
+        // log info and above for the synapse_auto_compressor
-        .filter_target("auto_compressor".to_owned(), LevelFilter::Debug)
+        .filter_target("synapse_auto_compressor".to_owned(), LevelFilter::Debug)
         .install();
     // ensure any panics produce error messages in the log
     log_panics::init();
@@ -92,7 +93,7 @@ fn auto_compressor(_py: Python, m: &PyModule) -> PyResult<()> {
         number_of_chunks: i64,
     ) -> PyResult<()> {
         // Announce the start of the program to the logs
-        log::info!("auto_compressor started");
+        log::info!("synapse_auto_compressor started");
 
         // Parse the default_level string into a LevelInfo struct
         let default_levels: LevelInfo = match default_levels.parse() {
@@ -120,7 +121,7 @@ fn auto_compressor(_py: Python, m: &PyModule) -> PyResult<()> {
             return Err(PyErr::new::<PyRuntimeError, _>(format!("{:?}", e)));
         }
 
-        log::info!("auto_compressor finished");
+        log::info!("synapse_auto_compressor finished");
         Ok(())
     }
     Ok(())

synapse_auto_compressor/src/main.rs

@@ -19,20 +19,20 @@
 #[global_allocator]
 static GLOBAL: jemallocator::Jemalloc = jemallocator::Jemalloc;
 
-use auto_compressor::{manager, state_saving, LevelInfo};
 use clap::{crate_authors, crate_description, crate_name, crate_version, value_t, App, Arg};
 use log::LevelFilter;
 use std::{env, fs::OpenOptions};
+use synapse_auto_compressor::{manager, state_saving, LevelInfo};
 
 /// Execution starts here
 fn main() {
-    // setup the logger for the auto_compressor
+    // setup the logger for the synapse_auto_compressor
     // The default can be overwritten with RUST_LOG
     // see the README for more information
     let log_file = OpenOptions::new()
         .append(true)
         .create(true)
-        .open("auto_compressor.log")
+        .open("synapse_auto_compressor.log")
         .unwrap_or_else(|e| panic!("Error occured while opening the log file: {}", e));
 
     if env::var("RUST_LOG").is_err() {
@@ -41,8 +41,8 @@ fn main() {
         log_builder.filter_module("panic", LevelFilter::Error);
         // Only output errors from the synapse_compress state library
         log_builder.filter_module("synapse_compress_state", LevelFilter::Error);
-        // Output log levels info and above from auto_compressor
+        // Output log levels info and above from synapse_auto_compressor
-        log_builder.filter_module("auto_compressor", LevelFilter::Info);
+        log_builder.filter_module("synapse_auto_compressor", LevelFilter::Info);
         log_builder.init();
     } else {
         // If RUST_LOG was set then use that
@@ -54,7 +54,7 @@ fn main() {
     }
     log_panics::init();
     // Announce the start of the program to the logs
-    log::info!("auto_compressor started");
+    log::info!("synapse_auto_compressor started");
 
     // parse the command line arguments using the clap crate
     let arguments = App::new(crate_name!())
@@ -155,5 +155,5 @@ fn main() {
     manager::compress_chunks_of_database(db_url, chunk_size, &default_levels.0, number_of_chunks)
         .unwrap();
 
-    log::info!("auto_compressor finished");
+    log::info!("synapse_auto_compressor finished");
 }