heimoshuiyu/rust-synapse-compress-state
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Make the auto compressor uploadable to pypi (#75)

Browse Source
This commit is contained in:
Erik Johnston
2021-09-28 16:57:13 +01:00
committed by GitHub
parent bf57e81f54
commit 0111079153
15 changed files with 118 additions and 98 deletions
Download Patch File Download Diff File Expand all files Collapse all files

42
Cargo.lock generated
View File

@@ -54,26 +54,6 @@ dependencies = [
"winapi",
]
[[package]]
name = "auto_compressor"
version = "0.1.0"
dependencies = [
"anyhow",
"clap",
"env_logger",
"jemallocator",
"log",
"log-panics",
"openssl",
"postgres",
"postgres-openssl",
"pyo3",
"pyo3-log",
"rand",
"serial_test",
"synapse_compress_state",
]
[[package]]
name = "autocfg"
version = "1.0.1"
@@ -144,7 +124,6 @@ dependencies = [
name = "compressor_integration_tests"
version = "0.1.0"
dependencies = [
"auto_compressor",
"env_logger",
"log",
"openssl",
@@ -154,6 +133,7 @@ dependencies = [
"serial_test",
"state-map",
"string_cache",
"synapse_auto_compressor",
"synapse_compress_state",
]
@@ -1122,6 +1102,26 @@ dependencies = [
"unicode-xid",
]
[[package]]
name = "synapse_auto_compressor"
version = "0.1.1"
dependencies = [
"anyhow",
"clap",
"env_logger",
"jemallocator",
"log",
"log-panics",
"openssl",
"postgres",
"postgres-openssl",
"pyo3",
"pyo3-log",
"rand",
"serial_test",
"synapse_compress_state",
]
[[package]]
name = "synapse_compress_state"
version = "0.1.0"

2
Cargo.toml
View File

@@ -1,5 +1,5 @@
[workspace]
members = ["auto_compressor", "compressor_integration_tests"]
members = ["synapse_auto_compressor", "compressor_integration_tests"]
[package]
authors = ["Erik Johnston"]

23
README.md
View File

@@ -3,7 +3,7 @@
This workspace contains experimental tools that attempt to reduce the number of
rows in the `state_groups_state` table inside of a Synapse Postgresql database.
# Automated tool: auto_compressor
# Automated tool: synapse_auto_compressor
## Introduction:
@@ -11,7 +11,7 @@ This tool is significantly more simple to use than the manual tool (described be
It scans through all of the rows in the `state_groups` database table from the start. When
it finds a group that hasn't been compressed, it runs the compressor for a while on that
group's room, saving where it got up to. After compressing a number of these chunks it stops,
saving where it got up to for the next run of the `auto_compressor`.
saving where it got up to for the next run of the `synapse_auto_compressor`.
It creates three extra tables in the database: `state_compressor_state` which stores the
information needed to stop and start the compressor for each room, `state_compressor_progress`
@@ -26,14 +26,15 @@ periodically.
This tool requires `cargo` to be installed. See https://www.rust-lang.org/tools/install
for instructions on how to do this.
To build `auto_compressor`, clone this repository and navigate to the `autocompressor/`
subdirectory. Then execute `cargo build`.
To build `synapse_auto_compressor`, clone this repository and navigate to the
`synapse_auto_compressor/` subdirectory. Then execute `cargo build`.
This will create an executable and store it in `auto_compressor/target/debug/auto_compressor`.
This will create an executable and store it in
`synapse_auto_compressor/target/debug/synapse_auto_compressor`.
## Example usage
```
$ auto_compressor -p postgresql://user:pass@localhost/synapse -c 500 -n 100
$ synapse_auto_compressor -p postgresql://user:pass@localhost/synapse -c 500 -n 100
```
## Running Options
@@ -199,7 +200,7 @@ If you want to use the compressor in another project, it is recomended that you
use jemalloc `https://github.com/gnzlbg/jemallocator`.
To prevent the progress bars from being shown, use the `no-progress-bars` feature.
(See `auto_compressor/Cargo.toml` for an example)
(See `synapse_auto_compressor/Cargo.toml` for an example)
# Troubleshooting
@@ -225,17 +226,17 @@ setting in [`postgresql.conf`](https://www.postgresql.org/docs/current/runtime-c
The amount of output the tools produce can be altered by setting the RUST_LOG
environment variable to something.
To get more logs when running the auto_compressor tool try the following:
To get more logs when running the synapse_auto_compressor tool try the following:
```
$ RUST_LOG=debug auto_compressor -p postgresql://user:pass@localhost/synapse -c 50 -n 100
$ RUST_LOG=debug synapse_auto_compressor -p postgresql://user:pass@localhost/synapse -c 50 -n 100
```
If you want to suppress all the debugging info you are getting from the
Postgres client then try:
```
RUST_LOG=auto_compressor=debug,synapse_compress_state=debug auto_compressor [etc.]
RUST_LOG=synapse_auto_compressor=debug,synapse_compress_state=debug synapse_auto_compressor [etc.]
```
This will only print the debugging information from those two packages. For more info see
@@ -265,7 +266,7 @@ be a large problem.
## Compressor is trying to increase the number of rows
Backfilling can lead to issues with compression. The auto_compressor will
Backfilling can lead to issues with compression. The synapse_auto_compressor will
skip chunks it can't reduce the size of and so this should help jump over the backfilled
state_groups. Lots of state resolution might also impact the ability to use the compressor.

2
compressor_integration_tests/Cargo.toml
View File

@@ -13,7 +13,7 @@ postgres = "0.19.0"
postgres-openssl = "0.5.0"
rand = "0.8.0"
synapse_compress_state = { path = "../", features = ["no-progress-bars"] }
auto_compressor = { path = "../auto_compressor/" }
synapse_auto_compressor = { path = "../synapse_auto_compressor/" }
env_logger = "0.9.0"
log = "0.4.14"

4
compressor_integration_tests/src/lib.rs
View File

@@ -356,7 +356,7 @@ fn functions_are_self_consistent() {
}
pub fn setup_logger() {
// setup the logger for the auto_compressor
// setup the logger for the synapse_auto_compressor
// The default can be overwritten with RUST_LOG
// see the README for more information
if env::var("RUST_LOG").is_err() {
@@ -366,7 +366,7 @@ pub fn setup_logger() {
// default to printing the debug information for both packages being tested
// (Note that just setting the global level to debug will log every sql transaction)
log_builder.filter_module("synapse_compress_state", LevelFilter::Debug);
log_builder.filter_module("auto_compressor", LevelFilter::Debug);
log_builder.filter_module("synapse_auto_compressor", LevelFilter::Debug);
// use try_init() incase the logger has been setup by some previous test
let _ = log_builder.try_init();
} else {

8
compressor_integration_tests/tests/auto_compressor_manager_tests.rs
View File

@@ -1,9 +1,5 @@
use std::collections::BTreeMap;
use auto_compressor::{
manager::{compress_chunks_of_database, run_compressor_on_room_chunk},
state_saving::{connect_to_database, create_tables_if_needed},
};
use compressor_integration_tests::{
add_contents_to_database, clear_compressor_state, database_collapsed_states_match_map,
database_structure_matches_map, empty_database,
@@ -14,6 +10,10 @@ use compressor_integration_tests::{
setup_logger, DB_URL,
};
use serial_test::serial;
use synapse_auto_compressor::{
manager::{compress_chunks_of_database, run_compressor_on_room_chunk},
state_saving::{connect_to_database, create_tables_if_needed},
};
use synapse_compress_state::Level;
#[test]

6
compressor_integration_tests/tests/auto_compressor_state_saving_tests.rs
View File

@@ -1,9 +1,9 @@
use auto_compressor::state_saving::{
use compressor_integration_tests::{clear_compressor_state, setup_logger, DB_URL};
use serial_test::serial;
use synapse_auto_compressor::state_saving::{
connect_to_database, create_tables_if_needed, read_room_compressor_state,
write_room_compressor_state,
};
use compressor_integration_tests::{clear_compressor_state, setup_logger, DB_URL};
use serial_test::serial;
use synapse_compress_state::Level;
#[test]

6
docs/python.md
View File

@@ -18,7 +18,7 @@ the compressor is run.
3. Navigate to the correct location
For the automatic tool:
`$ cd /home/synapse/rust-synapse-compress-state/auto_compressor`
`$ cd /home/synapse/rust-synapse-compress-state/synpase_auto_compressor`
For the manual tool:
`$ cd /home/synapse/rust-synapse-compress-state`
@@ -30,9 +30,9 @@ This will install the relevant compressor tool into the activated virtual enviro
## Automatic tool example:
```python
import auto_compressor
import synapse_auto_compressor
auto_compressor.compress_state_events_table(
synapse_auto_compressor.compress_state_events_table(
db_url="postgresql://localhost/synapse",
chunk_size=500,
default_levels="100,50,25",

2
src/lib.rs
View File

@@ -788,7 +788,7 @@ fn synapse_compress_state(_py: Python, m: &PyModule) -> PyResult<()> {
let _ = pyo3_log::Logger::default()
// don't send out anything lower than a warning from other crates
.filter(LevelFilter::Warn)
// don't log warnings from synapse_compress_state, the auto_compressor handles these
// don't log warnings from synapse_compress_state, the synapse_auto_compressor handles these
// situations and provides better log messages
.filter_target("synapse_compress_state".to_owned(), LevelFilter::Debug)
.install();

12
auto_compressor/Cargo.toml → synapse_auto_compressor/Cargo.toml
View File

@@ -1,10 +1,16 @@
[package]
name = "auto_compressor"
name = "synapse_auto_compressor"
authors = ["William Ashton"]
version = "0.1.0"
version = "0.1.1"
edition = "2018"
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
[package.metadata.maturin]
requires-python = ">=3.6"
project-url = {Source = "https://github.com/matrix-org/rust-synapse-compress-state"}
classifier = [
"Development Status :: 4 - Beta",
"Programming Language :: Rust",
]
[dependencies]
clap = "2.33.0"

12
synapse_auto_compressor/README.md Normal file
View File

@@ -0,0 +1,12 @@
# Auto Compressor
See the top level readme for information.
## Publishing to PyPI
Bump the version number and run from the root directory of the repo:
```
docker run -it --rm -v $(pwd):/io -e OPENSSL_STATIC=1 konstin2/maturin publish -m synapse_auto_compressor/Cargo.toml --cargo-extra-args "\--features='openssl/vendored'"
```

15
auto_compressor/src/lib.rs → synapse_auto_compressor/src/lib.rs
View File

@@ -57,15 +57,16 @@ impl FromStr for LevelInfo {
// PyO3 INTERFACE STARTS HERE
#[pymodule]
fn auto_compressor(_py: Python, m: &PyModule) -> PyResult<()> {
fn synapse_auto_compressor(_py: Python, m: &PyModule) -> PyResult<()> {
let _ = pyo3_log::Logger::default()
// don't send out anything lower than a warning from other crates
.filter(LevelFilter::Warn)
// don't log warnings from synapse_compress_state, the auto_compressor handles these
// situations and provides better log messages
// don't log warnings from synapse_compress_state, the
// synapse_auto_compressor handles these situations and provides better
// log messages
.filter_target("synapse_compress_state".to_owned(), LevelFilter::Error)
// log info and above for the auto_compressor
.filter_target("auto_compressor".to_owned(), LevelFilter::Debug)
// log info and above for the synapse_auto_compressor
.filter_target("synapse_auto_compressor".to_owned(), LevelFilter::Debug)
.install();
// ensure any panics produce error messages in the log
log_panics::init();
@@ -92,7 +93,7 @@ fn auto_compressor(_py: Python, m: &PyModule) -> PyResult<()> {
number_of_chunks: i64,
) -> PyResult<()> {
// Announce the start of the program to the logs
log::info!("auto_compressor started");
log::info!("synapse_auto_compressor started");
// Parse the default_level string into a LevelInfo struct
let default_levels: LevelInfo = match default_levels.parse() {
@@ -120,7 +121,7 @@ fn auto_compressor(_py: Python, m: &PyModule) -> PyResult<()> {
return Err(PyErr::new::<PyRuntimeError, _>(format!("{:?}", e)));
}
log::info!("auto_compressor finished");
log::info!("synapse_auto_compressor finished");
Ok(())
}
Ok(())

14
auto_compressor/src/main.rs → synapse_auto_compressor/src/main.rs
View File

@@ -19,20 +19,20 @@
#[global_allocator]
static GLOBAL: jemallocator::Jemalloc = jemallocator::Jemalloc;
use auto_compressor::{manager, state_saving, LevelInfo};
use clap::{crate_authors, crate_description, crate_name, crate_version, value_t, App, Arg};
use log::LevelFilter;
use std::{env, fs::OpenOptions};
use synapse_auto_compressor::{manager, state_saving, LevelInfo};
/// Execution starts here
fn main() {
// setup the logger for the auto_compressor
// setup the logger for the synapse_auto_compressor
// The default can be overwritten with RUST_LOG
// see the README for more information
let log_file = OpenOptions::new()
.append(true)
.create(true)
.open("auto_compressor.log")
.open("synapse_auto_compressor.log")
.unwrap_or_else(|e| panic!("Error occured while opening the log file: {}", e));
if env::var("RUST_LOG").is_err() {
@@ -41,8 +41,8 @@ fn main() {
log_builder.filter_module("panic", LevelFilter::Error);
// Only output errors from the synapse_compress state library
log_builder.filter_module("synapse_compress_state", LevelFilter::Error);
// Output log levels info and above from auto_compressor
log_builder.filter_module("auto_compressor", LevelFilter::Info);
// Output log levels info and above from synapse_auto_compressor
log_builder.filter_module("synapse_auto_compressor", LevelFilter::Info);
log_builder.init();
} else {
// If RUST_LOG was set then use that
@@ -54,7 +54,7 @@ fn main() {
}
log_panics::init();
// Announce the start of the program to the logs
log::info!("auto_compressor started");
log::info!("synapse_auto_compressor started");
// parse the command line arguments using the clap crate
let arguments = App::new(crate_name!())
@@ -155,5 +155,5 @@ fn main() {
manager::compress_chunks_of_database(db_url, chunk_size, &default_levels.0, number_of_chunks)
.unwrap();
log::info!("auto_compressor finished");
log::info!("synapse_auto_compressor finished");
}

0
auto_compressor/src/manager.rs → synapse_auto_compressor/src/manager.rs
View File

0
auto_compressor/src/state_saving.rs → synapse_auto_compressor/src/state_saving.rs
View File