heimoshuiyu/rust-synapse-compress-state
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fix clippy warnings, update dependencies (#91)

Browse Source
This commit is contained in:
Jan Alexander Steffens
2022-06-06 11:34:07 +02:00
committed by GitHub
parent 04ac0529e1
commit b4f3d8adbd
12 changed files with 428 additions and 408 deletions
Download Patch File Download Diff File Expand all files Collapse all files

649
Cargo.lock generated
View File

File diff suppressed because it is too large Load Diff

13
Cargo.toml
View File

@@ -9,7 +9,6 @@ version = "0.1.0"
edition = "2018"
[dependencies]
clap = "2.33.0"
indicatif = "0.16.0"
openssl = "0.10.32"
postgres = "0.19.0"
@@ -19,7 +18,7 @@ rayon = "1.3.0"
string_cache = "0.8.0"
env_logger = "0.9.0"
log = "0.4.14"
pyo3-log = "0.4.0"
pyo3-log = "0.6.0"
log-panics = "2.0.0"
[dependencies.state-map]
@@ -29,12 +28,16 @@ git = "https://github.com/matrix-org/rust-matrix-state-map"
[lib]
crate-type = ["cdylib", "rlib"]
[dependencies.clap]
version = "3.1.14"
features = ["cargo"]
[dependencies.pyo3]
version = "0.14.1"
features = ["extension-module","abi3-py36"]
version = "0.16.4"
features = ["extension-module"]
[dependencies.tikv-jemallocator]
version = "0.4.1"
version = "0.5.0"
optional = true
[features]

10
compressor_integration_tests/tests/compressor_config_tests.rs
View File

@@ -49,8 +49,8 @@ fn run_succeeds_without_crashing() {
let verify = true;
let config = Config::new(
db_url.clone(),
room_id.clone(),
db_url,
room_id,
output_file,
min_state_group,
groups_to_compress,
@@ -525,14 +525,14 @@ fn run_is_idempotent_when_run_on_whole_room() {
.unwrap();
let config2 = Config::new(
db_url.clone(),
room_id.clone(),
db_url,
room_id,
output_file2,
min_state_group,
groups_to_compress,
min_saved_rows,
max_state_group,
level_sizes.clone(),
level_sizes,
transactions,
graphs,
commit_changes,

2
compressor_integration_tests/tests/compressor_continue_run_tests.rs
View File

@@ -56,7 +56,7 @@ fn continue_run_called_twice_same_as_run() {
let start = Some(6);
let chunk_size = 7;
let level_info = chunk_stats_1.new_level_info.clone();
let level_info = chunk_stats_1.new_level_info;
// Run the compressor with those settings
let chunk_stats_2 = continue_run(start, chunk_size, &db_url, &room_id, &level_info).unwrap();

9
src/compressor.rs
View File

@@ -181,12 +181,11 @@ impl<'a> Compressor<'a> {
panic!("Can only call `create_new_tree` once");
}
let pb: ProgressBar;
if cfg!(feature = "no-progress-bars") {
pb = ProgressBar::hidden();
let pb = if cfg!(feature = "no-progress-bars") {
ProgressBar::hidden()
} else {
pb = ProgressBar::new(self.original_state_map.len() as u64);
}
ProgressBar::new(self.original_state_map.len() as u64)
};
pb.set_style(
ProgressStyle::default_bar().template("[{elapsed_precise}] {bar} {pos}/{len} {msg}"),
);

6
src/compressor/compressor_tests.rs
View File

@@ -96,11 +96,7 @@ fn create_new_tree_does_nothing_if_already_compressed() {
let pred_group = initial_edges.get(&i);
// Need Option<i64> not Option<&i64>
let prev;
match pred_group {
Some(i) => prev = Some(*i),
None => prev = None,
}
let prev = pred_group.copied();
// insert that edge into the initial map
initial.insert(

6
src/compressor/level_tests.rs
View File

@@ -54,7 +54,7 @@ fn get_head_returns_head() {
#[test]
fn has_space_returns_true_if_empty() {
let l = Level::new(15);
assert_eq!(l.has_space(), true);
assert!(l.has_space());
}
#[test]
@@ -65,7 +65,7 @@ fn has_space_returns_true_if_part_full() {
l.update(1, true);
l.update(143, true);
l.update(15, true);
assert_eq!(l.has_space(), true);
assert!(l.has_space());
}
#[test]
@@ -76,5 +76,5 @@ fn has_space_returns_false_if_full() {
l.update(3, true);
l.update(4, true);
l.update(5, true);
assert_eq!(l.has_space(), false);
assert!(!l.has_space());
}

6
src/compressor/stats_tests.rs
View File

@@ -145,11 +145,7 @@ fn stats_correct_if_no_changes() {
let pred_group = initial_edges.get(&i);
// Need Option<i64> not Option<&i64>
let prev;
match pred_group {
Some(i) => prev = Some(*i),
None => prev = None,
}
let prev = pred_group.copied();
// insert that edge into the initial map
initial.insert(

18
src/database.rs
View File

@@ -372,12 +372,11 @@ fn get_initial_data_from_db(
// Copy the data from the database into a map
let mut state_group_map: BTreeMap<i64, StateGroupEntry> = BTreeMap::new();
let pb: ProgressBar;
if cfg!(feature = "no-progress-bars") {
pb = ProgressBar::hidden();
let pb = if cfg!(feature = "no-progress-bars") {
ProgressBar::hidden()
} else {
pb = ProgressBar::new_spinner();
}
ProgressBar::new_spinner()
};
pb.set_style(
ProgressStyle::default_spinner().template("{spinner} [{elapsed}] {pos} rows retrieved"),
);
@@ -537,12 +536,11 @@ pub fn send_changes_to_db(
debug!("Writing changes...");
// setup the progress bar
let pb: ProgressBar;
if cfg!(feature = "no-progress-bars") {
pb = ProgressBar::hidden();
let pb = if cfg!(feature = "no-progress-bars") {
ProgressBar::hidden()
} else {
pb = ProgressBar::new(old_map.len() as u64);
}
ProgressBar::new(old_map.len() as u64)
};
pb.set_style(
ProgressStyle::default_bar().template("[{elapsed_precise}] {bar} {pos}/{len} {msg}"),
);

81
src/lib.rs
View File

@@ -23,7 +23,7 @@
use log::{info, warn, LevelFilter};
use pyo3::{exceptions, prelude::*};
use clap::{crate_authors, crate_description, crate_name, crate_version, value_t, App, Arg};
use clap::{crate_authors, crate_description, crate_name, crate_version, Arg, Command};
use indicatif::{ProgressBar, ProgressStyle};
use rayon::prelude::*;
use state_map::StateMap;
@@ -117,13 +117,13 @@ pub struct Config {
impl Config {
/// Build up config from command line arguments
pub fn parse_arguments() -> Config {
let matches = App::new(crate_name!())
let matches = Command::new(crate_name!())
.version(crate_version!())
.author(crate_authors!("\n"))
.about(crate_description!())
.arg(
Arg::with_name("postgres-url")
.short("p")
Arg::new("postgres-url")
.short('p')
.value_name("POSTGRES_LOCATION")
.help("The configruation for connecting to the postgres database.")
.long_help(concat!(
@@ -136,8 +136,8 @@ impl Config {
.takes_value(true)
.required(true),
).arg(
Arg::with_name("room_id")
.short("r")
Arg::new("room_id")
.short('r')
.value_name("ROOM_ID")
.help("The room to process")
.long_help(concat!(
@@ -147,23 +147,23 @@ impl Config {
.takes_value(true)
.required(true),
).arg(
Arg::with_name("min_state_group")
.short("b")
Arg::new("min_state_group")
.short('b')
.value_name("MIN_STATE_GROUP")
.help("The state group to start processing from (non inclusive)")
.takes_value(true)
.required(false),
).arg(
Arg::with_name("min_saved_rows")
.short("m")
Arg::new("min_saved_rows")
.short('m')
.value_name("COUNT")
.help("Abort if fewer than COUNT rows would be saved")
.long_help("If the compressor cannot save this many rows from the database then it will stop early")
.takes_value(true)
.required(false),
).arg(
Arg::with_name("groups_to_compress")
.short("n")
Arg::new("groups_to_compress")
.short('n')
.value_name("GROUPS_TO_COMPRESS")
.help("How many groups to load into memory to compress")
.long_help(concat!(
@@ -172,14 +172,14 @@ impl Config {
.takes_value(true)
.required(false),
).arg(
Arg::with_name("output_file")
.short("o")
Arg::new("output_file")
.short('o')
.value_name("FILE")
.help("File to output the changes to in SQL")
.takes_value(true),
).arg(
Arg::with_name("max_state_group")
.short("s")
Arg::new("max_state_group")
.short('s')
.value_name("MAX_STATE_GROUP")
.help("The maximum state group to process up to")
.long_help(concat!(
@@ -188,8 +188,8 @@ impl Config {
.takes_value(true)
.required(false),
).arg(
Arg::with_name("level_sizes")
.short("l")
Arg::new("level_sizes")
.short('l')
.value_name("LEVELS")
.help("Sizes of each new level in the compression algorithm, as a comma separated list.")
.long_help(concat!(
@@ -205,30 +205,30 @@ impl Config {
.default_value("100,50,25")
.takes_value(true),
).arg(
Arg::with_name("transactions")
.short("t")
Arg::new("transactions")
.short('t')
.help("Whether to wrap each state group change in a transaction")
.long_help(concat!("If this flag is set then then each change to a particular",
" state group is wrapped in a transaction. This should be done if you wish to",
" apply the changes while synapse is still running."))
.requires("output_file"),
).arg(
Arg::with_name("graphs")
.short("g")
Arg::new("graphs")
.short('g')
.help("Output before and after graphs")
.long_help(concat!("If this flag is set then output the node and edge information for",
" the state_group directed graph built up from the predecessor state_group links.",
" These can be looked at in something like Gephi (https://gephi.org)")),
).arg(
Arg::with_name("commit_changes")
.short("c")
Arg::new("commit_changes")
.short('c')
.help("Commit changes to the database")
.long_help(concat!("If this flag is set then the changes the compressor makes will",
" be committed to the database. This should be safe to use while synapse is running",
" as it assumes by default that the transactions flag is set")),
).arg(
Arg::with_name("no_verify")
.short("N")
Arg::new("no_verify")
.short('N')
.help("Do not double-check that the compression was performed correctly")
.long_help(concat!("If this flag is set then the verification of the compressed",
" state groups, which compares them to the original groups, is skipped. This",
@@ -263,7 +263,8 @@ impl Config {
.value_of("max_state_group")
.map(|s| s.parse().expect("max_state_group must be an integer"));
let level_sizes = value_t!(matches, "level_sizes", LevelSizes)
let level_sizes = matches
.value_of_t::<LevelSizes>("level_sizes")
.unwrap_or_else(|e| panic!("Unable to parse level_sizes: {}", e));
let transactions = matches.is_present("transactions");
@@ -507,12 +508,11 @@ fn output_sql(
info!("Writing changes...");
let pb: ProgressBar;
if cfg!(feature = "no-progress-bars") {
pb = ProgressBar::hidden();
let pb = if cfg!(feature = "no-progress-bars") {
ProgressBar::hidden()
} else {
pb = ProgressBar::new(old_map.len() as u64);
}
ProgressBar::new(old_map.len() as u64)
};
pb.set_style(
ProgressStyle::default_bar().template("[{elapsed_precise}] {bar} {pos}/{len} {msg}"),
);
@@ -622,12 +622,11 @@ fn check_that_maps_match(
) {
info!("Checking that state maps match...");
let pb: ProgressBar;
if cfg!(feature = "no-progress-bars") {
pb = ProgressBar::hidden();
let pb = if cfg!(feature = "no-progress-bars") {
ProgressBar::hidden()
} else {
pb = ProgressBar::new(old_map.len() as u64);
}
ProgressBar::new(old_map.len() as u64)
};
pb.set_style(
ProgressStyle::default_bar().template("[{elapsed_precise}] {bar} {pos}/{len} {msg}"),
);
@@ -975,7 +974,6 @@ mod lib_tests {
#[test]
fn check_that_maps_match_returns_if_both_empty() {
check_that_maps_match(&BTreeMap::new(), &BTreeMap::new());
assert!(true);
}
#[test]
@@ -1008,7 +1006,6 @@ mod lib_tests {
}
check_that_maps_match(&old_map, &BTreeMap::new());
assert!(true);
}
#[test]
@@ -1044,7 +1041,6 @@ mod lib_tests {
}
check_that_maps_match(&BTreeMap::new(), &new_map);
assert!(true);
}
#[test]
@@ -1076,7 +1072,6 @@ mod lib_tests {
}
check_that_maps_match(&BTreeMap::new(), &old_map.clone());
assert!(true);
}
#[test]
@@ -1139,7 +1134,6 @@ mod lib_tests {
}
check_that_maps_match(&old_map, &new_map);
assert!(true);
}
#[test]
@@ -1221,7 +1215,6 @@ mod lib_tests {
);
check_that_maps_match(&old_map, &new_map);
assert!(true);
}
//TODO: tests for correct SQL code produced by output_sql
@@ -1311,7 +1304,7 @@ mod pyo3_tests {
.unwrap();
assert_eq!(config.db_url, db_url);
assert!(!config.output_file.is_none());
assert!(config.output_file.is_some());
assert_eq!(config.room_id, room_id);
assert_eq!(config.min_state_group, Some(3225));
assert_eq!(config.groups_to_compress, Some(970));

13
synapse_auto_compressor/Cargo.toml
View File

@@ -13,11 +13,10 @@ classifier = [
]
[dependencies]
clap = "2.33.0"
openssl = "0.10.32"
postgres = "0.19.0"
postgres-openssl = "0.5.0"
tikv-jemallocator = "0.4.1"
tikv-jemallocator = "0.5.0"
rand = "0.8.0"
serial_test = "0.5.1"
synapse_compress_state = { path = "../", features = ["no-progress-bars"] }
@@ -25,12 +24,16 @@ env_logger = "0.9.0"
log = "0.4.14"
log-panics = "2.0.0"
anyhow = "1.0.42"
pyo3-log = "0.4.0"
pyo3-log = "0.6.0"
# Needed for pyo3 support
[lib]
crate-type = ["cdylib", "rlib"]
[dependencies.clap]
version = "3.1.14"
features = ["cargo"]
[dependencies.pyo3]
version = "0.14.1"
features = ["extension-module","abi3-py36"]
version = "0.16.4"
features = ["extension-module"]

23
synapse_auto_compressor/src/main.rs
View File

@@ -19,7 +19,7 @@
#[global_allocator]
static GLOBAL: tikv_jemallocator::Jemalloc = tikv_jemallocator::Jemalloc;
use clap::{crate_authors, crate_description, crate_name, crate_version, value_t, App, Arg};
use clap::{crate_authors, crate_description, crate_name, crate_version, Arg, Command};
use log::LevelFilter;
use std::env;
use synapse_auto_compressor::{manager, state_saving, LevelInfo};
@@ -50,13 +50,13 @@ fn main() {
log::info!("synapse_auto_compressor started");
// parse the command line arguments using the clap crate
let arguments = App::new(crate_name!())
let arguments = Command::new(crate_name!())
.version(crate_version!())
.author(crate_authors!("\n"))
.about(crate_description!())
.arg(
Arg::with_name("postgres-url")
.short("p")
Arg::new("postgres-url")
.short('p')
.value_name("POSTGRES_LOCATION")
.help("The configruation for connecting to the postgres database.")
.long_help(concat!(
@@ -69,8 +69,8 @@ fn main() {
.takes_value(true)
.required(true),
).arg(
Arg::with_name("chunk_size")
.short("c")
Arg::new("chunk_size")
.short('c')
.value_name("COUNT")
.help("The maximum number of state groups to load into memroy at once")
.long_help(concat!(
@@ -85,8 +85,8 @@ fn main() {
.takes_value(true)
.required(true),
).arg(
Arg::with_name("default_levels")
.short("l")
Arg::new("default_levels")
.short('l')
.value_name("LEVELS")
.help("Sizes of each new level in the compression algorithm, as a comma separated list.")
.long_help(concat!(
@@ -103,8 +103,8 @@ fn main() {
.takes_value(true)
.required(false),
).arg(
Arg::with_name("number_of_chunks")
.short("n")
Arg::new("number_of_chunks")
.short('n')
.value_name("CHUNKS_TO_COMPRESS")
.help("The number of chunks to compress")
.long_help(concat!(
@@ -127,7 +127,8 @@ fn main() {
.expect("A chunk size is required");
// The default structure to use when compressing
let default_levels = value_t!(arguments, "default_levels", LevelInfo)
let default_levels = arguments
.value_of_t::<LevelInfo>("default_levels")
.unwrap_or_else(|e| panic!("Unable to parse default levels: {}", e));
// The number of rooms to compress with this tool