heimoshuiyu/rust-synapse-compress-state
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add integration tests that check various config options (#54)

Browse Source
This commit is contained in:
Azrenbeth
2021-09-09 14:26:14 +01:00
committed by GitHub
parent d908d13f8f
commit d32f49303b
4 changed files with 483 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
README.md
View File

@@ -158,3 +158,6 @@ $ docker-compose up -d
$ cargo test --workspace
$ docker-compose down
```
Note, any output from these tests goes into `compressor_integration_tests/tmp/` so if this
directory doesn't already exist then you will need to create it.

55
compressor_integration_tests/src/map_builder.rs
View File

@@ -160,3 +160,58 @@ pub fn compressed_3_3_from_0_to_13_with_state() -> BTreeMap<i64, StateGroupEntry
}
expected
}
/// Generates state map structure that corresponds to edges (with deltas)
///
/// Each group i has state:
// ('node','is', i)
// ('group', j, 'seen') - for all j less than i
pub fn structure_from_edges_with_state(
edges: BTreeMap<i64, i64>,
start: i64,
end: i64,
) -> BTreeMap<i64, StateGroupEntry> {
let mut expected: BTreeMap<i64, StateGroupEntry> = BTreeMap::new();
// Each group i has state:
// ('node','is', i)
// ('group', j, 'seen') - for all j less than i
for i in start..=end {
let prev = edges.get(&i);
//change from Option<&i64> to Option<i64>
let prev = prev.copied();
// create a blank entry for it
let mut entry = StateGroupEntry {
in_range: true,
prev_state_group: prev,
state_map: StateMap::new(),
};
// Add in all state between predecessor and now (non inclusive)
if let Some(p) = prev {
for j in (p + 1)..i {
entry
.state_map
.insert("group", &j.to_string(), "seen".into());
}
} else {
for j in start..i {
entry
.state_map
.insert("group", &j.to_string(), "seen".into());
}
}
// add in the new state for this state group
entry
.state_map
.insert("group", &i.to_string(), "seen".into());
entry.state_map.insert("node", "is", i.to_string().into());
// put it into the expected map
expected.insert(i, entry);
}
expected
}

423
compressor_integration_tests/tests/compressor_config_tests.rs
View File

@@ -1,8 +1,11 @@
use std::collections::BTreeMap;
use compressor_integration_tests::{
add_contents_to_database, database_collapsed_states_match_map, database_structure_matches_map,
empty_database,
map_builder::{
compressed_3_3_from_0_to_13_with_state, line_segments_with_state, line_with_state,
structure_from_edges_with_state,
},
DB_URL,
};
@@ -125,3 +128,423 @@ fn changes_commited_if_no_min_saved_rows() {
// Check that the structure of the database matches the expected structure
assert!(database_structure_matches_map(&expected))
}
#[test]
#[serial(db)]
fn changes_commited_if_min_saved_rows_exceeded() {
// This starts with the following structure
//
// 0-1-2 3-4-5 6-7-8 9-10-11 12-13
//
// Each group i has state:
// ('node','is', i)
// ('group', j, 'seen') - for all j less than i
let initial = line_segments_with_state(0, 13);
// Place this initial state into an empty database
empty_database();
add_contents_to_database("room1", &initial);
// set up the config options
let db_url = DB_URL.to_string();
let room_id = "room1".to_string();
let output_file = Some("./tests/tmp/changes_commited_if_no_min_saved_rows.sql".to_string());
let min_state_group = None;
let min_saved_rows = Some(10);
let groups_to_compress = None;
let max_state_group = None;
let level_sizes = "3,3".to_string();
let transactions = true;
let graphs = false;
let commit_changes = true;
let config = Config::new(
db_url,
room_id,
output_file,
min_state_group,
groups_to_compress,
min_saved_rows,
max_state_group,
level_sizes,
transactions,
graphs,
commit_changes,
)
.unwrap();
// Run the compressor with those settings
run(config);
// This should have created the following structure in the database
// i.e. groups 6 and 9 should have changed from before
// N.B. this saves 11 rows
//
// 0 3\ 12
// 1 4 6\ 13
// 2 5 7 9
// 8 10
// 11
let expected = compressed_3_3_from_0_to_13_with_state();
// Check that the database still gives correct states for each group!
assert!(database_collapsed_states_match_map(&initial));
// Check that the structure of the database matches the expected structure
assert!(database_structure_matches_map(&expected));
}
#[test]
#[serial(db)]
fn changes_not_commited_if_fewer_than_min_saved_rows() {
// This starts with the following structure
//
// 0-1-2 3-4-5 6-7-8 9-10-11 12-13
//
// Each group i has state:
// ('node','is', i)
// ('group', j, 'seen') - for all j less than i
let initial = line_segments_with_state(0, 13);
// Place this initial state into an empty database
empty_database();
add_contents_to_database("room1", &initial);
// set up the config options
let db_url = DB_URL.to_string();
let room_id = "room1".to_string();
let output_file =
Some("./tests/tmp/changes_not_commited_if_fewer_than_min_saved_rows.sql".to_string());
let min_state_group = None;
let min_saved_rows = Some(12);
let groups_to_compress = None;
let max_state_group = None;
let level_sizes = "3,3".to_string();
let transactions = true;
let graphs = false;
let commit_changes = true;
let config = Config::new(
db_url,
room_id,
output_file,
min_state_group,
groups_to_compress,
min_saved_rows,
max_state_group,
level_sizes,
transactions,
graphs,
commit_changes,
)
.unwrap();
// Run the compressor with those settings
run(config);
// This should have created the following structure when running
// (i.e. try and change groups 6 and 9 only)
// BUT: This saves 11 rows which is fewer than min_saved_rows
// therefore there should be no changes committed!
//
// 0 3\ 12
// 1 4 6\ 13
// 2 5 7 9
// 8 10
// 11
// Check that the database still gives correct states for each group!
assert!(database_collapsed_states_match_map(&initial));
// Check that the structure of the database matches the expected structure
assert!(database_structure_matches_map(&initial));
}
#[test]
#[should_panic(expected = "Error connecting to the database:")]
fn run_panics_if_invalid_db_url() {
// set up the config options
let db_url = "thisIsAnInvalidURL".to_string();
let room_id = "room1".to_string();
let output_file = Some("./tests/tmp/run_panics_if_invalid_db_url.sql".to_string());
let min_state_group = None;
let min_saved_rows = None;
let groups_to_compress = None;
let max_state_group = None;
let level_sizes = "3,3".to_string();
let transactions = true;
let graphs = false;
let commit_changes = true;
let config = Config::new(
db_url,
room_id,
output_file,
min_state_group,
groups_to_compress,
min_saved_rows,
max_state_group,
level_sizes,
transactions,
graphs,
commit_changes,
)
.unwrap();
// Run the compressor with those settings
run(config);
}
#[test]
#[serial(db)]
fn run_only_affects_given_room_id() {
// build room1 stuff up
// This starts with the following structure
//
// 0-1-2 3-4-5 6-7-8 9-10-11 12-13
//
// Each group i has state:
// ('node','is', i)
// ('group', j, 'seen') - for all j less than i
let initial_room_1 = line_segments_with_state(0, 13);
// build room2 stuff up
// This starts with the same structure as room 1 but just all group ids
// 14 higher
let initial_room_2 = line_segments_with_state(14, 28);
// Place this initial state into an empty database
empty_database();
add_contents_to_database("room1", &initial_room_1);
add_contents_to_database("room2", &initial_room_2);
// set up the config options
let db_url = DB_URL.to_string();
let room_id = "room1".to_string();
let output_file = Some("./tests/tmp/run_only_affects_given_room_id.sql".to_string());
let min_state_group = None;
let min_saved_rows = None;
let groups_to_compress = None;
let max_state_group = None;
let level_sizes = "3,3".to_string();
let transactions = true;
let graphs = false;
let commit_changes = true;
let config = Config::new(
db_url,
room_id,
output_file,
min_state_group,
groups_to_compress,
min_saved_rows,
max_state_group,
level_sizes,
transactions,
graphs,
commit_changes,
)
.unwrap();
// Run the compressor with those settings
run(config);
// This should have created the following structure in the database
// i.e. groups 6 and 9 should have changed from before
// N.B. this saves 11 rows
//
// 0 3\ 12
// 1 4 6\ 13
// 2 5 7 9
// 8 10
// 11
let expected = compressed_3_3_from_0_to_13_with_state();
// Check that the database still gives correct states for each group
// in both room1 and room2
assert!(database_collapsed_states_match_map(&initial_room_1));
assert!(database_collapsed_states_match_map(&initial_room_2));
// Check that the structure of the database matches the expected structure
// in both room1 and room2
assert!(database_structure_matches_map(&expected));
assert!(database_structure_matches_map(&initial_room_2));
}
#[test]
#[serial(db)]
fn run_respects_groups_to_compress() {
// This starts with the following structure
//
// 0-1-2 3-4-5 6-7-8 9-10-11 12-13
//
// Each group i has state:
// ('node','is', i)
// ('group', j, 'seen') - for all j less than i
let initial = line_segments_with_state(0, 13);
// Place this initial state into an empty database
empty_database();
add_contents_to_database("room1", &initial);
// set up the config options
let db_url = DB_URL.to_string();
let room_id = "room1".to_string();
let output_file = Some("./tests/tmp/run_respects_groups_to_compress.sql".to_string());
let min_state_group = Some(2);
let min_saved_rows = None;
let groups_to_compress = Some(9);
let max_state_group = None;
let level_sizes = "3,3".to_string();
let transactions = true;
let graphs = false;
let commit_changes = true;
let config = Config::new(
db_url,
room_id,
output_file,
min_state_group,
groups_to_compress,
min_saved_rows,
max_state_group,
level_sizes,
transactions,
graphs,
commit_changes,
)
.unwrap();
// Run the compressor with those settings
run(config);
// This should have created the following structure in the database
// as it should only compress from groups higher than 2 (non inclusive)
// and should only compress a total of 9 groups
// i.e. so only group 9 should have changed from before
// N.B. this saves 7 rows
//
// 0 3 6\ 12
// 1 4 7 9 13
// 2 5 8 10
// 11
//
let expected_edges: BTreeMap<i64, i64> = vec![
(1, 0),
(2, 1),
(4, 3),
(5, 4),
(7, 6),
(8, 7),
(9, 6),
(10, 9),
(11, 10),
(13, 12),
]
.into_iter()
.collect();
let expected = structure_from_edges_with_state(expected_edges, 0, 13);
// Check that the database still gives correct states for each group!
assert!(database_collapsed_states_match_map(&initial));
// Check that the structure of the database matches the expected structure
assert!(database_structure_matches_map(&expected))
}
#[test]
#[serial(db)]
fn run_is_idempotent_when_run_on_whole_room() {
// This starts with the following structure
//
// 0-1-2 3-4-5 6-7-8 9-10-11 12-13
//
// Each group i has state:
// ('node','is', i)
// ('group', j, 'seen') - for all j less than i
let initial = line_segments_with_state(0, 13);
// Place this initial state into an empty database
empty_database();
add_contents_to_database("room1", &initial);
// set up the config options
let db_url = DB_URL.to_string();
let room_id = "room1".to_string();
let output_file1 =
Some("./tests/tmp/run_is_idempotent_when_run_on_whole_room_1.sql".to_string());
let output_file2 =
Some("./tests/tmp/run_is_idempotent_when_run_on_whole_room_2.sql".to_string());
let min_state_group = None;
let min_saved_rows = None;
let groups_to_compress = None;
let max_state_group = None;
let level_sizes = "3,3".to_string();
let transactions = true;
let graphs = false;
let commit_changes = true;
let config1 = Config::new(
db_url.clone(),
room_id.clone(),
output_file1,
min_state_group,
groups_to_compress,
min_saved_rows,
max_state_group,
level_sizes.clone(),
transactions,
graphs,
commit_changes,
)
.unwrap();
let config2 = Config::new(
db_url.clone(),
room_id.clone(),
output_file2,
min_state_group,
groups_to_compress,
min_saved_rows,
max_state_group,
level_sizes.clone(),
transactions,
graphs,
commit_changes,
)
.unwrap();
// We are aiming for the following structure in the database
// i.e. groups 6 and 9 should have changed from initial map
// N.B. this saves 11 rows
//
// 0 3\ 12
// 1 4 6\ 13
// 2 5 7 9
// 8 10
// 11
//
// Where each group i has state:
// ('node','is', i)
// ('group', j, 'seen') - for all j less than i
let expected = compressed_3_3_from_0_to_13_with_state();
// Run the compressor with those settings for the first time
run(config1);
// Check that the database still gives correct states for each group!
assert!(database_collapsed_states_match_map(&initial));
// Check that the structure of the database matches the expected structure
assert!(database_structure_matches_map(&expected));
// Run the compressor with those settings for the second time
run(config2);
// Check that the database still gives correct states for each group!
assert!(database_collapsed_states_match_map(&initial));
// Check that the structure of the database still matches the expected structure
assert!(database_structure_matches_map(&expected));
}

3
src/database.rs
View File

@@ -55,7 +55,8 @@ pub fn get_data_from_db(
builder.set_verify(SslVerifyMode::NONE);
let connector = MakeTlsConnector::new(builder.build());
let mut client = Client::connect(db_url, connector).unwrap();
let mut client = Client::connect(db_url, connector)
.unwrap_or_else(|e| panic!("Error connecting to the database: {}", e));
// Search for the group id of the groups_to_compress'th group after min_state_group
// If this is saved, then the compressor can continue by having min_state_group being