From d32f49303b60305640791e698dc869208e6eb6a0 Mon Sep 17 00:00:00 2001 From: Azrenbeth <77782548+Azrenbeth@users.noreply.github.com> Date: Thu, 9 Sep 2021 14:26:14 +0100 Subject: [PATCH] Add integration tests that check various config options (#54) --- README.md | 3 + .../src/map_builder.rs | 55 +++ .../tests/compressor_config_tests.rs | 423 ++++++++++++++++++ src/database.rs | 3 +- 4 files changed, 483 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index e1f5816..4d67560 100644 --- a/README.md +++ b/README.md @@ -158,3 +158,6 @@ $ docker-compose up -d $ cargo test --workspace $ docker-compose down ``` + +Note, any output from these tests goes into `compressor_integration_tests/tmp/` so if this +directory doesn't already exist then you will need to create it. diff --git a/compressor_integration_tests/src/map_builder.rs b/compressor_integration_tests/src/map_builder.rs index 9069e8b..f821976 100644 --- a/compressor_integration_tests/src/map_builder.rs +++ b/compressor_integration_tests/src/map_builder.rs @@ -160,3 +160,58 @@ pub fn compressed_3_3_from_0_to_13_with_state() -> BTreeMap, + start: i64, + end: i64, +) -> BTreeMap { + let mut expected: BTreeMap = BTreeMap::new(); + + // Each group i has state: + // ('node','is', i) + // ('group', j, 'seen') - for all j less than i + for i in start..=end { + let prev = edges.get(&i); + + //change from Option<&i64> to Option + let prev = prev.copied(); + + // create a blank entry for it + let mut entry = StateGroupEntry { + in_range: true, + prev_state_group: prev, + state_map: StateMap::new(), + }; + + // Add in all state between predecessor and now (non inclusive) + if let Some(p) = prev { + for j in (p + 1)..i { + entry + .state_map + .insert("group", &j.to_string(), "seen".into()); + } + } else { + for j in start..i { + entry + .state_map + .insert("group", &j.to_string(), "seen".into()); + } + } + + // add in the new state for this state group + entry + .state_map + .insert("group", &i.to_string(), "seen".into()); + entry.state_map.insert("node", "is", i.to_string().into()); + + // put it into the expected map + expected.insert(i, entry); + } + expected +} diff --git a/compressor_integration_tests/tests/compressor_config_tests.rs b/compressor_integration_tests/tests/compressor_config_tests.rs index b99eccd..6c83b3c 100644 --- a/compressor_integration_tests/tests/compressor_config_tests.rs +++ b/compressor_integration_tests/tests/compressor_config_tests.rs @@ -1,8 +1,11 @@ +use std::collections::BTreeMap; + use compressor_integration_tests::{ add_contents_to_database, database_collapsed_states_match_map, database_structure_matches_map, empty_database, map_builder::{ compressed_3_3_from_0_to_13_with_state, line_segments_with_state, line_with_state, + structure_from_edges_with_state, }, DB_URL, }; @@ -125,3 +128,423 @@ fn changes_commited_if_no_min_saved_rows() { // Check that the structure of the database matches the expected structure assert!(database_structure_matches_map(&expected)) } + +#[test] +#[serial(db)] +fn changes_commited_if_min_saved_rows_exceeded() { + // This starts with the following structure + // + // 0-1-2 3-4-5 6-7-8 9-10-11 12-13 + // + // Each group i has state: + // ('node','is', i) + // ('group', j, 'seen') - for all j less than i + let initial = line_segments_with_state(0, 13); + + // Place this initial state into an empty database + empty_database(); + add_contents_to_database("room1", &initial); + + // set up the config options + let db_url = DB_URL.to_string(); + let room_id = "room1".to_string(); + let output_file = Some("./tests/tmp/changes_commited_if_no_min_saved_rows.sql".to_string()); + let min_state_group = None; + let min_saved_rows = Some(10); + let groups_to_compress = None; + let max_state_group = None; + let level_sizes = "3,3".to_string(); + let transactions = true; + let graphs = false; + let commit_changes = true; + + let config = Config::new( + db_url, + room_id, + output_file, + min_state_group, + groups_to_compress, + min_saved_rows, + max_state_group, + level_sizes, + transactions, + graphs, + commit_changes, + ) + .unwrap(); + + // Run the compressor with those settings + run(config); + + // This should have created the following structure in the database + // i.e. groups 6 and 9 should have changed from before + // N.B. this saves 11 rows + // + // 0 3\ 12 + // 1 4 6\ 13 + // 2 5 7 9 + // 8 10 + // 11 + let expected = compressed_3_3_from_0_to_13_with_state(); + + // Check that the database still gives correct states for each group! + assert!(database_collapsed_states_match_map(&initial)); + + // Check that the structure of the database matches the expected structure + assert!(database_structure_matches_map(&expected)); +} + +#[test] +#[serial(db)] +fn changes_not_commited_if_fewer_than_min_saved_rows() { + // This starts with the following structure + // + // 0-1-2 3-4-5 6-7-8 9-10-11 12-13 + // + // Each group i has state: + // ('node','is', i) + // ('group', j, 'seen') - for all j less than i + let initial = line_segments_with_state(0, 13); + + // Place this initial state into an empty database + empty_database(); + add_contents_to_database("room1", &initial); + + // set up the config options + let db_url = DB_URL.to_string(); + let room_id = "room1".to_string(); + let output_file = + Some("./tests/tmp/changes_not_commited_if_fewer_than_min_saved_rows.sql".to_string()); + let min_state_group = None; + let min_saved_rows = Some(12); + let groups_to_compress = None; + let max_state_group = None; + let level_sizes = "3,3".to_string(); + let transactions = true; + let graphs = false; + let commit_changes = true; + + let config = Config::new( + db_url, + room_id, + output_file, + min_state_group, + groups_to_compress, + min_saved_rows, + max_state_group, + level_sizes, + transactions, + graphs, + commit_changes, + ) + .unwrap(); + + // Run the compressor with those settings + run(config); + + // This should have created the following structure when running + // (i.e. try and change groups 6 and 9 only) + // BUT: This saves 11 rows which is fewer than min_saved_rows + // therefore there should be no changes committed! + // + // 0 3\ 12 + // 1 4 6\ 13 + // 2 5 7 9 + // 8 10 + // 11 + + // Check that the database still gives correct states for each group! + assert!(database_collapsed_states_match_map(&initial)); + + // Check that the structure of the database matches the expected structure + assert!(database_structure_matches_map(&initial)); +} + +#[test] +#[should_panic(expected = "Error connecting to the database:")] +fn run_panics_if_invalid_db_url() { + // set up the config options + let db_url = "thisIsAnInvalidURL".to_string(); + let room_id = "room1".to_string(); + let output_file = Some("./tests/tmp/run_panics_if_invalid_db_url.sql".to_string()); + let min_state_group = None; + let min_saved_rows = None; + let groups_to_compress = None; + let max_state_group = None; + let level_sizes = "3,3".to_string(); + let transactions = true; + let graphs = false; + let commit_changes = true; + + let config = Config::new( + db_url, + room_id, + output_file, + min_state_group, + groups_to_compress, + min_saved_rows, + max_state_group, + level_sizes, + transactions, + graphs, + commit_changes, + ) + .unwrap(); + + // Run the compressor with those settings + run(config); +} + +#[test] +#[serial(db)] +fn run_only_affects_given_room_id() { + // build room1 stuff up + // This starts with the following structure + // + // 0-1-2 3-4-5 6-7-8 9-10-11 12-13 + // + // Each group i has state: + // ('node','is', i) + // ('group', j, 'seen') - for all j less than i + let initial_room_1 = line_segments_with_state(0, 13); + + // build room2 stuff up + // This starts with the same structure as room 1 but just all group ids + // 14 higher + let initial_room_2 = line_segments_with_state(14, 28); + + // Place this initial state into an empty database + empty_database(); + add_contents_to_database("room1", &initial_room_1); + add_contents_to_database("room2", &initial_room_2); + + // set up the config options + let db_url = DB_URL.to_string(); + let room_id = "room1".to_string(); + let output_file = Some("./tests/tmp/run_only_affects_given_room_id.sql".to_string()); + let min_state_group = None; + let min_saved_rows = None; + let groups_to_compress = None; + let max_state_group = None; + let level_sizes = "3,3".to_string(); + let transactions = true; + let graphs = false; + let commit_changes = true; + + let config = Config::new( + db_url, + room_id, + output_file, + min_state_group, + groups_to_compress, + min_saved_rows, + max_state_group, + level_sizes, + transactions, + graphs, + commit_changes, + ) + .unwrap(); + + // Run the compressor with those settings + run(config); + + // This should have created the following structure in the database + // i.e. groups 6 and 9 should have changed from before + // N.B. this saves 11 rows + // + // 0 3\ 12 + // 1 4 6\ 13 + // 2 5 7 9 + // 8 10 + // 11 + let expected = compressed_3_3_from_0_to_13_with_state(); + + // Check that the database still gives correct states for each group + // in both room1 and room2 + assert!(database_collapsed_states_match_map(&initial_room_1)); + assert!(database_collapsed_states_match_map(&initial_room_2)); + + // Check that the structure of the database matches the expected structure + // in both room1 and room2 + assert!(database_structure_matches_map(&expected)); + assert!(database_structure_matches_map(&initial_room_2)); +} + +#[test] +#[serial(db)] +fn run_respects_groups_to_compress() { + // This starts with the following structure + // + // 0-1-2 3-4-5 6-7-8 9-10-11 12-13 + // + // Each group i has state: + // ('node','is', i) + // ('group', j, 'seen') - for all j less than i + let initial = line_segments_with_state(0, 13); + + // Place this initial state into an empty database + empty_database(); + add_contents_to_database("room1", &initial); + + // set up the config options + let db_url = DB_URL.to_string(); + let room_id = "room1".to_string(); + let output_file = Some("./tests/tmp/run_respects_groups_to_compress.sql".to_string()); + let min_state_group = Some(2); + let min_saved_rows = None; + let groups_to_compress = Some(9); + let max_state_group = None; + let level_sizes = "3,3".to_string(); + let transactions = true; + let graphs = false; + let commit_changes = true; + + let config = Config::new( + db_url, + room_id, + output_file, + min_state_group, + groups_to_compress, + min_saved_rows, + max_state_group, + level_sizes, + transactions, + graphs, + commit_changes, + ) + .unwrap(); + + // Run the compressor with those settings + run(config); + + // This should have created the following structure in the database + // as it should only compress from groups higher than 2 (non inclusive) + // and should only compress a total of 9 groups + // i.e. so only group 9 should have changed from before + // N.B. this saves 7 rows + // + // 0 3 6\ 12 + // 1 4 7 9 13 + // 2 5 8 10 + // 11 + // + let expected_edges: BTreeMap = vec![ + (1, 0), + (2, 1), + (4, 3), + (5, 4), + (7, 6), + (8, 7), + (9, 6), + (10, 9), + (11, 10), + (13, 12), + ] + .into_iter() + .collect(); + + let expected = structure_from_edges_with_state(expected_edges, 0, 13); + + // Check that the database still gives correct states for each group! + assert!(database_collapsed_states_match_map(&initial)); + + // Check that the structure of the database matches the expected structure + assert!(database_structure_matches_map(&expected)) +} + +#[test] +#[serial(db)] +fn run_is_idempotent_when_run_on_whole_room() { + // This starts with the following structure + // + // 0-1-2 3-4-5 6-7-8 9-10-11 12-13 + // + // Each group i has state: + // ('node','is', i) + // ('group', j, 'seen') - for all j less than i + let initial = line_segments_with_state(0, 13); + + // Place this initial state into an empty database + empty_database(); + add_contents_to_database("room1", &initial); + + // set up the config options + let db_url = DB_URL.to_string(); + let room_id = "room1".to_string(); + let output_file1 = + Some("./tests/tmp/run_is_idempotent_when_run_on_whole_room_1.sql".to_string()); + let output_file2 = + Some("./tests/tmp/run_is_idempotent_when_run_on_whole_room_2.sql".to_string()); + let min_state_group = None; + let min_saved_rows = None; + let groups_to_compress = None; + let max_state_group = None; + let level_sizes = "3,3".to_string(); + let transactions = true; + let graphs = false; + let commit_changes = true; + + let config1 = Config::new( + db_url.clone(), + room_id.clone(), + output_file1, + min_state_group, + groups_to_compress, + min_saved_rows, + max_state_group, + level_sizes.clone(), + transactions, + graphs, + commit_changes, + ) + .unwrap(); + + let config2 = Config::new( + db_url.clone(), + room_id.clone(), + output_file2, + min_state_group, + groups_to_compress, + min_saved_rows, + max_state_group, + level_sizes.clone(), + transactions, + graphs, + commit_changes, + ) + .unwrap(); + + // We are aiming for the following structure in the database + // i.e. groups 6 and 9 should have changed from initial map + // N.B. this saves 11 rows + // + // 0 3\ 12 + // 1 4 6\ 13 + // 2 5 7 9 + // 8 10 + // 11 + // + // Where each group i has state: + // ('node','is', i) + // ('group', j, 'seen') - for all j less than i + let expected = compressed_3_3_from_0_to_13_with_state(); + + // Run the compressor with those settings for the first time + run(config1); + + // Check that the database still gives correct states for each group! + assert!(database_collapsed_states_match_map(&initial)); + + // Check that the structure of the database matches the expected structure + assert!(database_structure_matches_map(&expected)); + + // Run the compressor with those settings for the second time + run(config2); + + // Check that the database still gives correct states for each group! + assert!(database_collapsed_states_match_map(&initial)); + + // Check that the structure of the database still matches the expected structure + assert!(database_structure_matches_map(&expected)); +} diff --git a/src/database.rs b/src/database.rs index 6ecd1e3..9d1bf8c 100644 --- a/src/database.rs +++ b/src/database.rs @@ -55,7 +55,8 @@ pub fn get_data_from_db( builder.set_verify(SslVerifyMode::NONE); let connector = MakeTlsConnector::new(builder.build()); - let mut client = Client::connect(db_url, connector).unwrap(); + let mut client = Client::connect(db_url, connector) + .unwrap_or_else(|e| panic!("Error connecting to the database: {}", e)); // Search for the group id of the groups_to_compress'th group after min_state_group // If this is saved, then the compressor can continue by having min_state_group being