heimoshuiyu/rust-synapse-compress-state
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add method that compresses the chunks with lowest uncompressed state_group ids (#72)

Browse Source
This commit is contained in:
Azrenbeth
2021-09-27 09:49:34 +01:00
committed by GitHub
parent 3271221311
commit a069d8765a
6 changed files with 331 additions and 9 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
compressor_integration_tests/src/lib.rs
View File

@@ -314,6 +314,7 @@ pub fn clear_compressor_state() {
let sql = r"
TRUNCATE state_compressor_state;
TRUNCATE state_compressor_progress;
UPDATE state_compressor_total_progress SET lowest_uncompressed_group = 0;
";
client.batch_execute(sql).unwrap();

171
compressor_integration_tests/tests/auto_compressor_manager_tests.rs
View File

@@ -1,11 +1,16 @@
use std::collections::BTreeMap;
use auto_compressor::{
manager::run_compressor_on_room_chunk,
manager::{compress_chunks_of_database, run_compressor_on_room_chunk},
state_saving::{connect_to_database, create_tables_if_needed},
};
use compressor_integration_tests::{
add_contents_to_database, clear_compressor_state, database_collapsed_states_match_map,
database_structure_matches_map, empty_database,
map_builder::{compressed_3_3_from_0_to_13_with_state, line_segments_with_state},
map_builder::{
compressed_3_3_from_0_to_13_with_state, line_segments_with_state,
structure_from_edges_with_state,
},
setup_logger, DB_URL,
};
use serial_test::serial;
@@ -31,7 +36,7 @@ fn run_compressor_on_room_chunk_works() {
clear_compressor_state();
// compress in 3,3 level sizes by default
let default_levels = vec![Level::restore(3, 0, None), Level::restore(3, 0, None)];
let default_levels = vec![Level::new(3), Level::new(3)];
// compress the first 7 groups in the room
// structure should be the following afterwards
@@ -63,3 +68,163 @@ fn run_compressor_on_room_chunk_works() {
// Check that the structure of the database matches the expected structure
assert!(database_structure_matches_map(&expected));
}
#[test]
#[serial(db)]
fn compress_chunks_of_database_compresses_multiple_rooms() {
setup_logger();
// This creates 2 with the following structure
//
// 0-1-2 3-4-5 6-7-8 9-10-11 12-13
// (with room2's numbers shifted up 14)
//
// Each group i has state:
// ('node','is', i)
// ('group', j, 'seen') - for all j less than i in that room
let initial1 = line_segments_with_state(0, 13);
let initial2 = line_segments_with_state(14, 27);
empty_database();
add_contents_to_database("room1", &initial1);
add_contents_to_database("room2", &initial2);
let mut client = connect_to_database(DB_URL).unwrap();
create_tables_if_needed(&mut client).unwrap();
clear_compressor_state();
// compress in 3,3 level sizes by default
let default_levels = vec![Level::new(3), Level::new(3)];
// Compress 4 chunks of size 8.
// The first two should compress room1 and the second two should compress room2
compress_chunks_of_database(DB_URL, 8, &default_levels, 4).unwrap();
// We are aiming for the following structure in the database for room1
// i.e. groups 6 and 9 should have changed from initial map
// N.B. this saves 11 rows
//
// 0 3\ 12
// 1 4 6\ 13
// 2 5 7 9
// 8 10
// 11
//
// Where each group i has state:
// ('node','is', i)
// ('group', j, 'seen') - for all j less than i
let expected1 = compressed_3_3_from_0_to_13_with_state();
// Check that the database still gives correct states for each group in room1
assert!(database_collapsed_states_match_map(&initial1));
// Check that the structure of the database matches the expected structure for room1
assert!(database_structure_matches_map(&expected1));
// room 2 should have the same structure but will all numbers shifted up by 14
let expected_edges: BTreeMap<i64, i64> = vec![
(15, 14),
(16, 15),
(18, 17),
(19, 18),
(20, 17),
(21, 20),
(22, 21),
(23, 20),
(24, 23),
(25, 24),
(27, 26),
]
.into_iter()
.collect();
let expected2 = structure_from_edges_with_state(expected_edges, 14, 27);
// Check that the database still gives correct states for each group in room2
assert!(database_collapsed_states_match_map(&initial2));
// Check that the structure of the database matches the expected structure for room2
assert!(database_structure_matches_map(&expected2));
}
#[test]
#[serial(db)]
fn compress_chunks_of_database_continues_where_it_left_off() {
setup_logger();
// This creates 2 with the following structure
//
// 0-1-2 3-4-5 6-7-8 9-10-11 12-13
// (with room2's numbers shifted up 14)
//
// Each group i has state:
// ('node','is', i)
// ('group', j, 'seen') - for all j less than i in that room
let initial1 = line_segments_with_state(0, 13);
let initial2 = line_segments_with_state(14, 27);
empty_database();
add_contents_to_database("room1", &initial1);
add_contents_to_database("room2", &initial2);
let mut client = connect_to_database(DB_URL).unwrap();
create_tables_if_needed(&mut client).unwrap();
clear_compressor_state();
// compress in 3,3 level sizes by default
let default_levels = vec![Level::new(3), Level::new(3)];
// Compress chunks of various sizes:
//
// These two should compress room1
compress_chunks_of_database(DB_URL, 8, &default_levels, 1).unwrap();
compress_chunks_of_database(DB_URL, 100, &default_levels, 1).unwrap();
// These three should compress room2
compress_chunks_of_database(DB_URL, 1, &default_levels, 2).unwrap();
compress_chunks_of_database(DB_URL, 5, &default_levels, 1).unwrap();
compress_chunks_of_database(DB_URL, 5, &default_levels, 1).unwrap();
// We are aiming for the following structure in the database for room1
// i.e. groups 6 and 9 should have changed from initial map
// N.B. this saves 11 rows
//
// 0 3\ 12
// 1 4 6\ 13
// 2 5 7 9
// 8 10
// 11
//
// Where each group i has state:
// ('node','is', i)
// ('group', j, 'seen') - for all j less than i
let expected1 = compressed_3_3_from_0_to_13_with_state();
// Check that the database still gives correct states for each group in room1
assert!(database_collapsed_states_match_map(&initial1));
// Check that the structure of the database matches the expected structure for room1
assert!(database_structure_matches_map(&expected1));
// room 2 should have the same structure but will all numbers shifted up by 14
let expected_edges: BTreeMap<i64, i64> = vec![
(15, 14),
(16, 15),
(18, 17),
(19, 18),
(20, 17),
(21, 20),
(22, 21),
(23, 20),
(24, 23),
(25, 24),
(27, 26),
]
.into_iter()
.collect();
let expected2 = structure_from_edges_with_state(expected_edges, 14, 27);
// Check that the database still gives correct states for each group in room2
assert!(database_collapsed_states_match_map(&initial2));
// Check that the structure of the database matches the expected structure for room2
assert!(database_structure_matches_map(&expected2));
}