Add option to commit changes to the database automatically (#53)
This commit is contained in:
Azrenbeth
GitHub
@@ -1,8 +1,10 @@
|
||||
use openssl::ssl::{SslConnector, SslMethod, SslVerifyMode};
|
||||
use postgres::Client;
|
||||
use postgres::{fallible_iterator::FallibleIterator, Client};
|
||||
use postgres_openssl::MakeTlsConnector;
|
||||
use rand::{distributions::Alphanumeric, thread_rng, Rng};
|
||||
use state_map::StateMap;
|
||||
use std::{borrow::Cow, collections::BTreeMap, fmt};
|
||||
use string_cache::DefaultAtom as Atom;
|
||||
|
||||
use synapse_compress_state::StateGroupEntry;
|
||||
|
||||
@@ -105,3 +107,230 @@ impl<'a> fmt::Display for PGEscape<'a> {
|
||||
write!(f, "{}{}{}", delim, self.0, delim)
|
||||
}
|
||||
}
|
||||
|
||||
/// Checks whether the state at each state group is the same as what the map thinks it should be
|
||||
///
|
||||
/// i.e. when synapse tries to work out the state for a given state group by looking at
|
||||
/// the database. Will the state it gets be the same as what the map thinks it should be
|
||||
pub fn database_collapsed_states_match_map(
|
||||
state_group_map: &BTreeMap<i64, StateGroupEntry>,
|
||||
) -> bool {
|
||||
for sg in state_group_map.keys() {
|
||||
let map_state = collapse_state_with_map(state_group_map, *sg);
|
||||
let database_state = collapse_state_with_database(*sg);
|
||||
if map_state != database_state {
|
||||
println!("database state {} doesn't match", sg);
|
||||
println!("expected {:?}", map_state);
|
||||
println!("but found {:?}", database_state);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
true
|
||||
}
|
||||
|
||||
/// Gets the full state for a given group from the map (of deltas)
|
||||
fn collapse_state_with_map(
|
||||
map: &BTreeMap<i64, StateGroupEntry>,
|
||||
state_group: i64,
|
||||
) -> StateMap<Atom> {
|
||||
let mut entry = &map[&state_group];
|
||||
let mut state_map = StateMap::new();
|
||||
|
||||
let mut stack = vec![state_group];
|
||||
|
||||
while let Some(prev_state_group) = entry.prev_state_group {
|
||||
stack.push(prev_state_group);
|
||||
if !map.contains_key(&prev_state_group) {
|
||||
panic!("Missing {}", prev_state_group);
|
||||
}
|
||||
entry = &map[&prev_state_group];
|
||||
}
|
||||
|
||||
for sg in stack.iter().rev() {
|
||||
state_map.extend(
|
||||
map[sg]
|
||||
.state_map
|
||||
.iter()
|
||||
.map(|((t, s), e)| ((t, s), e.clone())),
|
||||
);
|
||||
}
|
||||
|
||||
state_map
|
||||
}
|
||||
|
||||
fn collapse_state_with_database(state_group: i64) -> StateMap<Atom> {
|
||||
// connect to the database
|
||||
let mut builder = SslConnector::builder(SslMethod::tls()).unwrap();
|
||||
builder.set_verify(SslVerifyMode::NONE);
|
||||
let connector = MakeTlsConnector::new(builder.build());
|
||||
|
||||
let mut client = Client::connect(DB_URL, connector).unwrap();
|
||||
|
||||
// Gets the delta for a specific state group
|
||||
let query_deltas = r#"
|
||||
SELECT m.id, type, state_key, s.event_id
|
||||
FROM state_groups AS m
|
||||
LEFT JOIN state_groups_state AS s ON (m.id = s.state_group)
|
||||
WHERE m.id = $1
|
||||
"#;
|
||||
|
||||
// If there is no delta for that specific state group, then we still want to find
|
||||
// the predecessor (so have split this into a different query)
|
||||
let query_pred = r#"
|
||||
SELECT prev_state_group
|
||||
FROM state_group_edges
|
||||
WHERE state_group = $1
|
||||
"#;
|
||||
|
||||
let mut state_map: StateMap<Atom> = StateMap::new();
|
||||
|
||||
let mut next_group = Some(state_group);
|
||||
|
||||
while let Some(sg) = next_group {
|
||||
// get predecessor from state_group_edges
|
||||
let mut pred = client.query_raw(query_pred, &[sg]).unwrap();
|
||||
|
||||
// set next_group to predecessor
|
||||
next_group = match pred.next().unwrap() {
|
||||
Some(p) => p.get(0),
|
||||
None => None,
|
||||
};
|
||||
|
||||
// if there was a predecessor then assert that it is unique
|
||||
if next_group.is_some() {
|
||||
assert!(pred.next().unwrap().is_none());
|
||||
}
|
||||
drop(pred);
|
||||
|
||||
let mut rows = client.query_raw(query_deltas, &[sg]).unwrap();
|
||||
|
||||
while let Some(row) = rows.next().unwrap() {
|
||||
// Copy the single delta from the predecessor stored in this row
|
||||
if let Some(etype) = row.get::<_, Option<String>>(1) {
|
||||
let key = &row.get::<_, String>(2);
|
||||
|
||||
// only insert if not overriding existing entry
|
||||
// this is because the newer delta is found FIRST
|
||||
if state_map.get(&etype, key).is_none() {
|
||||
state_map.insert(&etype, key, row.get::<_, String>(3).into());
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
state_map
|
||||
}
|
||||
|
||||
/// Check whether predecessors and deltas stored in the database are the same as in the map
|
||||
pub fn database_structure_matches_map(state_group_map: &BTreeMap<i64, StateGroupEntry>) -> bool {
|
||||
// connect to the database
|
||||
let mut builder = SslConnector::builder(SslMethod::tls()).unwrap();
|
||||
builder.set_verify(SslVerifyMode::NONE);
|
||||
let connector = MakeTlsConnector::new(builder.build());
|
||||
|
||||
let mut client = Client::connect(DB_URL, connector).unwrap();
|
||||
|
||||
// Gets the delta for a specific state group
|
||||
let query_deltas = r#"
|
||||
SELECT m.id, type, state_key, s.event_id
|
||||
FROM state_groups AS m
|
||||
LEFT JOIN state_groups_state AS s ON (m.id = s.state_group)
|
||||
WHERE m.id = $1
|
||||
"#;
|
||||
|
||||
// If there is no delta for that specific state group, then we still want to find
|
||||
// the predecessor (so have split this into a different query)
|
||||
let query_pred = r#"
|
||||
SELECT prev_state_group
|
||||
FROM state_group_edges
|
||||
WHERE state_group = $1
|
||||
"#;
|
||||
|
||||
for (sg, entry) in state_group_map {
|
||||
// get predecessor from state_group_edges
|
||||
let mut pred_iter = client.query_raw(query_pred, &[sg]).unwrap();
|
||||
|
||||
// read out the predecessor value from the database
|
||||
let database_pred = match pred_iter.next().unwrap() {
|
||||
Some(p) => p.get(0),
|
||||
None => None,
|
||||
};
|
||||
|
||||
// if there was a predecessor then assert that it is unique
|
||||
if database_pred.is_some() {
|
||||
assert!(pred_iter.next().unwrap().is_none());
|
||||
}
|
||||
|
||||
// check if it matches map
|
||||
if database_pred != entry.prev_state_group {
|
||||
println!(
|
||||
"ERROR: predecessor for {} was {:?} (expected {:?})",
|
||||
sg, database_pred, entry.prev_state_group
|
||||
);
|
||||
return false;
|
||||
}
|
||||
// needed so that can create another query
|
||||
drop(pred_iter);
|
||||
|
||||
// Now check that deltas are the same
|
||||
let mut state_map: StateMap<Atom> = StateMap::new();
|
||||
|
||||
// Get delta from state_groups_state
|
||||
let mut rows = client.query_raw(query_deltas, &[sg]).unwrap();
|
||||
|
||||
while let Some(row) = rows.next().unwrap() {
|
||||
// Copy the single delta from the predecessor stored in this row
|
||||
if let Some(etype) = row.get::<_, Option<String>>(1) {
|
||||
state_map.insert(
|
||||
&etype,
|
||||
&row.get::<_, String>(2),
|
||||
row.get::<_, String>(3).into(),
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
// Check that the delta matches the map
|
||||
if state_map != entry.state_map {
|
||||
println!("ERROR: delta for {} didn't match", sg);
|
||||
println!("Expected: {:?}", entry.state_map);
|
||||
println!("Actual: {:?}", state_map);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
true
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn functions_are_self_consistent() {
|
||||
let mut initial: BTreeMap<i64, StateGroupEntry> = BTreeMap::new();
|
||||
let mut prev = None;
|
||||
|
||||
// This starts with the following structure
|
||||
//
|
||||
// 0-1-2-3-4-5-6-7-8-9-10-11-12-13
|
||||
//
|
||||
// Each group i has state:
|
||||
// ('node','is', i)
|
||||
// ('group', j, 'seen') - for all j less than i
|
||||
for i in 0i64..=13i64 {
|
||||
let mut entry = StateGroupEntry {
|
||||
in_range: true,
|
||||
prev_state_group: prev,
|
||||
state_map: StateMap::new(),
|
||||
};
|
||||
entry
|
||||
.state_map
|
||||
.insert("group", &i.to_string(), "seen".into());
|
||||
entry.state_map.insert("node", "is", i.to_string().into());
|
||||
|
||||
initial.insert(i, entry);
|
||||
|
||||
prev = Some(i)
|
||||
}
|
||||
|
||||
empty_database();
|
||||
add_contents_to_database("room1", &initial);
|
||||
|
||||
assert!(database_collapsed_states_match_map(&initial));
|
||||
assert!(database_structure_matches_map(&initial));
|
||||
}
|
||||
|
||||
@@ -34,3 +34,129 @@ pub fn line_with_state(start: i64, end: i64) -> BTreeMap<i64, StateGroupEntry> {
|
||||
|
||||
initial
|
||||
}
|
||||
|
||||
/// Generates line segments in a chain of state groups each with state deltas
|
||||
///
|
||||
/// If called wiht start=0, end=13 this would build the following:
|
||||
///
|
||||
/// 0-1-2 3-4-5 6-7-8 9-10-11 12-13
|
||||
///
|
||||
/// Where each group i has state:
|
||||
/// ('node','is', i)
|
||||
/// ('group', j, 'seen') - for all j less than i
|
||||
pub fn line_segments_with_state(start: i64, end: i64) -> BTreeMap<i64, StateGroupEntry> {
|
||||
let mut initial: BTreeMap<i64, StateGroupEntry> = BTreeMap::new();
|
||||
let mut prev = None;
|
||||
|
||||
for i in start..=end {
|
||||
// if the state is a snapshot then set its predecessor to NONE
|
||||
if (i - start) % 3 == 0 {
|
||||
prev = None;
|
||||
}
|
||||
|
||||
// create a blank entry for it
|
||||
let mut entry = StateGroupEntry {
|
||||
in_range: true,
|
||||
prev_state_group: prev,
|
||||
state_map: StateMap::new(),
|
||||
};
|
||||
|
||||
// if it's a snapshot then add in all previous state
|
||||
if prev.is_none() {
|
||||
for j in start..i {
|
||||
entry
|
||||
.state_map
|
||||
.insert("group", &j.to_string(), "seen".into());
|
||||
}
|
||||
}
|
||||
|
||||
// add in the new state for this state group
|
||||
entry
|
||||
.state_map
|
||||
.insert("group", &i.to_string(), "seen".into());
|
||||
entry.state_map.insert("node", "is", i.to_string().into());
|
||||
|
||||
// put it into the initial map
|
||||
initial.insert(i, entry);
|
||||
|
||||
// set this group as the predecessor for the next
|
||||
prev = Some(i)
|
||||
}
|
||||
initial
|
||||
}
|
||||
|
||||
/// This generates the correct compressed structure with 3,3 levels
|
||||
///
|
||||
/// Note: only correct structure when no impossible predecessors
|
||||
///
|
||||
/// Structure generated:
|
||||
///
|
||||
/// 0 3\ 12
|
||||
/// 1 4 6\ 13
|
||||
/// 2 5 7 9
|
||||
/// 8 10
|
||||
/// 11
|
||||
/// Where each group i has state:
|
||||
/// ('node','is', i)
|
||||
/// ('group', j, 'seen') - for all j less than i
|
||||
pub fn compressed_3_3_from_0_to_13_with_state() -> BTreeMap<i64, StateGroupEntry> {
|
||||
let expected_edges: BTreeMap<i64, i64> = vec![
|
||||
(1, 0),
|
||||
(2, 1),
|
||||
(4, 3),
|
||||
(5, 4),
|
||||
(6, 3),
|
||||
(7, 6),
|
||||
(8, 7),
|
||||
(9, 6),
|
||||
(10, 9),
|
||||
(11, 10),
|
||||
(13, 12),
|
||||
]
|
||||
.into_iter()
|
||||
.collect();
|
||||
|
||||
let mut expected: BTreeMap<i64, StateGroupEntry> = BTreeMap::new();
|
||||
|
||||
// Each group i has state:
|
||||
// ('node','is', i)
|
||||
// ('group', j, 'seen') - for all j less than i
|
||||
for i in 0i64..=13i64 {
|
||||
let prev = expected_edges.get(&i);
|
||||
|
||||
//change from Option<&i64> to Option<i64>
|
||||
let prev = prev.copied();
|
||||
|
||||
// create a blank entry for it
|
||||
let mut entry = StateGroupEntry {
|
||||
in_range: true,
|
||||
prev_state_group: prev,
|
||||
state_map: StateMap::new(),
|
||||
};
|
||||
|
||||
// Add in all state between predecessor and now (non inclusive)
|
||||
if let Some(p) = prev {
|
||||
for j in (p + 1)..i {
|
||||
entry
|
||||
.state_map
|
||||
.insert("group", &j.to_string(), "seen".into());
|
||||
}
|
||||
} else {
|
||||
for j in 0i64..i {
|
||||
entry
|
||||
.state_map
|
||||
.insert("group", &j.to_string(), "seen".into());
|
||||
}
|
||||
}
|
||||
|
||||
// add in the new state for this state group
|
||||
entry
|
||||
.state_map
|
||||
.insert("group", &i.to_string(), "seen".into());
|
||||
entry.state_map.insert("node", "is", i.to_string().into());
|
||||
|
||||
// put it into the expected map
|
||||
expected.insert(i, entry);
|
||||
}
|
||||
expected
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user