heimoshuiyu/rust-synapse-compress-state
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fix clippy warnings, update dependencies (again) (#100)

Browse Source
This commit is contained in:
Jan Alexander Steffens
2022-08-03 11:52:47 +02:00
committed by GitHub
parent dd62afb3d5
commit da6271a331
6 changed files with 185 additions and 161 deletions
Download Patch File Download Diff File Expand all files Collapse all files

180
Cargo.lock generated
View File

@@ -13,15 +13,15 @@ dependencies = [
[[package]]
name = "anyhow"
version = "1.0.57"
version = "1.0.58"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "08f9b8508dccb7687a1d6c4ce66b2b0ecef467c94667de27d8d7fe1f8d2a9cdc"
checksum = "bb07d2053ccdbe10e2af2995a2f116c1330396493dc1269f6a91d0ae82e19704"
[[package]]
name = "arc-swap"
version = "1.5.0"
version = "1.5.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c5d78ce20460b82d3fa150275ed9d55e21064fc7951177baacf86a145c4a4b1f"
checksum = "983cd8b9d4b02a6dc6ffa557262eb5858a27a0038ffffe21a0f133eaa819a164"
[[package]]
name = "async-trait"
@@ -80,9 +80,9 @@ checksum = "14c189c53d098945499cdfa7ecc63567cf3886b3332b312a5b4585d8d3a6a610"
[[package]]
name = "bytes"
version = "1.1.0"
version = "1.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c4872d67bab6358e59559027aa3b9157c53d9358c51423c17554809a8858e0f8"
checksum = "ec8a7b6a70fde80372154c65702f00a0f56f3e1c36abbc6c440484be248856db"
[[package]]
name = "cc"
@@ -98,15 +98,15 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
[[package]]
name = "clap"
version = "3.1.18"
version = "3.2.16"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d2dbdf4bdacb33466e854ce889eee8dfd5729abf7ccd7664d0a2d60cd384440b"
checksum = "a3dbbb6653e7c55cc8595ad3e1f7be8f32aba4eb7ff7f0fd1163d4f3d137c0a9"
dependencies = [
"atty",
"bitflags",
"clap_lex",
"indexmap",
"lazy_static",
"once_cell",
"strsim",
"termcolor",
"textwrap",
@@ -114,9 +114,9 @@ dependencies = [
[[package]]
name = "clap_lex"
version = "0.2.0"
version = "0.2.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a37c35f1112dad5e6e0b1adaff798507497a18fceeb30cceb3bae7d1427b9213"
checksum = "2850f2f5a82cbf437dd5af4d49848fbdfc27c157c3d010345776f952765261c5"
dependencies = [
"os_str_bytes",
]
@@ -140,14 +140,15 @@ dependencies = [
[[package]]
name = "console"
version = "0.15.0"
version = "0.15.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a28b32d32ca44b70c3e4acd7db1babf555fa026e385fb95f18028f88848b3c31"
checksum = "89eab4d20ce20cea182308bca13088fecea9c05f6776cf287205d41a0ed3c847"
dependencies = [
"encode_unicode",
"libc",
"once_cell",
"terminal_size",
"unicode-width",
"winapi",
]
@@ -162,9 +163,9 @@ dependencies = [
[[package]]
name = "crossbeam-channel"
version = "0.5.4"
version = "0.5.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5aaa7bd5fb665c6864b5f963dd9097905c54125909c7aa94c9e18507cdbe6c53"
checksum = "c2dd04ddaf88237dc3b8d8f9a3c1004b506b54b3313403944054d23c0870c521"
dependencies = [
"cfg-if",
"crossbeam-utils",
@@ -172,9 +173,9 @@ dependencies = [
[[package]]
name = "crossbeam-deque"
version = "0.8.1"
version = "0.8.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6455c0ca19f0d2fbf751b908d5c55c1f5cbc65e03c4225427254b46890bdde1e"
checksum = "715e8152b692bba2d374b53d4875445368fdf21a94751410af607a5ac677d1fc"
dependencies = [
"cfg-if",
"crossbeam-epoch",
@@ -183,33 +184,33 @@ dependencies = [
[[package]]
name = "crossbeam-epoch"
version = "0.9.8"
version = "0.9.10"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1145cf131a2c6ba0615079ab6a638f7e1973ac9c2634fcbeaaad6114246efe8c"
checksum = "045ebe27666471bb549370b4b0b3e51b07f56325befa4284db65fc89c02511b1"
dependencies = [
"autocfg",
"cfg-if",
"crossbeam-utils",
"lazy_static",
"memoffset",
"once_cell",
"scopeguard",
]
[[package]]
name = "crossbeam-utils"
version = "0.8.8"
version = "0.8.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0bf124c720b7686e3c2663cf54062ab0f68a88af2fb6a030e87e30bf721fcb38"
checksum = "51887d4adc7b564537b15adcfb307936f8075dfcd5f00dde9a9f1d29383682bc"
dependencies = [
"cfg-if",
"lazy_static",
"once_cell",
]
[[package]]
name = "crypto-common"
version = "0.1.3"
version = "0.1.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "57952ca27b5e3606ff4dd79b0020231aaf9d6aa76dc05fd30137538c50bd3ce8"
checksum = "1bfb12502f3fc46cca1bb51ac28df9d618d813cdc3d2f25b9fe775a34af26bb3"
dependencies = [
"generic-array",
"typenum",
@@ -228,9 +229,9 @@ dependencies = [
[[package]]
name = "either"
version = "1.6.1"
version = "1.7.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e78d4f1cc4ae33bbfc157ed5d5a5ef3bc29227303d595861deb238fcec4e9457"
checksum = "3f107b87b6afc2a64fd13cac55fe06d6c8859f12d4b14cbcdd2c67d0976781be"
[[package]]
name = "encode_unicode"
@@ -379,20 +380,20 @@ dependencies = [
[[package]]
name = "getrandom"
version = "0.2.6"
version = "0.2.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9be70c98951c83b8d2f8f60d7065fa6d5146873094452a1008da8c2f1e4205ad"
checksum = "4eb1a864a501629691edf6c15a593b7a51eebaa1e8468e9ddc623de7c9b58ec6"
dependencies = [
"cfg-if",
"libc",
"wasi 0.10.2+wasi-snapshot-preview1",
"wasi",
]
[[package]]
name = "hashbrown"
version = "0.11.2"
version = "0.12.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ab5ef0d4909ef3724cc8cce6ccc8572c5c817592e9285f5464f8e86f8bd3726e"
checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888"
[[package]]
name = "hermit-abi"
@@ -420,9 +421,9 @@ checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4"
[[package]]
name = "indexmap"
version = "1.8.2"
version = "1.9.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e6012d540c5baa3589337a98ce73408de9b5a25ec9fc2c6fd6be8f0d39e0ca5a"
checksum = "10a35a97730320ffe8e2d410b5d3b69279b98d2c14bdb8b70ea89ecf7888d41e"
dependencies = [
"autocfg",
"hashbrown",
@@ -430,14 +431,13 @@ dependencies = [
[[package]]
name = "indicatif"
version = "0.16.2"
version = "0.17.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2d207dc617c7a380ab07ff572a6e52fa202a2a8f355860ac9c38e23f8196be1b"
checksum = "fcc42b206e70d86ec03285b123e65a5458c92027d1fb2ae3555878b8113b3ddf"
dependencies = [
"console",
"lazy_static",
"number_prefix",
"regex",
"unicode-width",
]
[[package]]
@@ -488,9 +488,9 @@ dependencies = [
[[package]]
name = "log-panics"
version = "2.0.0"
version = "2.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ae0136257df209261daa18d6c16394757c63e032e27aafd8b07788b051082bef"
checksum = "68f9dd8546191c1850ecf67d22f5ff00a935b890d0e84713159a55495cc2ac5f"
dependencies = [
"log",
]
@@ -521,13 +521,13 @@ dependencies = [
[[package]]
name = "mio"
version = "0.8.3"
version = "0.8.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "713d550d9b44d89174e066b7a6217ae06234c10cb47819a88290d2b353c31799"
checksum = "57ee1c23c7c63b0c9250c339ffdc69255f110b298b901b9f6c82547b7b87caaf"
dependencies = [
"libc",
"log",
"wasi 0.11.0+wasi-snapshot-preview1",
"wasi",
"windows-sys",
]
@@ -555,15 +555,15 @@ checksum = "830b246a0e5f20af87141b25c173cd1b609bd7779a4617d6ec582abaf90870f3"
[[package]]
name = "once_cell"
version = "1.12.0"
version = "1.13.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7709cef83f0c1f58f666e746a08b21e0085f7440fa6a29cc194d68aac97a4225"
checksum = "18a6dbe30758c9f83eb00cbea4ac95966305f5a7772f3f42ebfc7fc7eddbd8e1"
[[package]]
name = "openssl"
version = "0.10.40"
version = "0.10.41"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fb81a6430ac911acb25fe5ac8f1d2af1b4ea8a4fdfda0f1ee4292af2e2d8eb0e"
checksum = "618febf65336490dfcf20b73f885f5651a0c89c64c2d4a8c3662585a70bf5bd0"
dependencies = [
"bitflags",
"cfg-if",
@@ -587,9 +587,9 @@ dependencies = [
[[package]]
name = "openssl-sys"
version = "0.9.74"
version = "0.9.75"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "835363342df5fba8354c5b453325b110ffd54044e588c539cf2f20a8014e4cb1"
checksum = "e5f9bd0c2710541a3cda73d6f9ac4f1b240de4ae261065d309dbe73d9dceb42f"
dependencies = [
"autocfg",
"cc",
@@ -600,9 +600,9 @@ dependencies = [
[[package]]
name = "os_str_bytes"
version = "6.1.0"
version = "6.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "21326818e99cfe6ce1e524c2a805c189a99b5ae555a35d19f9a284b427d86afa"
checksum = "648001efe5d5c0102d8cea768e348da85d90af8ba91f0bea908f157951493cd4"
[[package]]
name = "parking_lot"
@@ -764,9 +764,9 @@ checksum = "925383efa346730478fb4838dbe9137d2a47675ad789c546d150a6e1dd4ab31c"
[[package]]
name = "proc-macro2"
version = "1.0.39"
version = "1.0.42"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c54b25569025b7fc9651de43004ae593a75ad88543b17178aa5e1b9c4f15f56f"
checksum = "c278e965f1d8cf32d6e0e96de3d3e79712178ae67986d9cf9151f51e95aac89b"
dependencies = [
"unicode-ident",
]
@@ -843,9 +843,9 @@ dependencies = [
[[package]]
name = "quote"
version = "1.0.18"
version = "1.0.20"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a1feb54ed693b93a84e14094943b84b7c4eae204c512b7ccb95ab0c66d278ad1"
checksum = "3bcdf212e9776fbcb2d23ab029360416bb1706b1aea2d1a5ba002727cbcab804"
dependencies = [
"proc-macro2",
]
@@ -906,18 +906,18 @@ dependencies = [
[[package]]
name = "redox_syscall"
version = "0.2.13"
version = "0.2.16"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "62f25bc4c7e55e0b0b7a1d43fb893f4fa1361d0abe38b9ce4f323c2adfe6ef42"
checksum = "fb5a58c1855b4b6819d59012155603f0b22ad30cad752600aadfcb695265519a"
dependencies = [
"bitflags",
]
[[package]]
name = "regex"
version = "1.5.6"
version = "1.6.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d83f127d94bdbcda4c8cc2e50f6f84f4b611f69c902699ca385a39c3a75f9ff1"
checksum = "4c4eb3267174b8c6c2f654116623910a0fef09c4753f8dd83db29c48a0df988b"
dependencies = [
"aho-corasick",
"memchr",
@@ -926,9 +926,9 @@ dependencies = [
[[package]]
name = "regex-syntax"
version = "0.6.26"
version = "0.6.27"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "49b3de9ec5dc0a3417da371aab17d729997c15010e7fd24ff707773a33bddb64"
checksum = "a3f87b73ce11b1619a3c6332f45341e0047173771e8b8b73f87bfeefb7b56244"
[[package]]
name = "scopeguard"
@@ -938,9 +938,9 @@ checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd"
[[package]]
name = "serde"
version = "1.0.137"
version = "1.0.140"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "61ea8d54c77f8315140a05f4c7237403bf38b72704d031543aa1d16abbf517d1"
checksum = "fc855a42c7967b7c369eb5860f7164ef1f6f81c20c7cc1141f2a604e18723b03"
[[package]]
name = "serial_test"
@@ -983,15 +983,18 @@ checksum = "7bd3e3206899af3f8b12af284fafc038cc1dc2b41d1b89dd17297221c5d225de"
[[package]]
name = "slab"
version = "0.4.6"
version = "0.4.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "eb703cfe953bccee95685111adeedb76fabe4e97549a58d16f03ea7b9367bb32"
checksum = "4614a76b2a8be0058caa9dbbaf66d988527d86d003c11a94fbd335d7661edcef"
dependencies = [
"autocfg",
]
[[package]]
name = "smallvec"
version = "1.8.0"
version = "1.9.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f2dd574626839106c320a323308629dcb1acfc96e32a8cba364ddc61ac23ee83"
checksum = "2fd0db749597d91ff862fd1d55ea87f7855a744a8425a64695b6fca237d1dad1"
[[package]]
name = "socket2"
@@ -1046,9 +1049,9 @@ checksum = "6bdef32e8150c2a081110b42772ffe7d7c9032b606bc226c8260fd97e0976601"
[[package]]
name = "syn"
version = "1.0.96"
version = "1.0.98"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0748dd251e24453cb8717f0354206b91557e4ec8703673a4b30208f2abaf1ebf"
checksum = "c50aef8a904de4c23c788f104b7dddc7d6f79c647c7c8ce4cc8f73eb0ca773dd"
dependencies = [
"proc-macro2",
"quote",
@@ -1129,9 +1132,9 @@ checksum = "b1141d4d61095b28419e22cb0bbf02755f5e54e0526f97f1e3d1d160e60885fb"
[[package]]
name = "tikv-jemalloc-sys"
version = "0.5.0+5.3.0"
version = "0.5.1+5.3.0-patched"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "aeab4310214fe0226df8bfeb893a291a58b19682e8a07e1e1d4483ad4200d315"
checksum = "931e876f91fed0827f863a2d153897790da0b24d882c721a79cb3beb0b903261"
dependencies = [
"cc",
"fs_extra",
@@ -1165,10 +1168,11 @@ checksum = "cda74da7e1a664f795bb1f8a87ec406fb89a02522cf6e50620d016add6dbbf5c"
[[package]]
name = "tokio"
version = "1.19.1"
version = "1.20.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "95eec79ea28c00a365f539f1961e9278fbcaf81c0ff6aaf0e93c181352446948"
checksum = "7a8325f63a7d4774dd041e363b2409ed1c5cbbd0f867795e661df066b2b0a581"
dependencies = [
"autocfg",
"bytes",
"libc",
"memchr",
@@ -1230,9 +1234,9 @@ dependencies = [
[[package]]
name = "tracing"
version = "0.1.34"
version = "0.1.36"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5d0ecdcb44a79f0fe9844f0c4f33a342cbcbb5117de8001e6ba0dc2351327d09"
checksum = "2fce9567bd60a67d08a16488756721ba392f24f29006402881e43b19aac64307"
dependencies = [
"cfg-if",
"pin-project-lite",
@@ -1241,11 +1245,11 @@ dependencies = [
[[package]]
name = "tracing-core"
version = "0.1.26"
version = "0.1.29"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f54c8ca710e81886d498c2fd3331b56c93aa248d49de2222ad2742247c60072f"
checksum = "5aeea4303076558a00714b823f9ad67d58a3bbda1df83d8827d21193156e22f7"
dependencies = [
"lazy_static",
"once_cell",
]
[[package]]
@@ -1262,19 +1266,25 @@ checksum = "099b7128301d285f79ddd55b9a83d5e6b9e97c92e0ea0daebee7263e932de992"
[[package]]
name = "unicode-ident"
version = "1.0.0"
version = "1.0.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d22af068fba1eb5edcb4aea19d382b2a3deb4c8f9d475c589b6ada9e0fd493ee"
checksum = "15c61ba63f9235225a22310255a29b806b907c9b8c964bcbd0a2c70f3f2deea7"
[[package]]
name = "unicode-normalization"
version = "0.1.19"
version = "0.1.21"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d54590932941a9e9266f0832deed84ebe1bf2e4c9e4a3554d393d18f5e854bf9"
checksum = "854cbdc4f7bc6ae19c820d44abdc3277ac3e1b2b93db20a636825d9322fb60e6"
dependencies = [
"tinyvec",
]
[[package]]
name = "unicode-width"
version = "0.1.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3ed742d4ea2bd1176e236172c8429aaf54486e7ac098db29ffe6529e0ce50973"
[[package]]
name = "unindent"
version = "0.1.9"
@@ -1293,12 +1303,6 @@ version = "0.9.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f"
[[package]]
name = "wasi"
version = "0.10.2+wasi-snapshot-preview1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fd6fbd9a79829dd1ad0cc20627bf1ed606756a7f77edff7b66b7064f9cb327c6"
[[package]]
name = "wasi"
version = "0.11.0+wasi-snapshot-preview1"

2
Cargo.toml
View File

@@ -9,7 +9,7 @@ version = "0.1.0"
edition = "2018"
[dependencies]
indicatif = "0.16.0"
indicatif = "0.17.0"
openssl = "0.10.32"
postgres = "0.19.0"
postgres-openssl = "0.5.0"

64
compressor_integration_tests/src/lib.rs
View File

@@ -4,7 +4,12 @@ use postgres::{fallible_iterator::FallibleIterator, Client};
use postgres_openssl::MakeTlsConnector;
use rand::{distributions::Alphanumeric, thread_rng, Rng};
use state_map::StateMap;
use std::{borrow::Cow, collections::BTreeMap, env, fmt};
use std::{
borrow::Cow,
collections::BTreeMap,
env,
fmt::{self, Write as _},
};
use string_cache::DefaultAtom as Atom;
use synapse_compress_state::StateGroupEntry;
@@ -23,47 +28,48 @@ pub fn add_contents_to_database(room_id: &str, state_group_map: &BTreeMap<i64, S
let mut client = Client::connect(DB_URL, connector).unwrap();
// build up the query
let mut sql = "".to_string();
let mut sql = String::new();
let room_id = PGEscape(room_id);
let event_id = PGEscape("left_blank");
for (sg, entry) in state_group_map {
// create the entry for state_groups
sql.push_str(&format!(
"INSERT INTO state_groups (id, room_id, event_id) VALUES ({},{},{});\n",
sg,
PGEscape(room_id),
PGEscape("left_blank")
));
writeln!(
sql,
"INSERT INTO state_groups (id, room_id, event_id) \
VALUES ({sg}, {room_id}, {event_id});",
)
.expect("Writing to a String cannot fail");
// create the entry in state_group_edges IF exists
if let Some(prev_sg) = entry.prev_state_group {
sql.push_str(&format!(
"INSERT INTO state_group_edges (state_group, prev_state_group) VALUES ({}, {});\n",
sg, prev_sg
));
writeln!(
sql,
"INSERT INTO state_group_edges (state_group, prev_state_group) \
VALUES ({sg}, {prev_sg});",
)
.unwrap();
}
// write entry for each row in delta
if !entry.state_map.is_empty() {
sql.push_str("INSERT INTO state_groups_state (state_group, room_id, type, state_key, event_id) VALUES");
sql.push_str(
"INSERT INTO state_groups_state \
(state_group, room_id, type, state_key, event_id) \
VALUES\n",
);
let mut first = true;
for ((t, s), e) in entry.state_map.iter() {
if first {
sql.push_str(" ");
first = false;
} else {
sql.push_str(" ,");
}
sql.push_str(&format!(
"({}, {}, {}, {}, {})",
sg,
PGEscape(room_id),
PGEscape(t),
PGEscape(s),
PGEscape(e)
));
let t = PGEscape(t);
let s = PGEscape(s);
let e = PGEscape(e);
writeln!(sql, " ({sg}, {room_id}, {t}, {s}, {e}),").unwrap();
}
sql.push_str(";\n");
// Replace the last comma with a semicolon
sql.replace_range((sql.len() - 2).., ";\n");
}
}

8
src/compressor.rs
View File

@@ -30,7 +30,7 @@
use indicatif::{ProgressBar, ProgressStyle};
use state_map::StateMap;
use std::collections::BTreeMap;
use std::{collections::BTreeMap, time::Duration};
use string_cache::DefaultAtom as Atom;
use super::{collapse_state_maps, StateGroupEntry};
@@ -187,10 +187,12 @@ impl<'a> Compressor<'a> {
ProgressBar::new(self.original_state_map.len() as u64)
};
pb.set_style(
ProgressStyle::default_bar().template("[{elapsed_precise}] {bar} {pos}/{len} {msg}"),
ProgressStyle::default_bar()
.template("[{elapsed_precise}] {bar} {pos}/{len} {msg}")
.unwrap(),
);
pb.set_message("state groups");
pb.enable_steady_tick(100);
pb.enable_steady_tick(Duration::from_millis(100));
for (&state_group, entry) in self.original_state_map {
// Check whether this entry is in_range or is just present in the map due to being

14
src/database.rs
View File

@@ -18,7 +18,7 @@ use openssl::ssl::{SslConnector, SslMethod, SslVerifyMode};
use postgres::{fallible_iterator::FallibleIterator, types::ToSql, Client};
use postgres_openssl::MakeTlsConnector;
use rand::{distributions::Alphanumeric, thread_rng, Rng};
use std::{borrow::Cow, collections::BTreeMap, fmt};
use std::{borrow::Cow, collections::BTreeMap, fmt, time::Duration};
use crate::{compressor::Level, generate_sql};
@@ -378,9 +378,11 @@ fn get_initial_data_from_db(
ProgressBar::new_spinner()
};
pb.set_style(
ProgressStyle::default_spinner().template("{spinner} [{elapsed}] {pos} rows retrieved"),
ProgressStyle::default_spinner()
.template("{spinner} [{elapsed}] {pos} rows retrieved")
.unwrap(),
);
pb.enable_steady_tick(100);
pb.enable_steady_tick(Duration::from_millis(100));
while let Some(row) = rows.next().unwrap() {
// The row in the map to copy the data to
@@ -542,10 +544,12 @@ pub fn send_changes_to_db(
ProgressBar::new(old_map.len() as u64)
};
pb.set_style(
ProgressStyle::default_bar().template("[{elapsed_precise}] {bar} {pos}/{len} {msg}"),
ProgressStyle::default_bar()
.template("[{elapsed_precise}] {bar} {pos}/{len} {msg}")
.unwrap(),
);
pb.set_message("state groups");
pb.enable_steady_tick(100);
pb.enable_steady_tick(Duration::from_millis(100));
for sql_transaction in generate_sql(old_map, new_map, room_id) {
if sql_transaction.is_empty() {

78
src/lib.rs
View File

@@ -27,7 +27,10 @@ use clap::{crate_authors, crate_description, crate_name, crate_version, Arg, Com
use indicatif::{ProgressBar, ProgressStyle};
use rayon::prelude::*;
use state_map::StateMap;
use std::{collections::BTreeMap, convert::TryInto, fs::File, io::Write, str::FromStr};
use std::{
collections::BTreeMap, convert::TryInto, fmt::Write as _, fs::File, io::Write, str::FromStr,
time::Duration,
};
use string_cache::DefaultAtom as Atom;
mod compressor;
@@ -424,8 +427,7 @@ fn generate_sql<'a>(
new_map: &'a BTreeMap<i64, StateGroupEntry>,
room_id: &'a str,
) -> impl Iterator<Item = String> + 'a {
old_map.iter().map(move |(sg,old_entry)| {
old_map.iter().map(move |(sg, old_entry)| {
let new_entry = &new_map[sg];
// Check if the new map has a different entry for this state group
@@ -435,48 +437,50 @@ fn generate_sql<'a>(
let mut sql = String::new();
// remove the current edge
sql.push_str(&format!(
"DELETE FROM state_group_edges WHERE state_group = {};\n",
sg
));
writeln!(
sql,
"DELETE FROM state_group_edges WHERE state_group = {sg};",
)
.expect("Writing to a String cannot fail");
// if the new entry has a predecessor then put that into state_group_edges
if let Some(prev_sg) = new_entry.prev_state_group {
sql.push_str(&format!("INSERT INTO state_group_edges (state_group, prev_state_group) VALUES ({}, {});\n", sg, prev_sg));
writeln!(
sql,
"INSERT INTO state_group_edges (state_group, prev_state_group) \
VALUES ({sg}, {prev_sg});",
)
.unwrap();
}
// remove the current deltas for this state group
sql.push_str(&format!(
"DELETE FROM state_groups_state WHERE state_group = {};\n",
sg
));
writeln!(
sql,
"DELETE FROM state_groups_state WHERE state_group = {sg};",
)
.unwrap();
if !new_entry.state_map.is_empty() {
// place all the deltas for the state group in the new map into state_groups_state
sql.push_str("INSERT INTO state_groups_state (state_group, room_id, type, state_key, event_id) VALUES\n");
sql.push_str(
"INSERT INTO state_groups_state \
(state_group, room_id, type, state_key, event_id) \
VALUES\n",
);
let mut first = true;
let room_id = PGEscape(room_id);
for ((t, s), e) in new_entry.state_map.iter() {
// Add a comma at the start if not the first row to be inserted
if first {
sql.push_str(" ");
first = false;
} else {
sql.push_str(" ,");
}
let t = PGEscape(t);
let s = PGEscape(s);
let e = PGEscape(e);
// write the row to be insterted of the form:
// write the row to be inserted of the form:
// (state_group, room_id, type, state_key, event_id)
sql.push_str(&format!(
"({}, {}, {}, {}, {})",
sg,
PGEscape(room_id),
PGEscape(t),
PGEscape(s),
PGEscape(e)
));
writeln!(sql, " ({sg}, {room_id}, {t}, {s}, {e}),").unwrap();
}
sql.push_str(";\n");
// Replace the last comma with a semicolon
sql.replace_range((sql.len() - 2).., ";\n");
}
sql
@@ -514,10 +518,12 @@ fn output_sql(
ProgressBar::new(old_map.len() as u64)
};
pb.set_style(
ProgressStyle::default_bar().template("[{elapsed_precise}] {bar} {pos}/{len} {msg}"),
ProgressStyle::default_bar()
.template("[{elapsed_precise}] {bar} {pos}/{len} {msg}")
.unwrap(),
);
pb.set_message("state groups");
pb.enable_steady_tick(100);
pb.enable_steady_tick(Duration::from_millis(100));
if let Some(output) = &mut config.output_file {
for mut sql_transaction in generate_sql(old_map, new_map, &config.room_id) {
@@ -628,10 +634,12 @@ fn check_that_maps_match(
ProgressBar::new(old_map.len() as u64)
};
pb.set_style(
ProgressStyle::default_bar().template("[{elapsed_precise}] {bar} {pos}/{len} {msg}"),
ProgressStyle::default_bar()
.template("[{elapsed_precise}] {bar} {pos}/{len} {msg}")
.unwrap(),
);
pb.set_message("state groups");
pb.enable_steady_tick(100);
pb.enable_steady_tick(Duration::from_millis(100));
// Now let's iterate through and assert that the state for each group
// matches between the two versions.