Fix clippy lints

PR #74 (Log to stderr not to a file) did not remove all references to
log_file, the file was still created, but remained empty.

The synapse_auto_compressor failed if run in a read only environment.

Signed-off-by: saces <saces@c-base.org>

.dockerignore

@@ -0,0 +1,3 @@
+.git
+.github
+/target

.github/CODEOWNERS

@@ -0,0 +1,2 @@
+# Automatically request reviews from the synapse-core team when a pull request comes in.
+* @matrix-org/synapse-core

.github/workflows/tests.yaml

@@ -0,0 +1,74 @@
+on:
+  pull_request:
+  push:
+    branches:
+      - main
+
+name: Continuous integration
+
+jobs:
+  check:
+    name: Cargo Check
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions-rs/toolchain@v1
+        with:
+          profile: minimal
+          toolchain: stable
+          override: true
+      - uses: Swatinem/rust-cache@v1
+      - uses: actions-rs/cargo@v1
+        with:
+          command: check
+
+  test:
+    name: Test Suite
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions-rs/toolchain@v1
+        with:
+          profile: minimal
+          toolchain: stable
+          override: true
+      - uses: Swatinem/rust-cache@v1
+      - run: cd compressor_integration_tests && docker-compose up -d
+      - uses: actions-rs/cargo@v1
+        with:
+          command: test
+          args: --workspace
+
+  fmt:
+    name: Rustfmt
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions-rs/toolchain@v1
+        with:
+          profile: minimal
+          toolchain: stable
+          override: true
+          components: rustfmt
+      - uses: Swatinem/rust-cache@v1
+      - uses: actions-rs/cargo@v1
+        with:
+          command: fmt
+          args: --all -- --check
+
+  clippy:
+    name: Clippy
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions-rs/toolchain@v1
+        with:
+          profile: minimal
+          toolchain: stable
+          override: true
+          components: clippy
+      - uses: Swatinem/rust-cache@v1
+      - uses: actions-rs/cargo@v1
+        with:
+          command: clippy
+          args: -- -D warnings

.gitignore

@@ -0,0 +1,7 @@
+/target
+**/*.rs.bk
+*.data
+*.old
+**.sql
+*.csv
+**.log

.vscode/settings.json

@@ -0,0 +1,3 @@
+{
+    "rust-analyzer.checkOnSave.command": "clippy"
+}

Cargo.toml

@@ -0,0 +1,43 @@
+[workspace]
+members = ["synapse_auto_compressor", "compressor_integration_tests"]
+
+[package]
+authors = ["Erik Johnston"]
+description = "A tool to compress some state in a Synapse instance's database"
+name = "synapse_compress_state"
+version = "0.1.0"
+edition = "2018"
+
+[dependencies]
+clap = "2.33.0"
+indicatif = "0.16.0"
+openssl = "0.10.32"
+postgres = "0.19.0"
+postgres-openssl = "0.5.0"
+rand = "0.8.0"
+rayon = "1.3.0"
+string_cache = "0.8.0"
+env_logger = "0.9.0"
+log = "0.4.14"
+pyo3-log = "0.4.0"
+log-panics = "2.0.0"
+
+[dependencies.state-map]
+git = "https://github.com/matrix-org/rust-matrix-state-map"
+
+# Needed for pyo3 support
+[lib]
+crate-type = ["cdylib", "rlib"]
+
+[dependencies.pyo3]
+version = "0.14.1"
+features = ["extension-module","abi3-py36"]
+
+[dependencies.tikv-jemallocator]
+version = "0.4.1"
+optional = true
+
+[features]
+default = ["jemalloc"]
+jemalloc = ["tikv-jemallocator"]
+no-progress-bars = []

Dockerfile

@@ -0,0 +1,22 @@
+FROM rust:alpine AS builder
+
+RUN apk add python3 musl-dev pkgconfig openssl-dev make
+
+ENV RUSTFLAGS="-C target-feature=-crt-static"
+
+WORKDIR /opt/synapse-compressor/
+
+COPY . .
+
+RUN cargo build
+
+WORKDIR /opt/synapse-compressor/synapse_auto_compressor/
+
+RUN cargo build
+
+FROM alpine
+
+RUN apk add --no-cache libgcc
+
+COPY --from=builder /opt/synapse-compressor/target/debug/synapse_compress_state /usr/local/bin/synapse_compress_state
+COPY --from=builder /opt/synapse-compressor/target/debug/synapse_auto_compressor /usr/local/bin/synapse_auto_compressor

LICENSE

@@ -0,0 +1,201 @@
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

README.md

@@ -0,0 +1,274 @@
+# Compress Synapse State Tables
+
+This workspace contains experimental tools that attempt to reduce the number of
+rows in the `state_groups_state` table inside of a Synapse Postgresql database.
+
+# Automated tool: synapse_auto_compressor
+
+## Introduction:
+
+This tool is significantly more simple to use than the manual tool (described below).
+It scans through all of the rows in the `state_groups` database table from the start. When
+it finds a group that hasn't been compressed, it runs the compressor for a while on that
+group's room, saving where it got up to. After compressing a number of these chunks it stops,
+saving where it got up to for the next run of the `synapse_auto_compressor`.
+
+It creates three extra tables in the database: `state_compressor_state` which stores the
+information needed to stop and start the compressor for each room, `state_compressor_progress`
+which stores the most recently compressed state group for each room and `state_compressor_total_progress`
+which stores how far through the `state_groups` table the compressor has scanned.
+
+The tool can be run manually when you are running out of space, or be scheduled to run
+periodically.
+
+## Building
+
+This tool requires `cargo` to be installed. See https://www.rust-lang.org/tools/install
+for instructions on how to do this.
+
+To build `synapse_auto_compressor`, clone this repository and navigate to the
+`synapse_auto_compressor/` subdirectory. Then execute `cargo build`.
+
+This will create an executable and store it in
+`synapse_auto_compressor/target/debug/synapse_auto_compressor`.
+
+## Example usage
+```
+$ synapse_auto_compressor -p postgresql://user:pass@localhost/synapse -c 500 -n 100
+```
+## Running Options
+
+- -p [POSTGRES_LOCATION] **Required**
+The configuration for connecting to the Postgres database. This should be of the form
+`"postgresql://username:password@mydomain.com/database"` or a key-value pair
+string: `"user=username password=password dbname=database host=mydomain.com"`
+See https://docs.rs/tokio-postgres/0.7.2/tokio_postgres/config/struct.Config.html
+for the full details.
+
+- -c [CHUNK_SIZE] **Required**
+The number of state groups to work on at once. All of the entries from state_groups_state are
+requested from the database for state groups that are worked on. Therefore small chunk
+sizes may be needed on machines with low memory. Note: if the compressor fails to find
+space savings on the chunk as a whole (which may well happen in rooms with lots of backfill
+in) then the entire chunk is skipped.
+
+- -n [CHUNKS_TO_COMPRESS] **Required**
+*CHUNKS_TO_COMPRESS* chunks of size *CHUNK_SIZE* will be compressed. The higher this
+number is set to, the longer the compressor will run for.
+
+- -d [LEVELS]
+Sizes of each new level in the compression algorithm, as a comma-separated list.
+The first entry in the list is for the lowest, most granular level, with each
+subsequent entry being for the next highest level. The number of entries in the
+list determines the number of levels that will be used. The sum of the sizes of
+the levels affects the performance of fetching the state from the database, as the
+sum of the sizes is the upper bound on the number of iterations needed to fetch a
+given set of state. [defaults to "100,50,25"]
+
+## Scheduling the compressor
+The automatic tool may put some strain on the database, so it might be best to schedule
+it to run at a quiet time for the server. This could be done by creating an executable
+script and scheduling it with something like
+[cron](https://www.man7.org/linux/man-pages/man1/crontab.1.html).
+
+# Manual tool: synapse_compress_state
+
+## Introduction
+
+A manual tool that reads in the rows from `state_groups_state` and `state_group_edges`
+tables for a specified room and calculates the changes that could be made that
+(hopefully) will significantly reduce the number of rows.
+
+This tool currently *does not* write to the database by default, so should be
+safe to run. If the `-o` option is specified then SQL will be written to the
+given file that would change the tables to match the calculated state. (Note
+that if `-t` is given then each change to a particular state group is wrapped
+in a transaction). If you do wish to send the changes to the database automatically
+then the `-c` flag can be set.
+
+The SQL generated is safe to apply against the database with Synapse running.
+This is because the `state_groups` and `state_groups_state` tables are append-only:
+once written to the database, they are never modified. There is therefore no danger
+of a modification racing against a running Synapse. Further, this script makes its
+changes within atomic transactions, and each transaction should not affect the results
+from any of the queries that Synapse performs.
+
+The tool will also ensure that the generated state deltas do give the same state
+as the existing state deltas before generating any SQL.
+
+## Building
+
+This tool requires `cargo` to be installed. See https://www.rust-lang.org/tools/install
+for instructions on how to do this.
+
+To build `synapse_compress_state`, clone this repository and then execute `cargo build`.
+
+This will create an executable and store it in `target/debug/synapse_compress_state`.
+
+## Example usage
+
+```
+$ synapse_compress_state -p "postgresql://localhost/synapse" -r '!some_room:example.com' -o out.sql -t
+Fetching state from DB for room '!some_room:example.com'...
+Got initial state from database. Checking for any missing state groups...
+Number of state groups: 73904
+Number of rows in current table: 2240043
+Number of rows after compression: 165754 (7.40%)
+Compression Statistics:
+  Number of forced resets due to lacking prev: 34
+  Number of compressed rows caused by the above: 17092
+  Number of state groups changed: 2748
+New state map matches old one
+
+# It's finished, so we can now go and rewrite the DB
+$ psql synapse < out.data
+```
+
+## Running Options
+
+- -p [POSTGRES_LOCATION] **Required**
+The configuration for connecting to the Postgres database. This should be of the form
+`"postgresql://username:password@mydomain.com/database"` or a key-value pair
+string: `"user=username password=password dbname=database host=mydomain.com"`
+See https://docs.rs/tokio-postgres/0.7.2/tokio_postgres/config/struct.Config.html
+for the full details.
+
+- -r [ROOM_ID] **Required**
+The room to process (this is the value found in the `rooms` table of the database
+not the common name for the room - it should look like: "!wOlkWNmgkAZFxbTaqj:matrix.org".
+
+- -b [MIN_STATE_GROUP]
+The state group to start processing from (non-inclusive).
+
+- -n [GROUPS_TO_COMPRESS]
+How many groups to load into memory to compress (starting
+from the 1st group in the room or the group specified by -b).
+
+- -l [LEVELS]
+Sizes of each new level in the compression algorithm, as a comma-separated list.
+The first entry in the list is for the lowest, most granular level, with each
+subsequent entry being for the next highest level. The number of entries in the
+list determines the number of levels that will be used. The sum of the sizes of
+the levels affects the performance of fetching the state from the database, as the
+sum of the sizes is the upper bound on the number of iterations needed to fetch a
+given set of state. [defaults to "100,50,25"]
+
+- -m [COUNT]
+If the compressor cannot save this many rows from the database then it will stop early.
+
+- -s [MAX_STATE_GROUP]
+If a max_state_group is specified then only state groups with id's lower than this
+number can be compressed.
+
+- -o [FILE]
+File to output the SQL transactions to (for later running on the database).
+
+- -t
+If this flag is set then each change to a particular state group is wrapped in a
+transaction. This should be done if you wish to apply the changes while synapse is
+still running.
+
+- -c
+If this flag is set then the changes the compressor makes will be committed to the
+database. This should be safe to use while synapse is running as it wraps the changes
+to every state group in it's own transaction (as if the transaction flag was set).
+
+- -g
+If this flag is set then output the node and edge information for the state_group
+directed graph built up from the predecessor state_group links. These can be looked
+at in something like Gephi (https://gephi.org).
+
+
+# Running tests
+
+There are integration tests for these tools stored in `compressor_integration_tests/`.
+
+To run the integration tests, you first need to start up a Postgres database
+for the library to talk to. There is a docker-compose file that sets one up
+with all of the correct tables. The tests can therefore be run as follows:
+
+```
+$ cd compressor_integration_tests/
+$ docker-compose up -d
+$ cargo test --workspace
+$ docker-compose down
+```
+
+# Using the synapse_compress_state library
+
+If you want to use the compressor in another project, it is recomended that you
+use jemalloc `https://github.com/tikv/jemallocator`.
+
+To prevent the progress bars from being shown, use the `no-progress-bars` feature.
+(See `synapse_auto_compressor/Cargo.toml` for an example)
+
+# Troubleshooting
+
+## Connecting to database
+
+### From local machine
+
+If you setup Synapse using the instructions on https://matrix-org.github.io/synapse/latest/postgres.html
+you should have a username and password to use to login to the postgres database. To run the compressor
+from the machine where Postgres is running, the url will be the following:
+
+`postgresql://synapse_user:synapse_password@localhost/synapse`
+
+### From remote machine
+
+If you wish to connect from a different machine, you'll need to edit your Postgres settings to allow
+remote connections. This requires updating the
+[`pg_hba.conf`](https://www.postgresql.org/docs/current/auth-pg-hba-conf.html) and the `listen_addresses`
+setting in [`postgresql.conf`](https://www.postgresql.org/docs/current/runtime-config-connection.html)
+
+## Printing debugging logs
+
+The amount of output the tools produce can be altered by setting the RUST_LOG
+environment variable to something.
+
+To get more logs when running the synapse_auto_compressor tool try the following:
+
+```
+$ RUST_LOG=debug synapse_auto_compressor -p postgresql://user:pass@localhost/synapse -c 50 -n 100
+```
+
+If you want to suppress all the debugging info you are getting from the
+Postgres client then try:
+
+```
+RUST_LOG=synapse_auto_compressor=debug,synapse_compress_state=debug synapse_auto_compressor [etc.]
+```
+
+This will only print the debugging information from those two packages. For more info see
+https://docs.rs/env_logger/0.9.0/env_logger/.
+
+## Building difficulties
+
+Building the `openssl-sys` dependency crate requires OpenSSL development tools to be installed,
+and building on Linux will also require `pkg-config`
+
+This can be done on Ubuntu  with: `$ apt-get install libssl-dev pkg-config`
+
+Note that building requires quite a lot of memory and out-of-memory errors might not be
+obvious. It's recomended you only build these tools on machines with at least 2GB of RAM.
+
+## Auto Compressor skips chunks when running on already compressed room
+
+If you have used the compressor before, with certain config options, the automatic tool will
+produce lots of warnings of the form: `The compressor tried to increase the number of rows in ...`
+
+To fix this, ensure that the chunk_size is set to at least the L1 level size (so if the level
+sizes are "100,50,25" then the chunk_size should be at least 100).
+
+Note: if the level sizes being used when rerunning are different to when run previously
+this might lead to less efficient compression and thus chunks being skipped, but this shouldn't
+be a large problem.
+
+## Compressor is trying to increase the number of rows
+
+Backfilling can lead to issues with compression. The synapse_auto_compressor will
+skip chunks it can't reduce the size of and so this should help jump over the backfilled
+state_groups. Lots of state resolution might also impact the ability to use the compressor.
+
+To examine the state_group hierarchy run the manual tool on a room with the `-g` option
+and look at the graphs.

compressor_integration_tests/Cargo.toml

@@ -0,0 +1,21 @@
+[package]
+name = "compressor_integration_tests"
+version = "0.1.0"
+edition = "2018"
+
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+
+[dependencies]
+string_cache = "0.8.0"
+serial_test = "0.5.1"
+openssl = "0.10.32"
+postgres = "0.19.0"
+postgres-openssl = "0.5.0"
+rand = "0.8.0"
+synapse_compress_state = { path = "../", features = ["no-progress-bars"] }
+synapse_auto_compressor = { path = "../synapse_auto_compressor/" }
+env_logger = "0.9.0"
+log = "0.4.14"
+
+[dependencies.state-map]
+git = "https://github.com/matrix-org/rust-matrix-state-map"

compressor_integration_tests/database_setup.sh

@@ -0,0 +1,28 @@
+#!/bin/sh
+
+#N.B. the database setup comes from:
+#https://github.com/matrix-org/synapse/blob/develop/synapse/storage/schema/state/full_schemas/54/full.sql
+
+# Setup the required tables for testing
+psql --username "$POSTGRES_USER" --dbname "$POSTGRES_DB" <<SQLCODE
+
+CREATE TABLE state_groups (
+    id BIGINT PRIMARY KEY,
+    room_id TEXT NOT NULL,
+    event_id TEXT NOT NULL
+);
+
+CREATE TABLE state_groups_state (
+    state_group BIGINT NOT NULL,
+    room_id TEXT NOT NULL,
+    type TEXT NOT NULL,
+    state_key TEXT NOT NULL,
+    event_id TEXT NOT NULL
+);
+
+CREATE TABLE state_group_edges (
+    state_group BIGINT NOT NULL,
+    prev_state_group BIGINT NOT NULL
+);
+
+SQLCODE

compressor_integration_tests/docker-compose.yaml

@@ -0,0 +1,20 @@
+version: '3'
+services:
+  postgres:
+    image: "postgres:latest"
+    
+    ports:
+      # N.B. format is [port on machine]:[port to expose from container]
+      - 5432:5432
+
+    environment: 
+      POSTGRES_USER: synapse_user
+      POSTGRES_PASSWORD: synapse_pass
+      POSTGRES_DB: synapse
+      PGDATA: /tmp/data
+
+    volumes:
+      - ./database_setup.sh:/docker-entrypoint-initdb.d/1_database_setup.sh 
+
+    tmpfs:
+      /tmp/data

compressor_integration_tests/src/lib.rs

@@ -0,0 +1,380 @@
+use log::LevelFilter;
+use openssl::ssl::{SslConnector, SslMethod, SslVerifyMode};
+use postgres::{fallible_iterator::FallibleIterator, Client};
+use postgres_openssl::MakeTlsConnector;
+use rand::{distributions::Alphanumeric, thread_rng, Rng};
+use state_map::StateMap;
+use std::{borrow::Cow, collections::BTreeMap, env, fmt};
+use string_cache::DefaultAtom as Atom;
+
+use synapse_compress_state::StateGroupEntry;
+
+pub mod map_builder;
+
+pub static DB_URL: &str = "postgresql://synapse_user:synapse_pass@localhost/synapse";
+
+/// Adds the contents of a state group map to the testing database
+pub fn add_contents_to_database(room_id: &str, state_group_map: &BTreeMap<i64, StateGroupEntry>) {
+    // connect to the database
+    let mut builder = SslConnector::builder(SslMethod::tls()).unwrap();
+    builder.set_verify(SslVerifyMode::NONE);
+    let connector = MakeTlsConnector::new(builder.build());
+
+    let mut client = Client::connect(DB_URL, connector).unwrap();
+
+    // build up the query
+    let mut sql = "".to_string();
+
+    for (sg, entry) in state_group_map {
+        // create the entry for state_groups
+        sql.push_str(&format!(
+            "INSERT INTO state_groups (id, room_id, event_id) VALUES ({},{},{});\n",
+            sg,
+            PGEscape(room_id),
+            PGEscape("left_blank")
+        ));
+
+        // create the entry in state_group_edges IF exists
+        if let Some(prev_sg) = entry.prev_state_group {
+            sql.push_str(&format!(
+                "INSERT INTO state_group_edges (state_group, prev_state_group) VALUES ({}, {});\n",
+                sg, prev_sg
+            ));
+        }
+
+        // write entry for each row in delta
+        if !entry.state_map.is_empty() {
+            sql.push_str("INSERT INTO state_groups_state (state_group, room_id, type, state_key, event_id) VALUES");
+
+            let mut first = true;
+            for ((t, s), e) in entry.state_map.iter() {
+                if first {
+                    sql.push_str("     ");
+                    first = false;
+                } else {
+                    sql.push_str("    ,");
+                }
+                sql.push_str(&format!(
+                    "({}, {}, {}, {}, {})",
+                    sg,
+                    PGEscape(room_id),
+                    PGEscape(t),
+                    PGEscape(s),
+                    PGEscape(e)
+                ));
+            }
+            sql.push_str(";\n");
+        }
+    }
+
+    client.batch_execute(&sql).unwrap();
+}
+
+/// Clears the contents of the testing database
+pub fn empty_database() {
+    // connect to the database
+    let mut builder = SslConnector::builder(SslMethod::tls()).unwrap();
+    builder.set_verify(SslVerifyMode::NONE);
+    let connector = MakeTlsConnector::new(builder.build());
+
+    let mut client = Client::connect(DB_URL, connector).unwrap();
+
+    // delete all the contents from all three tables
+    let sql = r"
+        TRUNCATE state_groups;
+        TRUNCATE state_group_edges;
+        TRUNCATE state_groups_state;
+    ";
+
+    client.batch_execute(sql).unwrap();
+}
+
+/// Safely escape the strings in sql queries
+struct PGEscape<'a>(pub &'a str);
+
+impl<'a> fmt::Display for PGEscape<'a> {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        let mut delim = Cow::from("$$");
+        while self.0.contains(&delim as &str) {
+            let s: String = thread_rng()
+                .sample_iter(&Alphanumeric)
+                .take(10)
+                .map(char::from)
+                .collect();
+
+            delim = format!("${}$", s).into();
+        }
+
+        write!(f, "{}{}{}", delim, self.0, delim)
+    }
+}
+
+/// Checks whether the state at each state group is the same as what the map thinks it should be
+///
+/// i.e. when synapse tries to work out the state for a given state group by looking at
+/// the database. Will the state it gets be the same as what the map thinks it should be
+pub fn database_collapsed_states_match_map(
+    state_group_map: &BTreeMap<i64, StateGroupEntry>,
+) -> bool {
+    for sg in state_group_map.keys() {
+        let map_state = collapse_state_with_map(state_group_map, *sg);
+        let database_state = collapse_state_with_database(*sg);
+        if map_state != database_state {
+            println!("database state {} doesn't match", sg);
+            println!("expected {:?}", map_state);
+            println!("but found {:?}", database_state);
+            return false;
+        }
+    }
+    true
+}
+
+/// Gets the full state for a given group from the map (of deltas)
+fn collapse_state_with_map(
+    map: &BTreeMap<i64, StateGroupEntry>,
+    state_group: i64,
+) -> StateMap<Atom> {
+    let mut entry = &map[&state_group];
+    let mut state_map = StateMap::new();
+
+    let mut stack = vec![state_group];
+
+    while let Some(prev_state_group) = entry.prev_state_group {
+        stack.push(prev_state_group);
+        if !map.contains_key(&prev_state_group) {
+            panic!("Missing {}", prev_state_group);
+        }
+        entry = &map[&prev_state_group];
+    }
+
+    for sg in stack.iter().rev() {
+        state_map.extend(
+            map[sg]
+                .state_map
+                .iter()
+                .map(|((t, s), e)| ((t, s), e.clone())),
+        );
+    }
+
+    state_map
+}
+
+fn collapse_state_with_database(state_group: i64) -> StateMap<Atom> {
+    // connect to the database
+    let mut builder = SslConnector::builder(SslMethod::tls()).unwrap();
+    builder.set_verify(SslVerifyMode::NONE);
+    let connector = MakeTlsConnector::new(builder.build());
+
+    let mut client = Client::connect(DB_URL, connector).unwrap();
+
+    // Gets the delta for a specific state group
+    let query_deltas = r#"
+        SELECT m.id, type, state_key, s.event_id
+        FROM state_groups AS m
+        LEFT JOIN state_groups_state AS s ON (m.id = s.state_group)
+        WHERE m.id = $1
+    "#;
+
+    // If there is no delta for that specific state group, then we still want to find
+    // the predecessor (so have split this into a different query)
+    let query_pred = r#"
+        SELECT prev_state_group
+        FROM state_group_edges
+        WHERE state_group = $1
+    "#;
+
+    let mut state_map: StateMap<Atom> = StateMap::new();
+
+    let mut next_group = Some(state_group);
+
+    while let Some(sg) = next_group {
+        // get predecessor from state_group_edges
+        let mut pred = client.query_raw(query_pred, &[sg]).unwrap();
+
+        // set next_group to predecessor
+        next_group = match pred.next().unwrap() {
+            Some(p) => p.get(0),
+            None => None,
+        };
+
+        // if there was a predecessor then assert that it is unique
+        if next_group.is_some() {
+            assert!(pred.next().unwrap().is_none());
+        }
+        drop(pred);
+
+        let mut rows = client.query_raw(query_deltas, &[sg]).unwrap();
+
+        while let Some(row) = rows.next().unwrap() {
+            // Copy the single delta from the predecessor stored in this row
+            if let Some(etype) = row.get::<_, Option<String>>(1) {
+                let key = &row.get::<_, String>(2);
+
+                // only insert if not overriding existing entry
+                // this is because the newer delta is found FIRST
+                if state_map.get(&etype, key).is_none() {
+                    state_map.insert(&etype, key, row.get::<_, String>(3).into());
+                }
+            }
+        }
+    }
+
+    state_map
+}
+
+/// Check whether predecessors and deltas stored in the database are the same as in the map
+pub fn database_structure_matches_map(state_group_map: &BTreeMap<i64, StateGroupEntry>) -> bool {
+    // connect to the database
+    let mut builder = SslConnector::builder(SslMethod::tls()).unwrap();
+    builder.set_verify(SslVerifyMode::NONE);
+    let connector = MakeTlsConnector::new(builder.build());
+
+    let mut client = Client::connect(DB_URL, connector).unwrap();
+
+    // Gets the delta for a specific state group
+    let query_deltas = r#"
+        SELECT m.id, type, state_key, s.event_id
+        FROM state_groups AS m
+        LEFT JOIN state_groups_state AS s ON (m.id = s.state_group)
+        WHERE m.id = $1
+    "#;
+
+    // If there is no delta for that specific state group, then we still want to find
+    // the predecessor (so have split this into a different query)
+    let query_pred = r#"
+        SELECT prev_state_group
+        FROM state_group_edges
+        WHERE state_group = $1
+    "#;
+
+    for (sg, entry) in state_group_map {
+        // get predecessor from state_group_edges
+        let mut pred_iter = client.query_raw(query_pred, &[sg]).unwrap();
+
+        // read out the predecessor value from the database
+        let database_pred = match pred_iter.next().unwrap() {
+            Some(p) => p.get(0),
+            None => None,
+        };
+
+        // if there was a predecessor then assert that it is unique
+        if database_pred.is_some() {
+            assert!(pred_iter.next().unwrap().is_none());
+        }
+
+        // check if it matches map
+        if database_pred != entry.prev_state_group {
+            println!(
+                "ERROR: predecessor for {} was {:?} (expected {:?})",
+                sg, database_pred, entry.prev_state_group
+            );
+            return false;
+        }
+        // needed so that can create another query
+        drop(pred_iter);
+
+        // Now check that deltas are the same
+        let mut state_map: StateMap<Atom> = StateMap::new();
+
+        // Get delta from state_groups_state
+        let mut rows = client.query_raw(query_deltas, &[sg]).unwrap();
+
+        while let Some(row) = rows.next().unwrap() {
+            // Copy the single delta from the predecessor stored in this row
+            if let Some(etype) = row.get::<_, Option<String>>(1) {
+                state_map.insert(
+                    &etype,
+                    &row.get::<_, String>(2),
+                    row.get::<_, String>(3).into(),
+                );
+            }
+        }
+
+        // Check that the delta matches the map
+        if state_map != entry.state_map {
+            println!("ERROR: delta for {} didn't match", sg);
+            println!("Expected: {:?}", entry.state_map);
+            println!("Actual: {:?}", state_map);
+            return false;
+        }
+    }
+    true
+}
+
+/// Clears the compressor state from the database
+pub fn clear_compressor_state() {
+    // connect to the database
+    let mut builder = SslConnector::builder(SslMethod::tls()).unwrap();
+    builder.set_verify(SslVerifyMode::NONE);
+    let connector = MakeTlsConnector::new(builder.build());
+
+    let mut client = Client::connect(DB_URL, connector).unwrap();
+
+    // delete all the contents from the state compressor tables
+    let sql = r"
+        TRUNCATE state_compressor_state;
+        TRUNCATE state_compressor_progress;
+        UPDATE state_compressor_total_progress SET lowest_uncompressed_group = 0;
+    ";
+
+    client.batch_execute(sql).unwrap();
+}
+
+#[test]
+fn functions_are_self_consistent() {
+    let mut initial: BTreeMap<i64, StateGroupEntry> = BTreeMap::new();
+    let mut prev = None;
+
+    // This starts with the following structure
+    //
+    // 0-1-2-3-4-5-6-7-8-9-10-11-12-13
+    //
+    // Each group i has state:
+    //     ('node','is',      i)
+    //     ('group',  j, 'seen') - for all j less than i
+    for i in 0i64..=13i64 {
+        let mut entry = StateGroupEntry {
+            in_range: true,
+            prev_state_group: prev,
+            state_map: StateMap::new(),
+        };
+        entry
+            .state_map
+            .insert("group", &i.to_string(), "seen".into());
+        entry.state_map.insert("node", "is", i.to_string().into());
+
+        initial.insert(i, entry);
+
+        prev = Some(i)
+    }
+
+    empty_database();
+    add_contents_to_database("room1", &initial);
+
+    assert!(database_collapsed_states_match_map(&initial));
+    assert!(database_structure_matches_map(&initial));
+}
+
+pub fn setup_logger() {
+    // setup the logger for the synapse_auto_compressor
+    // The default can be overwritten with RUST_LOG
+    // see the README for more information
+    if env::var("RUST_LOG").is_err() {
+        let mut log_builder = env_logger::builder();
+        // set is_test(true) so that the output is hidden by cargo test (unless the test fails)
+        log_builder.is_test(true);
+        // default to printing the debug information for both packages being tested
+        // (Note that just setting the global level to debug will log every sql transaction)
+        log_builder.filter_module("synapse_compress_state", LevelFilter::Debug);
+        log_builder.filter_module("synapse_auto_compressor", LevelFilter::Debug);
+        // use try_init() incase the logger has been setup by some previous test
+        let _ = log_builder.try_init();
+    } else {
+        // If RUST_LOG was set then use that
+        let mut log_builder = env_logger::Builder::from_env("RUST_LOG");
+        // set is_test(true) so that the output is hidden by cargo test (unless the test fails)
+        log_builder.is_test(true);
+        // use try_init() in case the logger has been setup by some previous test
+        let _ = log_builder.try_init();
+    }
+}

compressor_integration_tests/src/map_builder.rs

@@ -0,0 +1,217 @@
+use std::collections::BTreeMap;
+
+use state_map::StateMap;
+use synapse_compress_state::StateGroupEntry;
+
+/// Generates long chain of state groups each with state deltas
+///
+/// If called wiht start=0, end=13 this would build the following:
+///
+/// 0-1-2-3-4-5-6-7-8-9-10-11-12-13
+///
+/// Where each group i has state:
+///     ('node','is',      i)
+///     ('group',  j, 'seen') - for all j less than i
+pub fn line_with_state(start: i64, end: i64) -> BTreeMap<i64, StateGroupEntry> {
+    let mut initial: BTreeMap<i64, StateGroupEntry> = BTreeMap::new();
+    let mut prev = None;
+
+    for i in start..=end {
+        let mut entry = StateGroupEntry {
+            in_range: true,
+            prev_state_group: prev,
+            state_map: StateMap::new(),
+        };
+        entry
+            .state_map
+            .insert("group", &i.to_string(), "seen".into());
+        entry.state_map.insert("node", "is", i.to_string().into());
+
+        initial.insert(i, entry);
+
+        prev = Some(i)
+    }
+
+    initial
+}
+
+/// Generates line segments in a chain of state groups each with state deltas
+///
+/// If called wiht start=0, end=13 this would build the following:
+///
+/// 0-1-2 3-4-5 6-7-8 9-10-11 12-13
+///
+/// Where each group i has state:
+///     ('node','is',      i)
+///     ('group',  j, 'seen') - for all j less than i
+pub fn line_segments_with_state(start: i64, end: i64) -> BTreeMap<i64, StateGroupEntry> {
+    let mut initial: BTreeMap<i64, StateGroupEntry> = BTreeMap::new();
+    let mut prev = None;
+
+    for i in start..=end {
+        // if the state is a snapshot then set its predecessor to NONE
+        if (i - start) % 3 == 0 {
+            prev = None;
+        }
+
+        // create a blank entry for it
+        let mut entry = StateGroupEntry {
+            in_range: true,
+            prev_state_group: prev,
+            state_map: StateMap::new(),
+        };
+
+        // if it's a snapshot then add in all previous state
+        if prev.is_none() {
+            for j in start..i {
+                entry
+                    .state_map
+                    .insert("group", &j.to_string(), "seen".into());
+            }
+        }
+
+        // add in the new state for this state group
+        entry
+            .state_map
+            .insert("group", &i.to_string(), "seen".into());
+        entry.state_map.insert("node", "is", i.to_string().into());
+
+        // put it into the initial map
+        initial.insert(i, entry);
+
+        // set this group as the predecessor for the next
+        prev = Some(i)
+    }
+    initial
+}
+
+/// This generates the correct compressed structure with 3,3 levels
+///
+/// Note: only correct structure when no impossible predecessors
+///
+/// Structure generated:
+///
+/// 0  3\      12
+/// 1  4 6\    13
+/// 2  5 7 9
+///     8 10
+///        11
+/// Where each group i has state:
+///     ('node','is',      i)
+///     ('group',  j, 'seen') - for all j less than i
+pub fn compressed_3_3_from_0_to_13_with_state() -> BTreeMap<i64, StateGroupEntry> {
+    let expected_edges: BTreeMap<i64, i64> = vec![
+        (1, 0),
+        (2, 1),
+        (4, 3),
+        (5, 4),
+        (6, 3),
+        (7, 6),
+        (8, 7),
+        (9, 6),
+        (10, 9),
+        (11, 10),
+        (13, 12),
+    ]
+    .into_iter()
+    .collect();
+
+    let mut expected: BTreeMap<i64, StateGroupEntry> = BTreeMap::new();
+
+    // Each group i has state:
+    //     ('node','is',      i)
+    //     ('group',  j, 'seen') - for all j less than i
+    for i in 0i64..=13i64 {
+        let prev = expected_edges.get(&i);
+
+        //change from Option<&i64> to Option<i64>
+        let prev = prev.copied();
+
+        // create a blank entry for it
+        let mut entry = StateGroupEntry {
+            in_range: true,
+            prev_state_group: prev,
+            state_map: StateMap::new(),
+        };
+
+        // Add in all state between predecessor and now (non inclusive)
+        if let Some(p) = prev {
+            for j in (p + 1)..i {
+                entry
+                    .state_map
+                    .insert("group", &j.to_string(), "seen".into());
+            }
+        } else {
+            for j in 0i64..i {
+                entry
+                    .state_map
+                    .insert("group", &j.to_string(), "seen".into());
+            }
+        }
+
+        // add in the new state for this state group
+        entry
+            .state_map
+            .insert("group", &i.to_string(), "seen".into());
+        entry.state_map.insert("node", "is", i.to_string().into());
+
+        // put it into the expected map
+        expected.insert(i, entry);
+    }
+    expected
+}
+
+/// Generates state map structure that corresponds to edges (with deltas)
+///
+/// Each group i has state:
+//     ('node','is',      i)
+//     ('group',  j, 'seen') - for all j less than i
+pub fn structure_from_edges_with_state(
+    edges: BTreeMap<i64, i64>,
+    start: i64,
+    end: i64,
+) -> BTreeMap<i64, StateGroupEntry> {
+    let mut expected: BTreeMap<i64, StateGroupEntry> = BTreeMap::new();
+
+    // Each group i has state:
+    //     ('node','is',      i)
+    //     ('group',  j, 'seen') - for all j less than i
+    for i in start..=end {
+        let prev = edges.get(&i);
+
+        //change from Option<&i64> to Option<i64>
+        let prev = prev.copied();
+
+        // create a blank entry for it
+        let mut entry = StateGroupEntry {
+            in_range: true,
+            prev_state_group: prev,
+            state_map: StateMap::new(),
+        };
+
+        // Add in all state between predecessor and now (non inclusive)
+        if let Some(p) = prev {
+            for j in (p + 1)..i {
+                entry
+                    .state_map
+                    .insert("group", &j.to_string(), "seen".into());
+            }
+        } else {
+            for j in start..i {
+                entry
+                    .state_map
+                    .insert("group", &j.to_string(), "seen".into());
+            }
+        }
+
+        // add in the new state for this state group
+        entry
+            .state_map
+            .insert("group", &i.to_string(), "seen".into());
+        entry.state_map.insert("node", "is", i.to_string().into());
+
+        // put it into the expected map
+        expected.insert(i, entry);
+    }
+    expected
+}

compressor_integration_tests/tests/auto_compressor_manager_tests.rs

@@ -0,0 +1,230 @@
+use std::collections::BTreeMap;
+
+use compressor_integration_tests::{
+    add_contents_to_database, clear_compressor_state, database_collapsed_states_match_map,
+    database_structure_matches_map, empty_database,
+    map_builder::{
+        compressed_3_3_from_0_to_13_with_state, line_segments_with_state,
+        structure_from_edges_with_state,
+    },
+    setup_logger, DB_URL,
+};
+use serial_test::serial;
+use synapse_auto_compressor::{
+    manager::{compress_chunks_of_database, run_compressor_on_room_chunk},
+    state_saving::{connect_to_database, create_tables_if_needed},
+};
+use synapse_compress_state::Level;
+
+#[test]
+#[serial(db)]
+fn run_compressor_on_room_chunk_works() {
+    setup_logger();
+    // This starts with the following structure
+    //
+    // 0-1-2 3-4-5 6-7-8 9-10-11 12-13
+    //
+    // Each group i has state:
+    //     ('node','is',      i)
+    //     ('group',  j, 'seen') - for all j less than i
+    let initial = line_segments_with_state(0, 13);
+    empty_database();
+    add_contents_to_database("room1", &initial);
+
+    let mut client = connect_to_database(DB_URL).unwrap();
+    create_tables_if_needed(&mut client).unwrap();
+    clear_compressor_state();
+
+    // compress in 3,3 level sizes by default
+    let default_levels = vec![Level::new(3), Level::new(3)];
+
+    // compress the first 7 groups in the room
+    // structure should be the following afterwards
+    // (NOTE: only including compressed groups)
+    //
+    // 0  3\
+    // 1  4 6
+    // 2  5
+    run_compressor_on_room_chunk(DB_URL, "room1", 7, &default_levels).unwrap();
+
+    // compress the next 7 groups
+
+    run_compressor_on_room_chunk(DB_URL, "room1", 7, &default_levels).unwrap();
+
+    // This should have created the following structure in the database
+    // i.e. groups 6 and 9 should have changed from before
+    // N.B. this saves 11 rows
+    //
+    // 0  3\      12
+    // 1  4 6\    13
+    // 2  5 7 9
+    //      8 10
+    //        11
+    let expected = compressed_3_3_from_0_to_13_with_state();
+
+    // Check that the database still gives correct states for each group!
+    assert!(database_collapsed_states_match_map(&initial));
+
+    // Check that the structure of the database matches the expected structure
+    assert!(database_structure_matches_map(&expected));
+}
+
+#[test]
+#[serial(db)]
+fn compress_chunks_of_database_compresses_multiple_rooms() {
+    setup_logger();
+    // This creates 2 with the following structure
+    //
+    // 0-1-2 3-4-5 6-7-8 9-10-11 12-13
+    // (with room2's numbers shifted up 14)
+    //
+    // Each group i has state:
+    //     ('node','is',      i)
+    //     ('group',  j, 'seen') - for all j less than i in that room
+    let initial1 = line_segments_with_state(0, 13);
+    let initial2 = line_segments_with_state(14, 27);
+
+    empty_database();
+    add_contents_to_database("room1", &initial1);
+    add_contents_to_database("room2", &initial2);
+
+    let mut client = connect_to_database(DB_URL).unwrap();
+    create_tables_if_needed(&mut client).unwrap();
+    clear_compressor_state();
+
+    // compress in 3,3 level sizes by default
+    let default_levels = vec![Level::new(3), Level::new(3)];
+
+    // Compress 4 chunks of size 8.
+    // The first two should compress room1 and the second two should compress room2
+    compress_chunks_of_database(DB_URL, 8, &default_levels, 4).unwrap();
+
+    // We are aiming for the following structure in the database for room1
+    // i.e. groups 6 and 9 should have changed from initial map
+    // N.B. this saves 11 rows
+    //
+    // 0  3\      12
+    // 1  4 6\    13
+    // 2  5 7 9
+    //      8 10
+    //        11
+    //
+    // Where each group i has state:
+    //     ('node','is',      i)
+    //     ('group',  j, 'seen') - for all j less than i
+    let expected1 = compressed_3_3_from_0_to_13_with_state();
+
+    // Check that the database still gives correct states for each group in room1
+    assert!(database_collapsed_states_match_map(&initial1));
+
+    // Check that the structure of the database matches the expected structure for room1
+    assert!(database_structure_matches_map(&expected1));
+
+    // room 2 should have the same structure but will all numbers shifted up by 14
+    let expected_edges: BTreeMap<i64, i64> = vec![
+        (15, 14),
+        (16, 15),
+        (18, 17),
+        (19, 18),
+        (20, 17),
+        (21, 20),
+        (22, 21),
+        (23, 20),
+        (24, 23),
+        (25, 24),
+        (27, 26),
+    ]
+    .into_iter()
+    .collect();
+
+    let expected2 = structure_from_edges_with_state(expected_edges, 14, 27);
+
+    // Check that the database still gives correct states for each group in room2
+    assert!(database_collapsed_states_match_map(&initial2));
+
+    // Check that the structure of the database matches the expected structure for room2
+    assert!(database_structure_matches_map(&expected2));
+}
+
+#[test]
+#[serial(db)]
+fn compress_chunks_of_database_continues_where_it_left_off() {
+    setup_logger();
+    // This creates 2 with the following structure
+    //
+    // 0-1-2 3-4-5 6-7-8 9-10-11 12-13
+    // (with room2's numbers shifted up 14)
+    //
+    // Each group i has state:
+    //     ('node','is',      i)
+    //     ('group',  j, 'seen') - for all j less than i in that room
+    let initial1 = line_segments_with_state(0, 13);
+    let initial2 = line_segments_with_state(14, 27);
+
+    empty_database();
+    add_contents_to_database("room1", &initial1);
+    add_contents_to_database("room2", &initial2);
+
+    let mut client = connect_to_database(DB_URL).unwrap();
+    create_tables_if_needed(&mut client).unwrap();
+    clear_compressor_state();
+
+    // compress in 3,3 level sizes by default
+    let default_levels = vec![Level::new(3), Level::new(3)];
+
+    // Compress chunks of various sizes:
+    //
+    // These two should compress room1
+    compress_chunks_of_database(DB_URL, 8, &default_levels, 1).unwrap();
+    compress_chunks_of_database(DB_URL, 100, &default_levels, 1).unwrap();
+    // These three should compress room2
+    compress_chunks_of_database(DB_URL, 1, &default_levels, 2).unwrap();
+    compress_chunks_of_database(DB_URL, 5, &default_levels, 1).unwrap();
+    compress_chunks_of_database(DB_URL, 5, &default_levels, 1).unwrap();
+
+    // We are aiming for the following structure in the database for room1
+    // i.e. groups 6 and 9 should have changed from initial map
+    // N.B. this saves 11 rows
+    //
+    // 0  3\      12
+    // 1  4 6\    13
+    // 2  5 7 9
+    //      8 10
+    //        11
+    //
+    // Where each group i has state:
+    //     ('node','is',      i)
+    //     ('group',  j, 'seen') - for all j less than i
+    let expected1 = compressed_3_3_from_0_to_13_with_state();
+
+    // Check that the database still gives correct states for each group in room1
+    assert!(database_collapsed_states_match_map(&initial1));
+
+    // Check that the structure of the database matches the expected structure for room1
+    assert!(database_structure_matches_map(&expected1));
+
+    // room 2 should have the same structure but will all numbers shifted up by 14
+    let expected_edges: BTreeMap<i64, i64> = vec![
+        (15, 14),
+        (16, 15),
+        (18, 17),
+        (19, 18),
+        (20, 17),
+        (21, 20),
+        (22, 21),
+        (23, 20),
+        (24, 23),
+        (25, 24),
+        (27, 26),
+    ]
+    .into_iter()
+    .collect();
+
+    let expected2 = structure_from_edges_with_state(expected_edges, 14, 27);
+
+    // Check that the database still gives correct states for each group in room2
+    assert!(database_collapsed_states_match_map(&initial2));
+
+    // Check that the structure of the database matches the expected structure for room2
+    assert!(database_structure_matches_map(&expected2));
+}

compressor_integration_tests/tests/auto_compressor_state_saving_tests.rs

@@ -0,0 +1,29 @@
+use compressor_integration_tests::{clear_compressor_state, setup_logger, DB_URL};
+use serial_test::serial;
+use synapse_auto_compressor::state_saving::{
+    connect_to_database, create_tables_if_needed, read_room_compressor_state,
+    write_room_compressor_state,
+};
+use synapse_compress_state::Level;
+
+#[test]
+#[serial(db)]
+fn write_then_read_state_gives_correct_results() {
+    setup_logger();
+    let mut client = connect_to_database(DB_URL).unwrap();
+    create_tables_if_needed(&mut client).unwrap();
+    clear_compressor_state();
+
+    let room_id = "room1";
+    let written_info: Vec<Level> =
+        vec![Level::restore(3, 1, Some(6)), Level::restore(3, 2, Some(6))];
+    let written_num = 53;
+    write_room_compressor_state(&mut client, room_id, &written_info, written_num).unwrap();
+
+    let (read_num, read_info) = read_room_compressor_state(&mut client, room_id)
+        .unwrap()
+        .unwrap();
+
+    assert_eq!(written_info, read_info);
+    assert_eq!(written_num, read_num);
+}

compressor_integration_tests/tests/compressor_config_tests.rs

@@ -0,0 +1,575 @@
+use std::collections::BTreeMap;
+
+use compressor_integration_tests::{
+    add_contents_to_database, database_collapsed_states_match_map, database_structure_matches_map,
+    empty_database,
+    map_builder::{
+        compressed_3_3_from_0_to_13_with_state, line_segments_with_state, line_with_state,
+        structure_from_edges_with_state,
+    },
+    setup_logger, DB_URL,
+};
+use serial_test::serial;
+use synapse_compress_state::{run, Config};
+
+// Remember to add #[serial(db)] before any test that access the database.
+// Only one test with this annotation can run at once - preventing
+// concurrency bugs.
+//
+// You will probably also want to use common::empty_database() at the start
+// of each test as well (since their order of execution is not guaranteed)
+
+#[test]
+#[serial(db)]
+fn run_succeeds_without_crashing() {
+    setup_logger();
+    // This starts with the following structure
+    //
+    // 0-1-2-3-4-5-6-7-8-9-10-11-12-13
+    //
+    // Each group i has state:
+    //     ('node','is',      i)
+    //     ('group',  j, 'seen') - for all j less than i
+    let initial = line_with_state(0, 13);
+
+    empty_database();
+    add_contents_to_database("room1", &initial);
+
+    let db_url = DB_URL.to_string();
+    let room_id = "room1".to_string();
+    let output_file = Some("./tests/tmp/run_succeeds_without_crashing.sql".to_string());
+    let min_state_group = None;
+    let groups_to_compress = None;
+    let min_saved_rows = None;
+    let max_state_group = None;
+    let level_sizes = "3,3".to_string();
+    let transactions = true;
+    let graphs = false;
+    let commit_changes = false;
+    let verify = true;
+
+    let config = Config::new(
+        db_url,
+        room_id,
+        output_file,
+        min_state_group,
+        groups_to_compress,
+        min_saved_rows,
+        max_state_group,
+        level_sizes,
+        transactions,
+        graphs,
+        commit_changes,
+        verify,
+    )
+    .unwrap();
+
+    run(config);
+}
+
+#[test]
+#[serial(db)]
+fn changes_commited_if_no_min_saved_rows() {
+    setup_logger();
+    // This starts with the following structure
+    //
+    // 0-1-2 3-4-5 6-7-8 9-10-11 12-13
+    //
+    // Each group i has state:
+    //     ('node','is',      i)
+    //     ('group',  j, 'seen') - for all j less than i
+    let initial = line_segments_with_state(0, 13);
+
+    // Place this initial state into an empty database
+    empty_database();
+    add_contents_to_database("room1", &initial);
+
+    // set up the config options
+    let db_url = DB_URL.to_string();
+    let room_id = "room1".to_string();
+    let output_file = Some("./tests/tmp/changes_commited_if_no_min_saved_rows.sql".to_string());
+    let min_state_group = None;
+    let min_saved_rows = None;
+    let groups_to_compress = None;
+    let max_state_group = None;
+    let level_sizes = "3,3".to_string();
+    let transactions = true;
+    let graphs = false;
+    let commit_changes = true;
+    let verify = true;
+
+    let config = Config::new(
+        db_url,
+        room_id,
+        output_file,
+        min_state_group,
+        groups_to_compress,
+        min_saved_rows,
+        max_state_group,
+        level_sizes,
+        transactions,
+        graphs,
+        commit_changes,
+        verify,
+    )
+    .unwrap();
+
+    // Run the compressor with those settings
+    run(config);
+
+    // This should have created the following structure in the database
+    // i.e. groups 6 and 9 should have changed from before
+    // N.B. this saves 11 rows
+    //
+    // 0  3\      12
+    // 1  4 6\    13
+    // 2  5 7 9
+    //      8 10
+    //        11
+    let expected = compressed_3_3_from_0_to_13_with_state();
+
+    // Check that the database still gives correct states for each group!
+    assert!(database_collapsed_states_match_map(&initial));
+
+    // Check that the structure of the database matches the expected structure
+    assert!(database_structure_matches_map(&expected))
+}
+
+#[test]
+#[serial(db)]
+fn changes_commited_if_min_saved_rows_exceeded() {
+    setup_logger();
+    // This starts with the following structure
+    //
+    // 0-1-2 3-4-5 6-7-8 9-10-11 12-13
+    //
+    // Each group i has state:
+    //     ('node','is',      i)
+    //     ('group',  j, 'seen') - for all j less than i
+    let initial = line_segments_with_state(0, 13);
+
+    // Place this initial state into an empty database
+    empty_database();
+    add_contents_to_database("room1", &initial);
+
+    // set up the config options
+    let db_url = DB_URL.to_string();
+    let room_id = "room1".to_string();
+    let output_file = Some("./tests/tmp/changes_commited_if_no_min_saved_rows.sql".to_string());
+    let min_state_group = None;
+    let min_saved_rows = Some(10);
+    let groups_to_compress = None;
+    let max_state_group = None;
+    let level_sizes = "3,3".to_string();
+    let transactions = true;
+    let graphs = false;
+    let commit_changes = true;
+    let verify = true;
+
+    let config = Config::new(
+        db_url,
+        room_id,
+        output_file,
+        min_state_group,
+        groups_to_compress,
+        min_saved_rows,
+        max_state_group,
+        level_sizes,
+        transactions,
+        graphs,
+        commit_changes,
+        verify,
+    )
+    .unwrap();
+
+    // Run the compressor with those settings
+    run(config);
+
+    // This should have created the following structure in the database
+    // i.e. groups 6 and 9 should have changed from before
+    // N.B. this saves 11 rows
+    //
+    // 0  3\      12
+    // 1  4 6\    13
+    // 2  5 7 9
+    //      8 10
+    //        11
+    let expected = compressed_3_3_from_0_to_13_with_state();
+
+    // Check that the database still gives correct states for each group!
+    assert!(database_collapsed_states_match_map(&initial));
+
+    // Check that the structure of the database matches the expected structure
+    assert!(database_structure_matches_map(&expected));
+}
+
+#[test]
+#[serial(db)]
+fn changes_not_commited_if_fewer_than_min_saved_rows() {
+    setup_logger();
+    // This starts with the following structure
+    //
+    // 0-1-2 3-4-5 6-7-8 9-10-11 12-13
+    //
+    // Each group i has state:
+    //     ('node','is',      i)
+    //     ('group',  j, 'seen') - for all j less than i
+    let initial = line_segments_with_state(0, 13);
+
+    // Place this initial state into an empty database
+    empty_database();
+    add_contents_to_database("room1", &initial);
+
+    // set up the config options
+    let db_url = DB_URL.to_string();
+    let room_id = "room1".to_string();
+    let output_file =
+        Some("./tests/tmp/changes_not_commited_if_fewer_than_min_saved_rows.sql".to_string());
+    let min_state_group = None;
+    let min_saved_rows = Some(12);
+    let groups_to_compress = None;
+    let max_state_group = None;
+    let level_sizes = "3,3".to_string();
+    let transactions = true;
+    let graphs = false;
+    let commit_changes = true;
+    let verify = true;
+
+    let config = Config::new(
+        db_url,
+        room_id,
+        output_file,
+        min_state_group,
+        groups_to_compress,
+        min_saved_rows,
+        max_state_group,
+        level_sizes,
+        transactions,
+        graphs,
+        commit_changes,
+        verify,
+    )
+    .unwrap();
+
+    // Run the compressor with those settings
+    run(config);
+
+    // This should have created the following structure when running
+    // (i.e. try and change groups 6 and 9 only)
+    // BUT: This saves 11 rows which is fewer than min_saved_rows
+    // therefore there should be no changes committed!
+    //
+    // 0  3\      12
+    // 1  4 6\    13
+    // 2  5 7 9
+    //      8 10
+    //        11
+
+    // Check that the database still gives correct states for each group!
+    assert!(database_collapsed_states_match_map(&initial));
+
+    // Check that the structure of the database matches the expected structure
+    assert!(database_structure_matches_map(&initial));
+}
+
+#[test]
+#[should_panic(expected = "Error connecting to the database:")]
+fn run_panics_if_invalid_db_url() {
+    setup_logger();
+    // set up the config options
+    let db_url = "thisIsAnInvalidURL".to_string();
+    let room_id = "room1".to_string();
+    let output_file = Some("./tests/tmp/run_panics_if_invalid_db_url.sql".to_string());
+    let min_state_group = None;
+    let min_saved_rows = None;
+    let groups_to_compress = None;
+    let max_state_group = None;
+    let level_sizes = "3,3".to_string();
+    let transactions = true;
+    let graphs = false;
+    let commit_changes = true;
+    let verify = true;
+
+    let config = Config::new(
+        db_url,
+        room_id,
+        output_file,
+        min_state_group,
+        groups_to_compress,
+        min_saved_rows,
+        max_state_group,
+        level_sizes,
+        transactions,
+        graphs,
+        commit_changes,
+        verify,
+    )
+    .unwrap();
+
+    // Run the compressor with those settings
+    run(config);
+}
+
+#[test]
+#[serial(db)]
+fn run_only_affects_given_room_id() {
+    setup_logger();
+    // build room1 stuff up
+    // This starts with the following structure
+    //
+    // 0-1-2 3-4-5 6-7-8 9-10-11 12-13
+    //
+    // Each group i has state:
+    //     ('node','is',      i)
+    //     ('group',  j, 'seen') - for all j less than i
+    let initial_room_1 = line_segments_with_state(0, 13);
+
+    // build room2 stuff up
+    // This starts with the same structure as room 1 but just all group ids
+    // 14 higher
+    let initial_room_2 = line_segments_with_state(14, 28);
+
+    // Place this initial state into an empty database
+    empty_database();
+    add_contents_to_database("room1", &initial_room_1);
+    add_contents_to_database("room2", &initial_room_2);
+
+    // set up the config options
+    let db_url = DB_URL.to_string();
+    let room_id = "room1".to_string();
+    let output_file = Some("./tests/tmp/run_only_affects_given_room_id.sql".to_string());
+    let min_state_group = None;
+    let min_saved_rows = None;
+    let groups_to_compress = None;
+    let max_state_group = None;
+    let level_sizes = "3,3".to_string();
+    let transactions = true;
+    let graphs = false;
+    let commit_changes = true;
+    let verify = true;
+
+    let config = Config::new(
+        db_url,
+        room_id,
+        output_file,
+        min_state_group,
+        groups_to_compress,
+        min_saved_rows,
+        max_state_group,
+        level_sizes,
+        transactions,
+        graphs,
+        commit_changes,
+        verify,
+    )
+    .unwrap();
+
+    // Run the compressor with those settings
+    run(config);
+
+    // This should have created the following structure in the database
+    // i.e. groups 6 and 9 should have changed from before
+    // N.B. this saves 11 rows
+    //
+    // 0  3\      12
+    // 1  4 6\    13
+    // 2  5 7 9
+    //      8 10
+    //        11
+    let expected = compressed_3_3_from_0_to_13_with_state();
+
+    // Check that the database still gives correct states for each group
+    // in both room1 and room2
+    assert!(database_collapsed_states_match_map(&initial_room_1));
+    assert!(database_collapsed_states_match_map(&initial_room_2));
+
+    // Check that the structure of the database matches the expected structure
+    // in both room1 and room2
+    assert!(database_structure_matches_map(&expected));
+    assert!(database_structure_matches_map(&initial_room_2));
+}
+
+#[test]
+#[serial(db)]
+fn run_respects_groups_to_compress() {
+    setup_logger();
+    // This starts with the following structure
+    //
+    // 0-1-2 3-4-5 6-7-8 9-10-11 12-13
+    //
+    // Each group i has state:
+    //     ('node','is',      i)
+    //     ('group',  j, 'seen') - for all j less than i
+    let initial = line_segments_with_state(0, 13);
+
+    // Place this initial state into an empty database
+    empty_database();
+    add_contents_to_database("room1", &initial);
+
+    // set up the config options
+    let db_url = DB_URL.to_string();
+    let room_id = "room1".to_string();
+    let output_file = Some("./tests/tmp/run_respects_groups_to_compress.sql".to_string());
+    let min_state_group = Some(2);
+    let min_saved_rows = None;
+    let groups_to_compress = Some(9);
+    let max_state_group = None;
+    let level_sizes = "3,3".to_string();
+    let transactions = true;
+    let graphs = false;
+    let commit_changes = true;
+    let verify = true;
+
+    let config = Config::new(
+        db_url,
+        room_id,
+        output_file,
+        min_state_group,
+        groups_to_compress,
+        min_saved_rows,
+        max_state_group,
+        level_sizes,
+        transactions,
+        graphs,
+        commit_changes,
+        verify,
+    )
+    .unwrap();
+
+    // Run the compressor with those settings
+    run(config);
+
+    // This should have created the following structure in the database
+    // as it should only compress from groups higher than 2 (non inclusive)
+    // and should only compress a total of 9 groups
+    // i.e. so only group 9 should have changed from before
+    // N.B. this saves 7 rows
+    //
+    // 0  3  6\    12
+    // 1  4  7  9  13
+    // 2  5  8 10
+    //         11
+    //
+    let expected_edges: BTreeMap<i64, i64> = vec![
+        (1, 0),
+        (2, 1),
+        (4, 3),
+        (5, 4),
+        (7, 6),
+        (8, 7),
+        (9, 6),
+        (10, 9),
+        (11, 10),
+        (13, 12),
+    ]
+    .into_iter()
+    .collect();
+
+    let expected = structure_from_edges_with_state(expected_edges, 0, 13);
+
+    // Check that the database still gives correct states for each group!
+    assert!(database_collapsed_states_match_map(&initial));
+
+    // Check that the structure of the database matches the expected structure
+    assert!(database_structure_matches_map(&expected))
+}
+
+#[test]
+#[serial(db)]
+fn run_is_idempotent_when_run_on_whole_room() {
+    setup_logger();
+    // This starts with the following structure
+    //
+    // 0-1-2 3-4-5 6-7-8 9-10-11 12-13
+    //
+    // Each group i has state:
+    //     ('node','is',      i)
+    //     ('group',  j, 'seen') - for all j less than i
+    let initial = line_segments_with_state(0, 13);
+
+    // Place this initial state into an empty database
+    empty_database();
+    add_contents_to_database("room1", &initial);
+
+    // set up the config options
+    let db_url = DB_URL.to_string();
+    let room_id = "room1".to_string();
+    let output_file1 =
+        Some("./tests/tmp/run_is_idempotent_when_run_on_whole_room_1.sql".to_string());
+    let output_file2 =
+        Some("./tests/tmp/run_is_idempotent_when_run_on_whole_room_2.sql".to_string());
+    let min_state_group = None;
+    let min_saved_rows = None;
+    let groups_to_compress = None;
+    let max_state_group = None;
+    let level_sizes = "3,3".to_string();
+    let transactions = true;
+    let graphs = false;
+    let commit_changes = true;
+    let verify = true;
+
+    let config1 = Config::new(
+        db_url.clone(),
+        room_id.clone(),
+        output_file1,
+        min_state_group,
+        groups_to_compress,
+        min_saved_rows,
+        max_state_group,
+        level_sizes.clone(),
+        transactions,
+        graphs,
+        commit_changes,
+        verify,
+    )
+    .unwrap();
+
+    let config2 = Config::new(
+        db_url,
+        room_id,
+        output_file2,
+        min_state_group,
+        groups_to_compress,
+        min_saved_rows,
+        max_state_group,
+        level_sizes,
+        transactions,
+        graphs,
+        commit_changes,
+        verify,
+    )
+    .unwrap();
+
+    // We are aiming for the following structure in the database
+    // i.e. groups 6 and 9 should have changed from initial map
+    // N.B. this saves 11 rows
+    //
+    // 0  3\      12
+    // 1  4 6\    13
+    // 2  5 7 9
+    //      8 10
+    //        11
+    //
+    // Where each group i has state:
+    //     ('node','is',      i)
+    //     ('group',  j, 'seen') - for all j less than i
+    let expected = compressed_3_3_from_0_to_13_with_state();
+
+    // Run the compressor with those settings for the first time
+    run(config1);
+
+    // Check that the database still gives correct states for each group!
+    assert!(database_collapsed_states_match_map(&initial));
+
+    // Check that the structure of the database matches the expected structure
+    assert!(database_structure_matches_map(&expected));
+
+    // Run the compressor with those settings for the second time
+    run(config2);
+
+    // Check that the database still gives correct states for each group!
+    assert!(database_collapsed_states_match_map(&initial));
+
+    // Check that the structure of the database still matches the expected structure
+    assert!(database_structure_matches_map(&expected));
+}

compressor_integration_tests/tests/compressor_continue_run_tests.rs

@@ -0,0 +1,83 @@
+use compressor_integration_tests::{
+    add_contents_to_database, database_collapsed_states_match_map, database_structure_matches_map,
+    empty_database,
+    map_builder::{compressed_3_3_from_0_to_13_with_state, line_segments_with_state},
+    setup_logger, DB_URL,
+};
+use serial_test::serial;
+use synapse_compress_state::{continue_run, Level};
+
+// Tests the saving and continuing functionality
+// The compressor should produce the same results when run in one go
+// as when run in multiple stages
+#[test]
+#[serial(db)]
+fn continue_run_called_twice_same_as_run() {
+    setup_logger();
+    // This starts with the following structure
+    //
+    // 0-1-2 3-4-5 6-7-8 9-10-11 12-13
+    //
+    // Each group i has state:
+    //     ('node','is',      i)
+    //     ('group',  j, 'seen') - for all j less than i
+    let initial = line_segments_with_state(0, 13);
+
+    // Place this initial state into an empty database
+    empty_database();
+    add_contents_to_database("room1", &initial);
+
+    let db_url = DB_URL.to_string();
+    let room_id = "room1".to_string();
+
+    // will run the compression in two batches
+    let start = None;
+    let chunk_size = 7;
+
+    // compress in 3,3 level sizes
+    // since the compressor hasn't been run before they are empty
+    let level_info = vec![Level::new(3), Level::new(3)];
+
+    // Run the compressor with those settings
+    let chunk_stats_1 = continue_run(start, chunk_size, &db_url, &room_id, &level_info).unwrap();
+
+    // Assert that it stopped at 6 (i.e. after the 7 groups 0...6)
+    assert_eq!(chunk_stats_1.last_compressed_group, 6);
+    // structure should be the following at this point
+    // (NOTE: only including compressed groups)
+    //
+    // 0  3\
+    // 1  4 6
+    // 2  5
+    assert_eq!(
+        chunk_stats_1.new_level_info,
+        vec![Level::restore(3, 1, Some(6)), Level::restore(3, 2, Some(6))]
+    );
+
+    let start = Some(6);
+    let chunk_size = 7;
+    let level_info = chunk_stats_1.new_level_info;
+
+    // Run the compressor with those settings
+    let chunk_stats_2 = continue_run(start, chunk_size, &db_url, &room_id, &level_info).unwrap();
+
+    // Assert that it stopped at 7
+    assert_eq!(chunk_stats_2.last_compressed_group, 13);
+
+    // This should have created the following structure in the database
+    // i.e. groups 6 and 9 should have changed from before
+    // N.B. this saves 11 rows
+    //
+    // 0  3\      12
+    // 1  4 6\    13
+    // 2  5 7 9
+    //      8 10
+    //        11
+    let expected = compressed_3_3_from_0_to_13_with_state();
+
+    // Check that the database still gives correct states for each group!
+    assert!(database_collapsed_states_match_map(&initial));
+
+    // Check that the structure of the database matches the expected structure
+    assert!(database_structure_matches_map(&expected))
+}

compressor_integration_tests/tests/tmp/README.md

@@ -0,0 +1 @@
+This folder is where sql files generated by the integration tests are saved

docs/algorithm.md

@@ -0,0 +1,107 @@
+# Compression algorithm
+
+## What is state?
+State is things like who is in a room, what the room topic/name is, who has
+what privilege levels etc. Synapse keeps track of it for various reasons such as
+spotting invalid events (e.g. ones sent by banned users) and providing room membership
+information to clients.
+
+## What is a state group?
+
+Synapse needs to keep track of the state at the moment of each event. A state group
+corresponds to a unique state. The database table `event_to_state_groups` keeps track
+of the mapping from event ids to state group ids.
+
+Consider the following simplified example:
+```
+State group id   |          State
+_____________________________________________
+       1         |      Alice in room
+       2         | Alice in room, Bob in room
+       3         |        Bob in room
+
+
+Event id |     What the event was
+______________________________________
+    1    |    Alice sends a message
+    3    |     Bob joins the room
+    4    |     Bob sends a message
+    5    |    Alice leaves the room
+    6    |     Bob sends a message
+
+
+Event id | State group id
+_________________________
+    1    |       1
+    2    |       1
+    3    |       2
+    4    |       2
+    5    |       3
+    6    |       3
+```
+
+## What are deltas and predecessors?
+When a new state event happens (e.g. Bob joins the room) a new state group is created.
+BUT instead of copying all of the state from the previous state group, we just store
+the change from the previous group (saving on lots of storage space!). The difference
+from the previous state group is called the "delta".
+
+So for the previous example, we would have the following (Note only rows 1 and 2 will
+make sense at this point):
+
+```
+State group id | Previous state group id |      Delta
+____________________________________________________________
+       1       |          NONE           |   Alice in room
+       2       |           1             |    Bob in room
+       3       |          NONE           |    Bob in room
+```
+
+So why is state group 3's previous state group NONE and not 2? Well, the way that deltas
+work in Synapse is that they can only add in new state or overwrite old state, but they
+cannot remove it. (So if the room topic is changed then that is just overwriting state,
+but removing Alice from the room is neither an addition nor an overwriting). If it is
+impossible to find a delta, then you just start from scratch again with a "snapshot" of
+the entire state. 
+
+(NOTE this is not documentation on how synapse handles leaving rooms but is purely for illustrative
+purposes)
+
+The state of a state group is worked out by following the previous state group's and adding
+together all of the deltas (with the most recent taking precedence).
+
+The mapping from state group to previous state group takes place in `state_group_edges`
+and the deltas are stored in `state_groups_state`.
+
+## What are we compressing then?
+In order to speed up the conversion from state group id to state, there is a limit of 100 
+hops set by synapse (that is: we will only ever have to look up the deltas for a maximum of 
+100 state groups). It does this by taking another "snapshot" every 100 state groups.
+
+However, it is these snapshots that take up the bulk of the storage in a synapse database,
+so we want to find a way to reduce the number of them without dramatically increasing the
+maximum number of hops needed to do lookups.
+
+
+## Compression Algorithm
+
+The algorithm works by attempting to create a *tree* of deltas, produced by
+appending state groups to different "levels". Each level has a maximum size, where
+each state group is appended to the lowest level that is not full. This tool calls a 
+state group "compressed" once it has been added to
+one of these levels.
+
+This produces a graph that looks approximately like the following, in the case
+of having two levels with the bottom level (L1) having a maximum size of 3:
+
+```
+L2 <-------------------- L2 <---------- ...
+^--- L1 <--- L1 <--- L1  ^--- L1 <--- L1 <--- L1
+
+NOTE: A <--- B means that state group B's predecessor is A
+```
+The structure that synapse creates by default would be equivalent to having one level with
+a maximum length of 100. 
+
+**Note**: Increasing the sum of the sizes of levels will increase the time it
+takes to query the full state of a given state group.

docs/python.md

@@ -0,0 +1,54 @@
+# Running the compressor tools from python
+
+Both the automatic and manual tools use PyO3 to allow the compressor
+to be run from Python. 
+
+To see any output from the tools, logging must be setup in Python before
+the compressor is run.
+
+## Setting things up
+
+1. Create a virtual environment in the place you want to use the compressor from
+(if it doesn't already exist)  
+`$ virtualenv -p python3 venv`
+
+2. Activate the virtual environment and install `maturin` (if you haven't already)  
+`$ source venv/bin/activate`  
+`$ pip install maturin`  
+
+3. Navigate to the correct location  
+For the automatic tool:  
+`$ cd /home/synapse/rust-synapse-compress-state/synpase_auto_compressor`   
+For the manual tool:  
+`$ cd /home/synapse/rust-synapse-compress-state`   
+
+3. Build and install the library  
+`$ maturin develop`
+
+This will install the relevant compressor tool into the activated virtual environment.
+
+## Automatic tool example:
+
+```python
+import synapse_auto_compressor
+
+synapse_auto_compressor.compress_state_events_table(
+  db_url="postgresql://localhost/synapse",
+  chunk_size=500,
+  default_levels="100,50,25",
+  number_of_chunks=100
+)
+```
+
+# Manual tool example:
+
+```python
+import synapse_compress_state
+
+synapse_compress_state.run_compression(
+  db_url="postgresql://localhost/synapse",
+  room_id="!some_room:example.com",
+  output_file="out.sql",
+  transactions=True
+)
+```

pyproject.toml

@@ -0,0 +1,4 @@
+[build-system]
+requires = ["maturin>=0.11,<0.12"]
+build-backend = "maturin"
+cargo-extra-args = "--no-default-features"

src/compressor.rs

@@ -0,0 +1,310 @@
+// Copyright 2018 New Vector Ltd
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! This is the actual compression algorithm.
+//!
+//! The algorithm attempts to make a tree of deltas for the state group maps.
+//! This is done by having multiple "levels", where each level has a maximum
+//! size. The state groups are iterated over, with deltas being calculated
+//! against the smallest level that isn't yet full. When a state group is
+//! inserted into a level, or lower levels are reset to have their current
+//! "head" at the new state group.
+//!
+//! This produces graphs that look roughly like, for two levels:
+//!
+//! ```ignore
+//! L2 <-------------------- L2 <---------- ...
+//!  ^--- L1 <--- L1 <--- L1  ^--- L1 <--- L1 <--- L1
+//! ```
+
+use indicatif::{ProgressBar, ProgressStyle};
+use state_map::StateMap;
+use std::collections::BTreeMap;
+use string_cache::DefaultAtom as Atom;
+
+use super::{collapse_state_maps, StateGroupEntry};
+
+/// Holds information about a particular level.
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct Level {
+    /// The maximum size this level is allowed to be
+    max_length: usize,
+    /// The (approximate) current chain length of this level. This is equivalent
+    /// to recursively following `current`
+    current_chain_length: usize,
+    /// The head of this level
+    head: Option<i64>,
+}
+
+impl Level {
+    /// Creates a new Level with the given maximum length
+    pub fn new(max_length: usize) -> Level {
+        Level {
+            max_length,
+            current_chain_length: 0,
+            head: None,
+        }
+    }
+
+    /// Creates a new level from stored state
+    pub fn restore(max_length: usize, current_chain_length: usize, head: Option<i64>) -> Level {
+        Level {
+            max_length,
+            current_chain_length,
+            head,
+        }
+    }
+
+    /// Update the current head of this level. If delta is true then it means
+    /// that given state group will (probably) reference the previous head.
+    ///
+    /// Panics if `delta` is true and the level is already full.
+    fn update(&mut self, new_head: i64, delta: bool) {
+        self.head = Some(new_head);
+
+        if delta {
+            // If we're referencing the previous head then increment our chain
+            // length estimate
+            if !self.has_space() {
+                panic!("Tried to add to an already full level");
+            }
+
+            self.current_chain_length += 1;
+        } else {
+            // Otherwise, we've started a new chain with a single entry.
+            self.current_chain_length = 1;
+        }
+    }
+
+    /// Get the max length of the level
+    pub fn get_max_length(&self) -> usize {
+        self.max_length
+    }
+
+    /// Get the current length of the level
+    pub fn get_current_length(&self) -> usize {
+        self.current_chain_length
+    }
+
+    /// Get the current head of the level
+    pub fn get_head(&self) -> Option<i64> {
+        self.head
+    }
+
+    /// Whether there is space in the current chain at this level. If not then a
+    /// new chain should be started.
+    pub fn has_space(&self) -> bool {
+        self.current_chain_length < self.max_length
+    }
+}
+
+/// Keeps track of some statistics of a compression run.
+#[derive(Default)]
+pub struct Stats {
+    /// How many state groups we couldn't find a delta for, despite trying.
+    pub resets_no_suitable_prev: usize,
+    /// The sum of the rows of the state groups counted by
+    /// `resets_no_suitable_prev`.
+    pub resets_no_suitable_prev_size: usize,
+    /// How many state groups we have changed.
+    pub state_groups_changed: usize,
+}
+
+/// Attempts to compress a set of state deltas using the given level sizes.
+pub struct Compressor<'a> {
+    original_state_map: &'a BTreeMap<i64, StateGroupEntry>,
+    pub new_state_group_map: BTreeMap<i64, StateGroupEntry>,
+    levels: Vec<Level>,
+    pub stats: Stats,
+}
+
+impl<'a> Compressor<'a> {
+    /// Creates a compressor and runs the compression algorithm.
+    pub fn compress(
+        original_state_map: &'a BTreeMap<i64, StateGroupEntry>,
+        level_sizes: &[usize],
+    ) -> Compressor<'a> {
+        let mut compressor = Compressor {
+            original_state_map,
+            new_state_group_map: BTreeMap::new(),
+            levels: level_sizes.iter().map(|size| Level::new(*size)).collect(),
+            stats: Stats::default(),
+        };
+
+        compressor.create_new_tree();
+
+        compressor
+    }
+
+    /// Creates a compressor and runs the compression algorithm.
+    /// used when restoring compressor state from a previous run
+    /// in which case the levels heads are also known
+    pub fn compress_from_save(
+        original_state_map: &'a BTreeMap<i64, StateGroupEntry>,
+        level_info: &[Level],
+    ) -> Compressor<'a> {
+        let levels = level_info
+            .iter()
+            .map(|l| Level::restore((*l).max_length, (*l).current_chain_length, (*l).head))
+            .collect();
+
+        let mut compressor = Compressor {
+            original_state_map,
+            new_state_group_map: BTreeMap::new(),
+            levels,
+            stats: Stats::default(),
+        };
+
+        compressor.create_new_tree();
+        compressor
+    }
+
+    /// Returns all the state required to save the compressor so it can be continued later
+    pub fn get_level_info(&self) -> Vec<Level> {
+        self.levels.clone()
+    }
+
+    /// Actually runs the compression algorithm
+    fn create_new_tree(&mut self) {
+        if !self.new_state_group_map.is_empty() {
+            panic!("Can only call `create_new_tree` once");
+        }
+
+        let pb: ProgressBar = if cfg!(feature = "no-progress-bars") {
+            ProgressBar::hidden()
+        } else {
+            ProgressBar::new(self.original_state_map.len() as u64)
+        };
+        pb.set_style(
+            ProgressStyle::default_bar().template("[{elapsed_precise}] {bar} {pos}/{len} {msg}"),
+        );
+        pb.set_message("state groups");
+        pb.enable_steady_tick(100);
+
+        for (&state_group, entry) in self.original_state_map {
+            // Check whether this entry is in_range or is just present in the map due to being
+            // a predecessor of a group that IS in_range for compression
+            if !entry.in_range {
+                let new_entry = StateGroupEntry {
+                    // in_range is kept the same so that the new entry is equal to the old entry
+                    // otherwise it might trigger a useless database transaction
+                    in_range: entry.in_range,
+                    prev_state_group: entry.prev_state_group,
+                    state_map: entry.state_map.clone(),
+                };
+                // Paranoidly assert that not making changes to this entry
+                // could probably be removed...
+                assert!(new_entry == *entry);
+                self.new_state_group_map.insert(state_group, new_entry);
+
+                continue;
+            }
+            let mut prev_state_group = None;
+            for level in &mut self.levels {
+                if level.has_space() {
+                    prev_state_group = level.get_head();
+                    level.update(state_group, true);
+                    break;
+                } else {
+                    level.update(state_group, false);
+                }
+            }
+
+            let (delta, prev_state_group) = if entry.prev_state_group == prev_state_group {
+                (entry.state_map.clone(), prev_state_group)
+            } else {
+                self.stats.state_groups_changed += 1;
+                self.get_delta(prev_state_group, state_group)
+            };
+
+            self.new_state_group_map.insert(
+                state_group,
+                StateGroupEntry {
+                    in_range: true,
+                    prev_state_group,
+                    state_map: delta,
+                },
+            );
+
+            pb.inc(1);
+        }
+
+        pb.finish();
+    }
+
+    /// Attempts to calculate the delta between two state groups.
+    ///
+    /// This is not always possible if the given candidate previous state group
+    /// have state keys that are not in the new state group. In this case the
+    /// function will try and iterate back up the current tree to find a state
+    /// group that can be used as a base for a delta.
+    ///
+    /// Returns the state map and the actual base state group (if any) used.
+    fn get_delta(&mut self, prev_sg: Option<i64>, sg: i64) -> (StateMap<Atom>, Option<i64>) {
+        let state_map = collapse_state_maps(self.original_state_map, sg);
+
+        let mut prev_sg = if let Some(prev_sg) = prev_sg {
+            prev_sg
+        } else {
+            return (state_map, None);
+        };
+
+        // This is a loop to go through to find the first prev_sg which can be
+        // a valid base for the state group.
+        let mut prev_state_map;
+        'outer: loop {
+            prev_state_map = collapse_state_maps(self.original_state_map, prev_sg);
+            for (t, s) in prev_state_map.keys() {
+                if !state_map.contains_key(t, s) {
+                    // This is not a valid base as it contains key the new state
+                    // group doesn't have. Attempt to walk up the tree to find a
+                    // better base.
+                    if let Some(psg) = self.new_state_group_map[&prev_sg].prev_state_group {
+                        prev_sg = psg;
+                        continue 'outer;
+                    }
+
+                    // Couldn't find a new base, so we give up and just persist
+                    // a full state group here.
+                    self.stats.resets_no_suitable_prev += 1;
+                    self.stats.resets_no_suitable_prev_size += state_map.len();
+
+                    return (state_map, None);
+                }
+            }
+
+            break;
+        }
+
+        // We've found a valid base, now we just need to calculate the delta.
+        let mut delta_map = StateMap::new();
+
+        for ((t, s), e) in state_map.iter() {
+            if prev_state_map.get(t, s) != Some(e) {
+                delta_map.insert(t, s, e.clone());
+            }
+        }
+
+        (delta_map, Some(prev_sg))
+    }
+}
+
+#[cfg(test)]
+mod level_tests;
+
+#[cfg(test)]
+mod compressor_tests;
+
+#[cfg(test)]
+mod stats_tests;

src/compressor/compressor_tests.rs

@@ -0,0 +1,689 @@
+use crate::{
+    compressor::{Compressor, Level, Stats},
+    StateGroupEntry,
+};
+use state_map::StateMap;
+use std::collections::BTreeMap;
+use string_cache::DefaultAtom as Atom;
+
+#[test]
+fn compress_creates_correct_compressor() {
+    let mut initial: BTreeMap<i64, StateGroupEntry> = BTreeMap::new();
+    let mut prev = None;
+
+    // This starts with the following structure
+    //
+    // 0-1-2-3-4-5-6-7-8-9-10-11-12-13
+    for i in 0i64..=13i64 {
+        initial.insert(
+            i,
+            StateGroupEntry {
+                in_range: true,
+                prev_state_group: prev,
+                state_map: StateMap::new(),
+            },
+        );
+
+        prev = Some(i)
+    }
+
+    let compressor = Compressor::compress(&initial, &[3, 3]);
+
+    let new_state = &compressor.new_state_group_map;
+
+    // This should create the following structure
+    //
+    // 0  3\      12
+    // 1  4 6\    13
+    // 2  5 7 9
+    //      8 10
+    //        11
+    let expected_edges: BTreeMap<i64, i64> = vec![
+        (1, 0),
+        (2, 1),
+        (4, 3),
+        (5, 4),
+        (6, 3),
+        (7, 6),
+        (8, 7),
+        (9, 6),
+        (10, 9),
+        (11, 10),
+        (13, 12),
+    ]
+    .into_iter()
+    .collect();
+
+    for sg in 0i64..=13i64 {
+        assert_eq!(
+            expected_edges.get(&sg).cloned(),
+            new_state[&sg].prev_state_group,
+            "state group {} did not match expected",
+            sg,
+        );
+    }
+}
+
+#[test]
+fn create_new_tree_does_nothing_if_already_compressed() {
+    // This should create the following structure
+    //
+    // 0  3\      12
+    // 1  4 6\    13
+    // 2  5 7 9
+    //      8 10
+    //        11
+    let initial_edges: BTreeMap<i64, i64> = vec![
+        (1, 0),
+        (2, 1),
+        (4, 3),
+        (5, 4),
+        (6, 3),
+        (7, 6),
+        (8, 7),
+        (9, 6),
+        (10, 9),
+        (11, 10),
+        (13, 12),
+    ]
+    .into_iter()
+    .collect();
+
+    let mut initial: BTreeMap<i64, StateGroupEntry> = BTreeMap::new();
+
+    for i in 0i64..=13i64 {
+        // edge from map
+        let prev = initial_edges.get(&i).copied();
+
+        // insert that edge into the initial map
+        initial.insert(
+            i,
+            StateGroupEntry {
+                in_range: true,
+                prev_state_group: prev,
+                state_map: StateMap::new(),
+            },
+        );
+    }
+
+    let mut compressor = Compressor {
+        original_state_map: &initial,
+        new_state_group_map: BTreeMap::new(),
+        levels: vec![Level::new(3), Level::new(3)],
+        stats: Stats::default(),
+    };
+
+    compressor.create_new_tree();
+
+    let new_state = &compressor.new_state_group_map;
+
+    assert_eq!(initial, *new_state);
+}
+
+#[test]
+fn create_new_tree_respects_levels() {
+    let mut initial: BTreeMap<i64, StateGroupEntry> = BTreeMap::new();
+    let mut prev = None;
+
+    // This starts with the following structure
+    //
+    // 0-1-2-3-4-5-6-7-8-9-10-11-12-13
+    for i in 0i64..=13i64 {
+        initial.insert(
+            i,
+            StateGroupEntry {
+                in_range: true,
+                prev_state_group: prev,
+                state_map: StateMap::new(),
+            },
+        );
+
+        prev = Some(i)
+    }
+
+    let mut compressor = Compressor {
+        original_state_map: &initial,
+        new_state_group_map: BTreeMap::new(),
+        levels: vec![Level::new(3), Level::new(3)],
+        stats: Stats::default(),
+    };
+    compressor.create_new_tree();
+
+    let new_state = &compressor.new_state_group_map;
+
+    // This should create the following structure
+    //
+    // 0  3\      12
+    // 1  4 6\    13
+    // 2  5 7 9
+    //      8 10
+    //        11
+    let expected_edges: BTreeMap<i64, i64> = vec![
+        (1, 0),
+        (2, 1),
+        (4, 3),
+        (5, 4),
+        (6, 3),
+        (7, 6),
+        (8, 7),
+        (9, 6),
+        (10, 9),
+        (11, 10),
+        (13, 12),
+    ]
+    .into_iter()
+    .collect();
+
+    for sg in 0i64..=13i64 {
+        assert_eq!(
+            expected_edges.get(&sg).cloned(),
+            new_state[&sg].prev_state_group,
+            "state group {} did not match expected",
+            sg,
+        );
+    }
+}
+
+#[test]
+#[should_panic(expected = "Can only call `create_new_tree` once")]
+fn create_new_tree_panics_if_run_twice() {
+    let mut initial: BTreeMap<i64, StateGroupEntry> = BTreeMap::new();
+    let mut prev = None;
+
+    for i in 0i64..=13i64 {
+        initial.insert(
+            i,
+            StateGroupEntry {
+                in_range: true,
+                prev_state_group: prev,
+                state_map: StateMap::new(),
+            },
+        );
+
+        prev = Some(i)
+    }
+
+    let mut compressor = Compressor {
+        original_state_map: &initial,
+        new_state_group_map: BTreeMap::new(),
+        levels: vec![Level::new(3), Level::new(3)],
+        stats: Stats::default(),
+    };
+    compressor.create_new_tree();
+    compressor.create_new_tree();
+}
+
+#[test]
+fn create_new_tree_respects_all_not_in_range() {
+    let mut initial: BTreeMap<i64, StateGroupEntry> = BTreeMap::new();
+    let mut prev = None;
+
+    // This starts with the following structure
+    //
+    // 0-1-2-3-4-5-6-7-8-9-10-11-12-13
+    for i in 0i64..=13i64 {
+        initial.insert(
+            i,
+            StateGroupEntry {
+                in_range: false,
+                prev_state_group: prev,
+                state_map: StateMap::new(),
+            },
+        );
+
+        prev = Some(i)
+    }
+
+    let mut compressor = Compressor {
+        original_state_map: &initial,
+        new_state_group_map: BTreeMap::new(),
+        levels: vec![Level::new(3), Level::new(3)],
+        stats: Stats::default(),
+    };
+    compressor.create_new_tree();
+
+    let new_state = &compressor.new_state_group_map;
+
+    // This should create the following structure
+    //
+    // 0-1-2-3-4-5-6-7-8-9-10-11-12-13 (i.e. no change!)
+    let expected_edges: BTreeMap<i64, i64> = vec![
+        (1, 0),
+        (2, 1),
+        (3, 2),
+        (4, 3),
+        (5, 4),
+        (6, 5),
+        (7, 6),
+        (8, 7),
+        (9, 8),
+        (10, 9),
+        (11, 10),
+        (12, 11),
+        (13, 12),
+    ]
+    .into_iter()
+    .collect();
+
+    for sg in 0i64..=13i64 {
+        assert_eq!(
+            expected_edges.get(&sg).cloned(),
+            new_state[&sg].prev_state_group,
+            "state group {} did not match expected",
+            sg,
+        );
+    }
+}
+
+#[test]
+fn create_new_tree_respects_some_not_in_range() {
+    let mut initial: BTreeMap<i64, StateGroupEntry> = BTreeMap::new();
+    let mut prev = None;
+
+    // This starts with the following structure
+    //
+    // 0-1-2-3-4-5-6-7-8-9-10-11-12-13-14-15-16-17-18
+    for i in 0i64..=18i64 {
+        initial.insert(
+            i,
+            StateGroupEntry {
+                in_range: i > 4,
+                prev_state_group: prev,
+                state_map: StateMap::new(),
+            },
+        );
+
+        prev = Some(i)
+    }
+
+    let mut compressor = Compressor {
+        original_state_map: &initial,
+        new_state_group_map: BTreeMap::new(),
+        levels: vec![Level::new(3), Level::new(3)],
+        stats: Stats::default(),
+    };
+    compressor.create_new_tree();
+
+    let new_state = &compressor.new_state_group_map;
+
+    // This should create the following structure
+    //
+    // 0  5   8\       17
+    // 1  6   9 11\    18
+    // 2  7  10 12 14
+    // 3        13 15
+    // 4           16
+    let expected_edges: BTreeMap<i64, i64> = vec![
+        (1, 0),
+        (2, 1),
+        (3, 2),
+        (4, 3), // No compression of nodes 0,1,2,3,4
+        (6, 5), // Compresses in 3,3 leveling starting at 5
+        (7, 6),
+        (9, 8),
+        (10, 9),
+        (11, 8),
+        (12, 11),
+        (13, 12),
+        (14, 11),
+        (15, 14),
+        (16, 15),
+        (18, 17),
+    ]
+    .into_iter()
+    .collect();
+    for n in new_state {
+        println!("{:?}", n);
+    }
+
+    for sg in 0i64..=13i64 {
+        assert_eq!(
+            expected_edges.get(&sg).cloned(),
+            new_state[&sg].prev_state_group,
+            "state group {} did not match expected",
+            sg,
+        );
+    }
+}
+
+#[test]
+fn create_new_tree_deals_with_impossible_preds() {
+    let mut initial: BTreeMap<i64, StateGroupEntry> = BTreeMap::new();
+    let mut prev = None;
+
+    // This starts with the following structure
+    //
+    // (note missing 3-4 link)
+    // 0-1-2-3
+    // 4-5-6-7-8-9-10-11-12-13
+    //
+    // Each group i has state:
+    //     ('node','is',      i)
+    //     ('group',  j, 'seen') where j is ancestor of i
+    for i in 0i64..=13i64 {
+        if i == 4 {
+            prev = None
+        }
+        let mut entry = StateGroupEntry {
+            in_range: true,
+            prev_state_group: prev,
+            state_map: StateMap::new(),
+        };
+        entry
+            .state_map
+            .insert("group", &i.to_string(), "seen".into());
+        entry.state_map.insert("node", "is", i.to_string().into());
+
+        initial.insert(i, entry);
+
+        prev = Some(i)
+    }
+
+    let mut compressor = Compressor {
+        original_state_map: &initial,
+        new_state_group_map: BTreeMap::new(),
+        levels: vec![Level::new(3), Level::new(3)],
+        stats: Stats::default(),
+    };
+    compressor.create_new_tree();
+
+    let new_state = &compressor.new_state_group_map;
+
+    for n in new_state {
+        println!("{:?}", n);
+    }
+
+    // This should create the following structure
+    //
+    // Brackets mean that has NO predecessor but is in that position in the
+    // levels tree
+    //
+    // 0  3\        12
+    // 1 (4)(6)\    13
+    // 2  5  7  9
+    //       8  10
+    //          11
+    let expected_edges: BTreeMap<i64, i64> = vec![
+        (1, 0),
+        (2, 1),
+        (5, 4),
+        (7, 6),
+        (8, 7),
+        (9, 6),
+        (10, 9),
+        (11, 10),
+        (13, 12),
+    ]
+    .into_iter()
+    .collect();
+
+    for sg in 0i64..=13i64 {
+        assert_eq!(
+            expected_edges.get(&sg).cloned(),
+            new_state[&sg].prev_state_group,
+            "state group {} did not match expected",
+            sg,
+        );
+    }
+}
+
+#[test]
+fn get_delta_returns_snapshot_if_no_prev_given() {
+    let mut initial: BTreeMap<i64, StateGroupEntry> = BTreeMap::new();
+    let mut prev = None;
+
+    // This starts with the following structure
+    //
+    // 0-1-2-3-4-5-6-7-8-9-10-11-12-13
+    //
+    // Each group i has state:
+    //     ('node','is',      i)
+    //     ('group',  j, 'seen') - for all j less than i
+    for i in 0i64..=13i64 {
+        let mut entry = StateGroupEntry {
+            in_range: true,
+            prev_state_group: prev,
+            state_map: StateMap::new(),
+        };
+        entry
+            .state_map
+            .insert("group", &i.to_string(), "seen".into());
+        entry.state_map.insert("node", "is", i.to_string().into());
+
+        initial.insert(i, entry);
+
+        prev = Some(i)
+    }
+
+    // This should produce the following structure (tested above)
+    //
+    // 0  3\      12
+    // 1  4 6\    13
+    // 2  5 7 9
+    //      8 10
+    //        11
+    //
+    // State contents should be the same as before
+    let mut compressor = Compressor::compress(&initial, &[3, 3]);
+
+    let (found_delta, found_pred) = compressor.get_delta(None, 6);
+
+    let mut expected_delta: StateMap<Atom> = StateMap::new();
+    expected_delta.insert("node", "is", "6".into());
+    expected_delta.insert("group", "0", "seen".into());
+    expected_delta.insert("group", "1", "seen".into());
+    expected_delta.insert("group", "2", "seen".into());
+    expected_delta.insert("group", "3", "seen".into());
+    expected_delta.insert("group", "4", "seen".into());
+    expected_delta.insert("group", "5", "seen".into());
+    expected_delta.insert("group", "6", "seen".into());
+
+    assert_eq!(found_delta, expected_delta);
+    assert_eq!(found_pred, None);
+}
+
+#[test]
+fn get_delta_returns_delta_if_original_predecessor() {
+    let mut initial: BTreeMap<i64, StateGroupEntry> = BTreeMap::new();
+    let mut prev = None;
+
+    // This starts with the following structure
+    //
+    // 0-1-2-3-4-5-6-7-8-9-10-11-12-13
+    //
+    // Each group i has state:
+    //     ('node','is',      i)
+    //     ('group',  j, 'seen') - for all j less than i
+    for i in 0i64..=13i64 {
+        let mut entry = StateGroupEntry {
+            in_range: true,
+            prev_state_group: prev,
+            state_map: StateMap::new(),
+        };
+        entry
+            .state_map
+            .insert("group", &i.to_string(), "seen".into());
+        entry.state_map.insert("node", "is", i.to_string().into());
+
+        initial.insert(i, entry);
+
+        prev = Some(i)
+    }
+
+    // This should produce the following structure (tested above)
+    //
+    // 0  3\      12
+    // 1  4 6\    13
+    // 2  5 7 9
+    //      8 10
+    //        11
+    //
+    // State contents should be the same as before
+    let mut compressor = Compressor::compress(&initial, &[3, 3]);
+
+    let (found_delta, found_pred) = compressor.get_delta(Some(5), 6);
+
+    let mut expected_delta: StateMap<Atom> = StateMap::new();
+    expected_delta.insert("node", "is", "6".into());
+    expected_delta.insert("group", "6", "seen".into());
+
+    assert_eq!(found_delta, expected_delta);
+    assert_eq!(found_pred, Some(5));
+}
+
+#[test]
+fn get_delta_returns_delta_if_original_multi_hop_predecessor() {
+    let mut initial: BTreeMap<i64, StateGroupEntry> = BTreeMap::new();
+    let mut prev = None;
+
+    // This starts with the following structure
+    //
+    // 0-1-2-3-4-5-6-7-8-9-10-11-12-13
+    //
+    // Each group i has state:
+    //     ('node','is',      i)
+    //     ('group',  j, 'seen') - for all j less than i
+    for i in 0i64..=13i64 {
+        let mut entry = StateGroupEntry {
+            in_range: true,
+            prev_state_group: prev,
+            state_map: StateMap::new(),
+        };
+        entry
+            .state_map
+            .insert("group", &i.to_string(), "seen".into());
+        entry.state_map.insert("node", "is", i.to_string().into());
+
+        initial.insert(i, entry);
+
+        prev = Some(i)
+    }
+
+    // This should produce the following structure (tested above)
+    //
+    // 0  3\      12
+    // 1  4 6\    13
+    // 2  5 7 9
+    //      8 10
+    //        11
+    //
+    // State contents should be the same as before
+    let mut compressor = Compressor::compress(&initial, &[3, 3]);
+
+    let (found_delta, found_pred) = compressor.get_delta(Some(3), 6);
+
+    let mut expected_delta: StateMap<Atom> = StateMap::new();
+    expected_delta.insert("node", "is", "6".into());
+    expected_delta.insert("group", "4", "seen".into());
+    expected_delta.insert("group", "5", "seen".into());
+    expected_delta.insert("group", "6", "seen".into());
+
+    assert_eq!(found_delta, expected_delta);
+    assert_eq!(found_pred, Some(3));
+}
+
+#[test]
+fn get_delta_returns_snapshot_if_no_prev_possible() {
+    let mut initial: BTreeMap<i64, StateGroupEntry> = BTreeMap::new();
+    let mut prev = None;
+
+    // This starts with the following structure
+    //
+    // (note missing 3-4 link)
+    // 0-1-2-3
+    // 4-5-6-7-8-9-10-11-12-13
+    //
+    // Each group i has state:
+    //     ('node','is',      i)
+    //     ('group',  j, 'seen') where j is ancestor of i
+    for i in 0i64..=13i64 {
+        // don't add 3-4 link
+        if i == 4 {
+            prev = None
+        }
+
+        // populate the delta for this state
+        let mut entry = StateGroupEntry {
+            in_range: true,
+            prev_state_group: prev,
+            state_map: StateMap::new(),
+        };
+        entry
+            .state_map
+            .insert("group", &i.to_string(), "seen".into());
+        entry.state_map.insert("node", "is", i.to_string().into());
+
+        // put the entry into the initial map
+        initial.insert(i, entry);
+
+        prev = Some(i)
+    }
+
+    // This should create the following structure if create_new_tree() was run
+    // (tested in create_new_tree_deals_with_impossible_preds())
+    //
+    // Brackets mean that has NO predecessor but is in that position in the
+    // levels tree
+    //
+    // 0  3\        12
+    // 1 (4)(6)\    13
+    // 2  5  7  9
+    //       8  10
+    //          11
+    //
+    // State contents should be the same as before
+
+    // build up new_tree after 0,1,2,3 added
+    let mut new_map: BTreeMap<i64, StateGroupEntry> = BTreeMap::new();
+
+    // 0-1-2 is left the same
+    new_map.insert(0, initial.get(&0).unwrap().clone());
+    new_map.insert(1, initial.get(&1).unwrap().clone());
+    new_map.insert(2, initial.get(&2).unwrap().clone());
+
+    // 3 is now a snapshot
+    let mut entry_3: StateMap<Atom> = StateMap::new();
+    entry_3.insert("node", "is", "3".into());
+    entry_3.insert("group", "0", "seen".into());
+    entry_3.insert("group", "1", "seen".into());
+    entry_3.insert("group", "2", "seen".into());
+    entry_3.insert("group", "3", "seen".into());
+    new_map.insert(
+        3,
+        StateGroupEntry {
+            in_range: true,
+            prev_state_group: None,
+            state_map: entry_3,
+        },
+    );
+
+    // build the compressor with this partialy built new map
+    let mut compressor = Compressor {
+        original_state_map: &initial,
+        new_state_group_map: new_map,
+        levels: vec![Level::new(3), Level::new(3)],
+        stats: Stats::default(),
+    };
+
+    // make the levels how they would be after 0,1,2,3 added
+    // they should both be of length 1 and have 3 as the current head
+    let mut levels_iter = compressor.levels.iter_mut();
+
+    let l1 = levels_iter.next().unwrap();
+    l1.head = Some(3);
+    l1.current_chain_length = 1;
+
+    let l2 = levels_iter.next().unwrap();
+    l2.head = Some(3);
+    l2.current_chain_length = 1;
+
+    // Now try and find delta for 4 with 3 as pred
+    let (found_delta, found_pred) = compressor.get_delta(Some(3), 4);
+
+    let mut expected_delta: StateMap<Atom> = StateMap::new();
+    expected_delta.insert("node", "is", "4".into());
+    expected_delta.insert("group", "4", "seen".into());
+
+    assert_eq!(found_delta, expected_delta);
+    assert_eq!(found_pred, None);
+}

src/compressor/level_tests.rs

@@ -0,0 +1,80 @@
+use crate::compressor::Level;
+
+#[test]
+fn new_produces_empty_level() {
+    let l = Level::new(15);
+    assert_eq!(l.max_length, 15);
+    assert_eq!(l.current_chain_length, 0);
+    assert_eq!(l.head, None);
+}
+
+#[test]
+fn update_adds_to_non_full_level() {
+    let mut l = Level::new(10);
+    l.update(7, true);
+    assert_eq!(l.max_length, 10);
+    assert_eq!(l.current_chain_length, 1);
+    assert_eq!(l.head, Some(7));
+}
+
+#[test]
+#[should_panic(expected = "Tried to add to an already full level")]
+fn update_panics_if_adding_and_too_full() {
+    let mut l = Level::new(5);
+    l.update(1, true);
+    l.update(2, true);
+    l.update(3, true);
+    l.update(4, true);
+    l.update(5, true);
+    l.update(6, true);
+}
+
+#[test]
+fn update_resets_level_correctly() {
+    let mut l = Level::new(5);
+    l.update(1, true);
+    l.update(2, true);
+    l.update(3, true);
+    l.update(4, true);
+    l.update(5, true);
+    l.update(6, false);
+    assert_eq!(l.max_length, 5);
+    assert_eq!(l.current_chain_length, 1);
+    assert_eq!(l.head, Some(6));
+}
+
+#[test]
+fn get_head_returns_head() {
+    let mut l = Level::new(5);
+    assert_eq!(l.get_head(), None);
+    l.update(23, true);
+    assert_eq!(l.get_head(), Some(23));
+}
+
+#[test]
+fn has_space_returns_true_if_empty() {
+    let l = Level::new(15);
+    assert!(l.has_space());
+}
+
+#[test]
+fn has_space_returns_true_if_part_full() {
+    let mut l = Level::new(15);
+    l.update(12, true);
+    l.update(234, true);
+    l.update(1, true);
+    l.update(143, true);
+    l.update(15, true);
+    assert!(l.has_space());
+}
+
+#[test]
+fn has_space_returns_false_if_full() {
+    let mut l = Level::new(5);
+    l.update(1, true);
+    l.update(2, true);
+    l.update(3, true);
+    l.update(4, true);
+    l.update(5, true);
+    assert!(!l.has_space());
+}

src/compressor/stats_tests.rs

@@ -0,0 +1,178 @@
+use crate::{
+    compressor::{Compressor, Level, Stats},
+    StateGroupEntry,
+};
+use state_map::StateMap;
+use std::collections::BTreeMap;
+
+#[test]
+fn stats_correct_when_no_resets() {
+    let mut initial: BTreeMap<i64, StateGroupEntry> = BTreeMap::new();
+    let mut prev = None;
+
+    // This starts with the following structure
+    //
+    // 0-1-2-3-4-5-6-7-8-9-10-11-12-13
+    for i in 0i64..=13i64 {
+        initial.insert(
+            i,
+            StateGroupEntry {
+                in_range: true,
+                prev_state_group: prev,
+                state_map: StateMap::new(),
+            },
+        );
+
+        prev = Some(i)
+    }
+
+    let mut compressor = Compressor {
+        original_state_map: &initial,
+        new_state_group_map: BTreeMap::new(),
+        levels: vec![Level::new(3), Level::new(3)],
+        stats: Stats::default(),
+    };
+
+    // This should create the following structure
+    //
+    // 0  3\      12
+    // 1  4 6\    13
+    // 2  5 7 9
+    //      8 10
+    //        11
+    compressor.create_new_tree();
+
+    // No resets should have taken place
+    assert_eq!(compressor.stats.resets_no_suitable_prev, 0);
+    assert_eq!(compressor.stats.resets_no_suitable_prev_size, 0);
+
+    // Groups 3,6,9,12 should be the only ones changed
+    assert_eq!(compressor.stats.state_groups_changed, 4);
+}
+
+#[test]
+fn stats_correct_when_some_resets() {
+    let mut initial: BTreeMap<i64, StateGroupEntry> = BTreeMap::new();
+    let mut prev = None;
+
+    // This starts with the following structure
+    //
+    // (note missing 3-4 link)
+    // 0-1-2-3
+    // 4-5-6-7-8-9-10-11-12-13
+    //
+    // Each group i has state:
+    //     ('node','is',      i)
+    //     ('group',  j, 'seen') where j is ancestor of i
+    for i in 0i64..=13i64 {
+        if i == 4 {
+            prev = None
+        }
+        let mut entry = StateGroupEntry {
+            in_range: true,
+            prev_state_group: prev,
+            state_map: StateMap::new(),
+        };
+        entry
+            .state_map
+            .insert("group", &i.to_string(), "seen".into());
+        entry.state_map.insert("node", "is", i.to_string().into());
+
+        initial.insert(i, entry);
+
+        prev = Some(i)
+    }
+
+    let mut compressor = Compressor {
+        original_state_map: &initial,
+        new_state_group_map: BTreeMap::new(),
+        levels: vec![Level::new(3), Level::new(3)],
+        stats: Stats::default(),
+    };
+
+    // This should create the following structure
+    //
+    // Brackets mean that has NO predecessor but is in that position in the
+    // levels tree
+    //
+    // 0  3\        12
+    // 1 (4)(6)\    13
+    // 2  5  7  9
+    //       8  10
+    //          11
+    compressor.create_new_tree();
+
+    // the reset required for 4 contributes 2 to the size stat
+    // - (1 'node' and 1 'group') entry
+    // the reset required for 6 contributes 4 to the size stat
+    // - (1 'node' and 3 'group') entry
+    assert_eq!(compressor.stats.resets_no_suitable_prev, 2);
+    assert_eq!(compressor.stats.resets_no_suitable_prev_size, 6);
+
+    // groups 3,4,6,9,12 are the only ones changed
+    assert_eq!(compressor.stats.state_groups_changed, 5);
+}
+
+#[test]
+fn stats_correct_if_no_changes() {
+    // This should create the following structure
+    //
+    // 0  3\      12
+    // 1  4 6\    13
+    // 2  5 7 9
+    //      8 10
+    //        11
+    let initial_edges: BTreeMap<i64, i64> = vec![
+        (1, 0),
+        (2, 1),
+        (4, 3),
+        (5, 4),
+        (6, 3),
+        (7, 6),
+        (8, 7),
+        (9, 6),
+        (10, 9),
+        (11, 10),
+        (13, 12),
+    ]
+    .into_iter()
+    .collect();
+
+    let mut initial: BTreeMap<i64, StateGroupEntry> = BTreeMap::new();
+
+    for i in 0i64..=13i64 {
+        // edge from map
+        let prev = initial_edges.get(&i).copied();
+
+        // insert that edge into the initial map
+        initial.insert(
+            i,
+            StateGroupEntry {
+                in_range: true,
+                prev_state_group: prev,
+                state_map: StateMap::new(),
+            },
+        );
+    }
+
+    let mut compressor = Compressor {
+        original_state_map: &initial,
+        new_state_group_map: BTreeMap::new(),
+        levels: vec![Level::new(3), Level::new(3)],
+        stats: Stats::default(),
+    };
+
+    // This should create the following structure (i.e. no change)
+    //
+    // 0  3\      12
+    // 1  4 6\    13
+    // 2  5 7 9
+    //      8 10
+    //        11
+    compressor.create_new_tree();
+
+    // No changes should have been made (the old tree should be the same)
+    assert_eq!(compressor.stats.resets_no_suitable_prev, 0);
+    assert_eq!(compressor.stats.resets_no_suitable_prev_size, 0);
+    assert_eq!(compressor.stats.state_groups_changed, 0);
+}

src/database.rs

@@ -0,0 +1,569 @@
+// Copyright 2018 New Vector Ltd
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use indicatif::{ProgressBar, ProgressStyle};
+use log::{debug, trace};
+use openssl::ssl::{SslConnector, SslMethod, SslVerifyMode};
+use postgres::{fallible_iterator::FallibleIterator, types::ToSql, Client};
+use postgres_openssl::MakeTlsConnector;
+use rand::{distributions::Alphanumeric, thread_rng, Rng};
+use std::{borrow::Cow, collections::BTreeMap, fmt};
+
+use crate::{compressor::Level, generate_sql};
+
+use super::StateGroupEntry;
+
+/// Fetch the entries in state_groups_state (and their prev groups) for a
+/// specific room.
+///
+/// Returns with the state_group map and the id of the last group that was used
+/// Or None if there are no state groups within the range given
+///
+/// # Arguments
+///
+/// * `room_id`             -   The ID of the room in the database
+/// * `db_url`              -   The URL of a Postgres database. This should be of the
+///                             form: "postgresql://user:pass@domain:port/database"
+/// * `min_state_group`     -   If specified, then only fetch the entries for state
+///                             groups greater than (but not equal) to this number. It
+///                             also requires groups_to_compress to be specified
+/// * `max_state_group`     -   If specified, then only fetch the entries for state
+///                             groups lower than or equal to this number.
+/// * 'groups_to_compress'  -   The number of groups to get from the database before stopping
+pub fn get_data_from_db(
+    db_url: &str,
+    room_id: &str,
+    min_state_group: Option<i64>,
+    groups_to_compress: Option<i64>,
+    max_state_group: Option<i64>,
+) -> Option<(BTreeMap<i64, StateGroupEntry>, i64)> {
+    // connect to the database
+    let mut builder = SslConnector::builder(SslMethod::tls()).unwrap();
+    builder.set_verify(SslVerifyMode::NONE);
+    let connector = MakeTlsConnector::new(builder.build());
+
+    let mut client = Client::connect(db_url, connector)
+        .unwrap_or_else(|e| panic!("Error connecting to the database: {}", e));
+
+    // Search for the group id of the groups_to_compress'th group after min_state_group
+    // If this is saved, then the compressor can continue by having min_state_group being
+    // set to this maximum. If no such group can be found then return None.
+    let max_group_found = find_max_group(
+        &mut client,
+        room_id,
+        min_state_group,
+        groups_to_compress,
+        max_state_group,
+    )?;
+
+    let state_group_map: BTreeMap<i64, StateGroupEntry> = BTreeMap::new();
+
+    Some(load_map_from_db(
+        &mut client,
+        room_id,
+        min_state_group,
+        max_group_found,
+        state_group_map,
+    ))
+}
+
+/// Fetch the entries in state_groups_state (and their prev groups) for a
+/// specific room. This method should only be called if resuming the compressor from
+/// where it last finished - and as such also loads in the state groups from the heads
+/// of each of the levels (as they were at the end of the last run of the compressor)
+///
+/// Returns with the state_group map and the id of the last group that was used
+/// Or None if there are no state groups within the range given
+///
+/// # Arguments
+///
+/// * `room_id`             -   The ID of the room in the database
+/// * `db_url`              -   The URL of a Postgres database. This should be of the
+///                             form: "postgresql://user:pass@domain:port/database"
+/// * `min_state_group`     -   If specified, then only fetch the entries for state
+///                             groups greater than (but not equal) to this number. It
+///                             also requires groups_to_compress to be specified
+/// * 'groups_to_compress'  -   The number of groups to get from the database before stopping
+/// * 'level_info'          -   The maximum size, current length and current head for each
+///                             level (as it was when the compressor last finished for this
+///                             room)
+pub fn reload_data_from_db(
+    db_url: &str,
+    room_id: &str,
+    min_state_group: Option<i64>,
+    groups_to_compress: Option<i64>,
+    level_info: &[Level],
+) -> Option<(BTreeMap<i64, StateGroupEntry>, i64)> {
+    // connect to the database
+    let mut builder = SslConnector::builder(SslMethod::tls()).unwrap();
+    builder.set_verify(SslVerifyMode::NONE);
+    let connector = MakeTlsConnector::new(builder.build());
+
+    let mut client = Client::connect(db_url, connector)
+        .unwrap_or_else(|e| panic!("Error connecting to the database: {}", e));
+
+    // Search for the group id of the groups_to_compress'th group after min_state_group
+    // If this is saved, then the compressor can continue by having min_state_group being
+    // set to this maximum.If no such group can be found then return None.
+    let max_group_found = find_max_group(
+        &mut client,
+        room_id,
+        min_state_group,
+        groups_to_compress,
+        // max state group not used when saving and loading
+        None,
+    )?;
+
+    // load just the state_groups at the head of each level
+    // this doesn't load their predecessors as that will be done at the end of
+    // load_map_from_db()
+    let state_group_map: BTreeMap<i64, StateGroupEntry> = load_level_heads(&mut client, level_info);
+
+    Some(load_map_from_db(
+        &mut client,
+        room_id,
+        min_state_group,
+        max_group_found,
+        state_group_map,
+    ))
+}
+
+/// Finds the state_groups that are at the head of each compressor level
+/// NOTE this does not also retrieve their predecessors
+///
+/// # Arguments
+///
+/// * `client'  -   A Postgres client to make requests with
+/// * `levels'  -   The levels who's heads are being requested
+fn load_level_heads(client: &mut Client, level_info: &[Level]) -> BTreeMap<i64, StateGroupEntry> {
+    // obtain all of the heads that aren't None from level_info
+    let level_heads: Vec<i64> = level_info.iter().filter_map(|l| (*l).get_head()).collect();
+
+    // Query to get id, predecessor and deltas for each state group
+    let sql = r#"
+        SELECT m.id, prev_state_group, type, state_key, s.event_id
+        FROM state_groups AS m
+        LEFT JOIN state_groups_state AS s ON (m.id = s.state_group)
+        LEFT JOIN state_group_edges AS e ON (m.id = e.state_group)
+        WHERE m.id = ANY($1)
+        ORDER BY m.id
+    "#;
+
+    // Actually do the query
+    let mut rows = client.query_raw(sql, &[&level_heads]).unwrap();
+
+    // Copy the data from the database into a map
+    let mut state_group_map: BTreeMap<i64, StateGroupEntry> = BTreeMap::new();
+
+    while let Some(row) = rows.next().unwrap() {
+        // The row in the map to copy the data to
+        // NOTE: default StateGroupEntry has in_range as false
+        // This is what we want since as a level head, it has already been compressed by the
+        // previous run!
+        let entry = state_group_map.entry(row.get(0)).or_default();
+
+        // Save the predecessor (this may already be there)
+        entry.prev_state_group = row.get(1);
+
+        // Copy the single delta from the predecessor stored in this row
+        if let Some(etype) = row.get::<_, Option<String>>(2) {
+            entry.state_map.insert(
+                &etype,
+                &row.get::<_, String>(3),
+                row.get::<_, String>(4).into(),
+            );
+        }
+    }
+    state_group_map
+}
+
+/// Fetch the entries in state_groups_state (and their prev groups) for a
+/// specific room within a certain range. These are appended onto the provided
+/// map.
+///
+/// - Fetches the first [group] rows with group id after [min]
+/// - Recursively searches for missing predecessors and adds those
+///
+/// Returns with the state_group map and the id of the last group that was used
+///
+/// # Arguments
+///
+/// * `client`              -   A Postgres client to make requests with
+/// * `room_id`             -   The ID of the room in the database
+/// * `min_state_group`     -   If specified, then only fetch the entries for state
+///                             groups greater than (but not equal) to this number. It
+///                             also requires groups_to_compress to be specified
+/// * 'max_group_found'     -   The last group to get from the database before stopping
+/// * 'state_group_map'     -   The map to populate with the entries from the database
+
+fn load_map_from_db(
+    client: &mut Client,
+    room_id: &str,
+    min_state_group: Option<i64>,
+    max_group_found: i64,
+    mut state_group_map: BTreeMap<i64, StateGroupEntry>,
+) -> (BTreeMap<i64, StateGroupEntry>, i64) {
+    state_group_map.append(&mut get_initial_data_from_db(
+        client,
+        room_id,
+        min_state_group,
+        max_group_found,
+    ));
+
+    debug!("Got initial state from database. Checking for any missing state groups...");
+
+    // Due to reasons some of the state groups appear in the edges table, but
+    // not in the state_groups_state table.
+    //
+    // Also it is likely that the predecessor of a node will not be within the
+    // chunk that was specified by min_state_group and groups_to_compress.
+    // This means they don't get included in our DB queries, so we have to fetch
+    // any missing groups explicitly.
+    //
+    // Since the returned groups may themselves reference groups we don't have,
+    // we need to do this recursively until we don't find any more missing.
+    loop {
+        let mut missing_sgs: Vec<_> = state_group_map
+            .iter()
+            .filter_map(|(_sg, entry)| {
+                if let Some(prev_sg) = entry.prev_state_group {
+                    if state_group_map.contains_key(&prev_sg) {
+                        None
+                    } else {
+                        Some(prev_sg)
+                    }
+                } else {
+                    None
+                }
+            })
+            .collect();
+
+        if missing_sgs.is_empty() {
+            trace!("No missing state groups");
+            break;
+        }
+
+        missing_sgs.sort_unstable();
+        missing_sgs.dedup();
+
+        trace!("Missing {} state groups", missing_sgs.len());
+
+        // find state groups not picked up already and add them to the map
+        let map = get_missing_from_db(client, &missing_sgs, min_state_group, max_group_found);
+        for (k, v) in map {
+            state_group_map.entry(k).or_insert(v);
+        }
+    }
+
+    (state_group_map, max_group_found)
+}
+
+/// Returns the group ID of the last group to be compressed
+///
+/// This can be saved so that future runs of the compressor only
+/// continue from after this point. If no groups can be found in
+/// the range specified it returns None.
+///
+/// # Arguments
+///
+/// * `client`              -   A Postgres client to make requests with
+/// * `room_id`             -   The ID of the room in the database
+/// * `min_state_group`     -   The lower limit (non inclusive) of group id's to compress
+/// * 'groups_to_compress'  -   How many groups to compress
+/// * `max_state_group`     -   The upper bound on what this method can return
+fn find_max_group(
+    client: &mut Client,
+    room_id: &str,
+    min_state_group: Option<i64>,
+    groups_to_compress: Option<i64>,
+    max_state_group: Option<i64>,
+) -> Option<i64> {
+    // Get list of state_id's in a certain room
+    let mut query_chunk_of_ids = "SELECT id FROM state_groups WHERE room_id = $1".to_string();
+    let params: Vec<&(dyn ToSql + Sync)>;
+
+    if let Some(max) = max_state_group {
+        query_chunk_of_ids = format!("{} AND id <= {}", query_chunk_of_ids, max)
+    }
+
+    // Adds additional constraint if a groups_to_compress or min_state_group have been specified
+    // Note a min state group is only used if groups_to_compress also is
+    if min_state_group.is_some() && groups_to_compress.is_some() {
+        params = vec![&room_id, &min_state_group, &groups_to_compress];
+        query_chunk_of_ids = format!(
+            r"{} AND id > $2 ORDER BY id ASC LIMIT $3",
+            query_chunk_of_ids
+        );
+    } else if groups_to_compress.is_some() {
+        params = vec![&room_id, &groups_to_compress];
+        query_chunk_of_ids = format!(r"{} ORDER BY id ASC LIMIT $2", query_chunk_of_ids);
+    } else {
+        params = vec![&room_id];
+    }
+
+    let sql_query = format!(
+        "SELECT id FROM ({}) AS ids ORDER BY ids.id DESC LIMIT 1",
+        query_chunk_of_ids
+    );
+
+    // This vector should have length 0 or 1
+    let rows = client
+        .query(sql_query.as_str(), &params)
+        .expect("Something went wrong while querying the database");
+
+    // If no row can be found then return None
+    let final_row = rows.last()?;
+
+    // Else return the id of the group found
+    Some(final_row.get::<_, i64>(0))
+}
+
+/// Fetch the entries in state_groups_state and immediate predecessors for
+/// a specific room.
+///
+/// - Fetches first [groups_to_compress] rows with group id higher than min
+/// - Stores the group id, predecessor id and deltas into a map
+/// - returns map and maximum row that was considered
+///
+/// # Arguments
+///
+/// * `client`          -   A Postgres client to make requests with
+/// * `room_id`         -   The ID of the room in the database
+/// * `min_state_group` -   If specified, then only fetch the entries for state
+///                         groups greater than (but not equal) to this number. It
+///                         also requires groups_to_compress to be specified
+/// * 'max_group_found' -   The upper limit on state_groups ids to get from the database
+fn get_initial_data_from_db(
+    client: &mut Client,
+    room_id: &str,
+    min_state_group: Option<i64>,
+    max_group_found: i64,
+) -> BTreeMap<i64, StateGroupEntry> {
+    // Query to get id, predecessor and deltas for each state group
+    let sql = r#"
+        SELECT m.id, prev_state_group, type, state_key, s.event_id
+        FROM state_groups AS m
+        LEFT JOIN state_groups_state AS s ON (m.id = s.state_group)
+        LEFT JOIN state_group_edges AS e ON (m.id = e.state_group)
+        WHERE m.room_id = $1 AND m.id <= $2
+    "#;
+
+    // Adds additional constraint if minimum state_group has been specified.
+    let mut rows = if let Some(min) = min_state_group {
+        let params: Vec<&dyn ToSql> = vec![&room_id, &max_group_found, &min];
+        client.query_raw(format!(r"{} AND m.id > $3", sql).as_str(), params)
+    } else {
+        let params: Vec<&dyn ToSql> = vec![&room_id, &max_group_found];
+        client.query_raw(sql, params)
+    }
+    .expect("Something went wrong while querying the database");
+
+    // Copy the data from the database into a map
+    let mut state_group_map: BTreeMap<i64, StateGroupEntry> = BTreeMap::new();
+
+    let pb: ProgressBar = if cfg!(feature = "no-progress-bars") {
+        ProgressBar::hidden()
+    } else {
+        ProgressBar::new_spinner()
+    };
+    pb.set_style(
+        ProgressStyle::default_spinner().template("{spinner} [{elapsed}] {pos} rows retrieved"),
+    );
+    pb.enable_steady_tick(100);
+
+    while let Some(row) = rows.next().unwrap() {
+        // The row in the map to copy the data to
+        let entry = state_group_map.entry(row.get(0)).or_default();
+
+        // Save the predecessor and mark for compression (this may already be there)
+        // TODO: slightly fewer redundant rewrites
+        entry.prev_state_group = row.get(1);
+        entry.in_range = true;
+
+        // Copy the single delta from the predecessor stored in this row
+        if let Some(etype) = row.get::<_, Option<String>>(2) {
+            entry.state_map.insert(
+                &etype,
+                &row.get::<_, String>(3),
+                row.get::<_, String>(4).into(),
+            );
+        }
+
+        pb.inc(1);
+    }
+
+    pb.set_length(pb.position());
+    pb.finish();
+
+    state_group_map
+}
+
+/// Finds the predecessors of missing state groups
+///
+/// N.B. this does NOT find their deltas
+///
+/// # Arguments
+///
+/// * `client`          -   A Postgres client to make requests with
+/// * `missing_sgs`     -   An array of missing state_group ids
+/// * 'min_state_group' -   Minimum state_group id to mark as in range
+/// * 'max_group_found' -   Maximum state_group id to mark as in range
+fn get_missing_from_db(
+    client: &mut Client,
+    missing_sgs: &[i64],
+    min_state_group: Option<i64>,
+    max_group_found: i64,
+) -> BTreeMap<i64, StateGroupEntry> {
+    // "Due to reasons" it is possible that some states only appear in edges table and not in state_groups table
+    // so since we know the IDs we're looking for as they are the missing predecessors, we can find them by
+    // left joining onto the edges table (instead of the state_group table!)
+    let sql = r#"
+        SELECT target.prev_state_group, source.prev_state_group, state.type, state.state_key, state.event_id
+        FROM state_group_edges AS target
+        LEFT JOIN state_group_edges AS source ON (target.prev_state_group = source.state_group)
+        LEFT JOIN state_groups_state AS state ON (target.prev_state_group = state.state_group)
+        WHERE target.prev_state_group = ANY($1)
+    "#;
+
+    let mut rows = client.query_raw(sql, &[missing_sgs]).unwrap();
+
+    let mut state_group_map: BTreeMap<i64, StateGroupEntry> = BTreeMap::new();
+
+    while let Some(row) = rows.next().unwrap() {
+        let id = row.get(0);
+        // The row in the map to copy the data to
+        let entry = state_group_map.entry(id).or_default();
+
+        // Save the predecessor and mark for compression (this may already be there)
+        // Also may well not exist!
+        entry.prev_state_group = row.get(1);
+        if let Some(min) = min_state_group {
+            if min < id && id <= max_group_found {
+                entry.in_range = true
+            }
+        }
+
+        // Copy the single delta from the predecessor stored in this row
+        if let Some(etype) = row.get::<_, Option<String>>(2) {
+            entry.state_map.insert(
+                &etype,
+                &row.get::<_, String>(3),
+                row.get::<_, String>(4).into(),
+            );
+        }
+    }
+
+    state_group_map
+}
+
+// TODO: find a library that has an existing safe postgres escape function
+/// Helper function that escapes the wrapped text when writing SQL
+pub struct PGEscape<'a>(pub &'a str);
+
+impl<'a> fmt::Display for PGEscape<'a> {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        let mut delim = Cow::from("$$");
+        while self.0.contains(&delim as &str) {
+            let s: String = thread_rng()
+                .sample_iter(&Alphanumeric)
+                .take(10)
+                .map(char::from)
+                .collect();
+
+            delim = format!("${}$", s).into();
+        }
+
+        write!(f, "{}{}{}", delim, self.0, delim)
+    }
+}
+
+#[test]
+fn test_pg_escape() {
+    let s = format!("{}", PGEscape("test"));
+    assert_eq!(s, "$$test$$");
+
+    let dodgy_string = "test$$ing";
+
+    let s = format!("{}", PGEscape(dodgy_string));
+
+    // prefix and suffixes should match
+    let start_pos = s.find(dodgy_string).expect("expected to find dodgy string");
+    let end_pos = start_pos + dodgy_string.len();
+    assert_eq!(s[..start_pos], s[end_pos..]);
+
+    // .. and they should start and end with '$'
+    assert_eq!(&s[0..1], "$");
+    assert_eq!(&s[start_pos - 1..start_pos], "$");
+}
+
+/// Send changes to the database
+///
+/// Note that currently ignores config.transactions and wraps every state
+/// group in it's own transaction (i.e. as if config.transactions was true)
+///
+/// # Arguments
+///
+/// * `db_url`  -   The URL of a Postgres database. This should be of the
+///                 form: "postgresql://user:pass@domain:port/database"
+/// * `room_id` -   The ID of the room in the database
+/// * `old_map` -   The state group data originally in the database
+/// * `new_map` -   The state group data generated by the compressor to
+///                 replace replace the old contents
+pub fn send_changes_to_db(
+    db_url: &str,
+    room_id: &str,
+    old_map: &BTreeMap<i64, StateGroupEntry>,
+    new_map: &BTreeMap<i64, StateGroupEntry>,
+) {
+    // connect to the database
+    let mut builder = SslConnector::builder(SslMethod::tls()).unwrap();
+    builder.set_verify(SslVerifyMode::NONE);
+    let connector = MakeTlsConnector::new(builder.build());
+
+    let mut client = Client::connect(db_url, connector).unwrap();
+
+    debug!("Writing changes...");
+
+    // setup the progress bar
+    let pb: ProgressBar = if cfg!(feature = "no-progress-bars") {
+        ProgressBar::hidden()
+    } else {
+        ProgressBar::new(old_map.len() as u64)
+    };
+    pb.set_style(
+        ProgressStyle::default_bar().template("[{elapsed_precise}] {bar} {pos}/{len} {msg}"),
+    );
+    pb.set_message("state groups");
+    pb.enable_steady_tick(100);
+
+    for sql_transaction in generate_sql(old_map, new_map, room_id) {
+        if sql_transaction.is_empty() {
+            pb.inc(1);
+            continue;
+        }
+
+        // commit this change to the database
+        // N.B. this is a synchronous library so will wait until finished before continueing...
+        // if want to speed up compressor then this might be a good place to start!
+        let mut single_group_transaction = client.transaction().unwrap();
+        single_group_transaction
+            .batch_execute(&sql_transaction)
+            .unwrap();
+        single_group_transaction.commit().unwrap();
+
+        pb.inc(1);
+    }
+
+    pb.finish();
+}

src/graphing.rs

@@ -0,0 +1,71 @@
+use std::collections::BTreeMap;
+use std::{fs::File, io::Write};
+
+use super::StateGroupEntry;
+
+type Graph = BTreeMap<i64, StateGroupEntry>;
+
+/// Outputs information from a state group graph into an edges file and a node file
+///
+/// These can be loaded into something like Gephi to visualise the graphs
+///
+/// # Arguments
+///
+/// * `groups`          - A map from state group ids to StateGroupEntries
+/// * `edges_output`    - The file to output the predecessor link information to
+/// * `nodes_output`    - The file to output the state group information to
+fn output_csv(groups: &Graph, edges_output: &mut File, nodes_output: &mut File) {
+    // The line A;B in the edges file means:
+    //      That state group A has predecessor B
+    writeln!(edges_output, "Source;Target",).unwrap();
+
+    // The line A;B;C;"B" in the nodes file means:
+    //      The state group id is A
+    //      This state group has B rows in the state_groups_state table
+    //      If C is true then A has no predecessor
+    writeln!(nodes_output, "Id;Rows;Root;Label",).unwrap();
+
+    for (source, entry) in groups {
+        // If the group has a predecessor then write an edge in the edges file
+        if let Some(target) = entry.prev_state_group {
+            writeln!(edges_output, "{};{}", source, target,).unwrap();
+        }
+
+        // Write the state group's information to the nodes file
+        writeln!(
+            nodes_output,
+            "{};{};{};\"{}\"",
+            source,
+            entry.state_map.len(),
+            entry.prev_state_group.is_none(),
+            entry.state_map.len(),
+        )
+        .unwrap();
+    }
+}
+
+/// Outputs information from two state group graph into files
+///
+/// These can be loaded into something like Gephi to visualise the graphs
+/// before and after the compressor is run
+///
+/// # Arguments
+///
+/// * `before`      - A map from state group ids to StateGroupEntries
+///                   the information from this map goes into before_edges.csv
+///                   and before_nodes.csv
+/// * `after`       - A map from state group ids to StateGroupEntries
+///                   the information from this map goes into after_edges.csv
+///                   and after_nodes.csv
+pub fn make_graphs(before: &Graph, after: &Graph) {
+    // Open all the files to output to
+    let mut before_edges_file = File::create("before_edges.csv").unwrap();
+    let mut before_nodes_file = File::create("before_nodes.csv").unwrap();
+    let mut after_edges_file = File::create("after_edges.csv").unwrap();
+    let mut after_nodes_file = File::create("after_nodes.csv").unwrap();
+
+    // Write before's information to before_edges and before_nodes
+    output_csv(before, &mut before_edges_file, &mut before_nodes_file);
+    // Write afters's information to after_edges and after_nodes
+    output_csv(after, &mut after_edges_file, &mut after_nodes_file);
+}

src/main.rs

@@ -0,0 +1,46 @@
+// Copyright 2018 New Vector Ltd
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! This is a tool that attempts to further compress state maps within a
+//! Synapse instance's database. Specifically, it aims to reduce the number of
+//! rows that a given room takes up in the `state_groups_state` table.
+
+#[cfg(feature = "jemalloc")]
+#[global_allocator]
+static GLOBAL: tikv_jemallocator::Jemalloc = tikv_jemallocator::Jemalloc;
+
+use log::LevelFilter;
+use std::env;
+use std::io::Write;
+
+use synapse_compress_state as comp_state;
+
+fn main() {
+    // setup the logger
+    // The default can be overwritten with RUST_LOG
+    // see the README for more information
+    if env::var("RUST_LOG").is_err() {
+        let mut log_builder = env_logger::builder();
+        // Only output the log message (and not the prefixed timestamp etc.)
+        log_builder.format(|buf, record| writeln!(buf, "{}", record.args()));
+        // By default print all of the debugging messages from this library
+        log_builder.filter_module("synapse_compress_state", LevelFilter::Debug);
+        log_builder.init();
+    } else {
+        // If RUST_LOG was set then use that
+        env_logger::Builder::from_env("RUST_LOG").init();
+    }
+
+    comp_state::run(comp_state::Config::parse_arguments());
+}

synapse_auto_compressor/Cargo.toml

@@ -0,0 +1,36 @@
+[package]
+name = "synapse_auto_compressor"
+authors = ["William Ashton"]
+version = "0.1.2"
+edition = "2018"
+
+[package.metadata.maturin]
+requires-python = ">=3.6"
+project-url = {Source = "https://github.com/matrix-org/rust-synapse-compress-state"}
+classifier = [
+    "Development Status :: 4 - Beta",
+    "Programming Language :: Rust",
+]
+
+[dependencies]
+clap = "2.33.0"
+openssl = "0.10.32"
+postgres = "0.19.0"
+postgres-openssl = "0.5.0"
+tikv-jemallocator = "0.4.1"
+rand = "0.8.0"
+serial_test = "0.5.1"
+synapse_compress_state = { path = "../", features = ["no-progress-bars"] }
+env_logger = "0.9.0"
+log = "0.4.14"
+log-panics = "2.0.0"
+anyhow = "1.0.42"
+pyo3-log = "0.4.0"
+
+# Needed for pyo3 support
+[lib]
+crate-type = ["cdylib", "rlib"]
+
+[dependencies.pyo3]
+version = "0.14.1"
+features = ["extension-module","abi3-py36"]

synapse_auto_compressor/README.md

@@ -0,0 +1,12 @@
+# Auto Compressor
+
+See the top level readme for information.
+
+
+## Publishing to PyPI
+
+Bump the version number and run from the root directory of the repo:
+
+```
+docker run -it --rm -v $(pwd):/io -e OPENSSL_STATIC=1 konstin2/maturin publish -m synapse_auto_compressor/Cargo.toml --cargo-extra-args "\--features='openssl/vendored'"
+```

synapse_auto_compressor/src/lib.rs

@@ -0,0 +1,128 @@
+//! This is a tool that uses the synapse_compress_state library to
+//! reduce the size of the synapse state_groups_state table in a postgres
+//! database.
+//!
+//! It adds the tables state_compressor_state and state_compressor_progress
+//! to the database and uses these to enable it to incrementally work
+//! on space reductions
+
+use anyhow::Result;
+use log::{error, LevelFilter};
+use pyo3::{
+    exceptions::PyRuntimeError, prelude::pymodule, types::PyModule, PyErr, PyResult, Python,
+};
+use std::str::FromStr;
+
+use synapse_compress_state::Level;
+
+pub mod manager;
+pub mod state_saving;
+
+/// Helper struct for parsing the `default_levels` argument.
+///
+/// The compressor keeps track of a number of Levels, each of which have a maximum length,
+/// current length, and an optional current head (None if level is empty, Some if a head
+/// exists).
+///
+/// This is needed since FromStr cannot be implemented for structs
+/// that aren't defined in this scope
+#[derive(PartialEq, Debug)]
+pub struct LevelInfo(pub Vec<Level>);
+
+// Implement FromStr so that an argument of the form "100,50,25"
+// can be used to create a vector of levels with max sizes 100, 50 and 25
+// For more info see the LevelState documentation in lib.rs
+impl FromStr for LevelInfo {
+    type Err = &'static str;
+
+    fn from_str(s: &str) -> Result<Self, Self::Err> {
+        // Stores the max sizes of each level
+        let mut level_info: Vec<Level> = Vec::new();
+
+        // Split the string up at each comma
+        for size_str in s.split(',') {
+            // try and convert each section into a number
+            // panic if that fails
+            let size: usize = size_str
+                .parse()
+                .map_err(|_| "Not a comma separated list of numbers")?;
+            // add this parsed number to the sizes struct
+            level_info.push(Level::new(size));
+        }
+
+        // Return the built up vector inside a LevelInfo struct
+        Ok(LevelInfo(level_info))
+    }
+}
+
+// PyO3 INTERFACE STARTS HERE
+#[pymodule]
+fn synapse_auto_compressor(_py: Python, m: &PyModule) -> PyResult<()> {
+    let _ = pyo3_log::Logger::default()
+        // don't send out anything lower than a warning from other crates
+        .filter(LevelFilter::Warn)
+        // don't log warnings from synapse_compress_state, the
+        // synapse_auto_compressor handles these situations and provides better
+        // log messages
+        .filter_target("synapse_compress_state".to_owned(), LevelFilter::Error)
+        // log info and above for the synapse_auto_compressor
+        .filter_target("synapse_auto_compressor".to_owned(), LevelFilter::Debug)
+        .install();
+    // ensure any panics produce error messages in the log
+    log_panics::init();
+
+    #[pyfn(m, compress_largest_rooms)]
+    fn compress_state_events_table(
+        py: Python,
+        db_url: String,
+        chunk_size: i64,
+        default_levels: String,
+        number_of_chunks: i64,
+    ) -> PyResult<()> {
+        // Stops the compressor from holding the GIL while running
+        py.allow_threads(|| {
+            _compress_state_events_table_body(db_url, chunk_size, default_levels, number_of_chunks)
+        })
+    }
+
+    // Not accessbile through Py03. It is a "private" function.
+    fn _compress_state_events_table_body(
+        db_url: String,
+        chunk_size: i64,
+        default_levels: String,
+        number_of_chunks: i64,
+    ) -> PyResult<()> {
+        // Announce the start of the program to the logs
+        log::info!("synapse_auto_compressor started");
+
+        // Parse the default_level string into a LevelInfo struct
+        let default_levels: LevelInfo = match default_levels.parse() {
+            Ok(l_sizes) => l_sizes,
+            Err(e) => {
+                return Err(PyErr::new::<PyRuntimeError, _>(format!(
+                    "Unable to parse level_sizes: {}",
+                    e
+                )))
+            }
+        };
+
+        // call compress_largest_rooms with the arguments supplied
+        let run_result = manager::compress_chunks_of_database(
+            &db_url,
+            chunk_size,
+            &default_levels.0,
+            number_of_chunks,
+        );
+
+        // (Note, need to do `{:?}` formatting to show error context)
+        // Don't log the context of errors but do use it in the PyRuntimeError
+        if let Err(e) = run_result {
+            error!("{}", e);
+            return Err(PyErr::new::<PyRuntimeError, _>(format!("{:?}", e)));
+        }
+
+        log::info!("synapse_auto_compressor finished");
+        Ok(())
+    }
+    Ok(())
+}

synapse_auto_compressor/src/main.rs

@@ -0,0 +1,152 @@
+//! This is a tool that uses the synapse_compress_state library to
+//! reduce the size of the synapse state_groups_state table in a postgres
+//! database.
+//!
+//! It adds the tables state_compressor_state and state_compressor_progress
+//! to the database and uses these to enable it to incrementally work
+//! on space reductions.
+//!
+//! This binary calls manager::compress_largest_rooms() with the arguments
+//! provided. That is, it compresses (in batches) the top N rooms ranked by
+//! amount of "uncompressed" state. This is measured by the number of rows in
+//! the state_groups_state table.
+//!
+//! After each batch, the rows processed are marked as "compressed" (using
+//! the state_compressor_progress table), and the program state is saved into
+//! the state_compressor_state table so that the compressor can seemlesly
+//! continue from where it left off.
+
+#[global_allocator]
+static GLOBAL: tikv_jemallocator::Jemalloc = tikv_jemallocator::Jemalloc;
+
+use clap::{crate_authors, crate_description, crate_name, crate_version, value_t, App, Arg};
+use log::LevelFilter;
+use std::env;
+use synapse_auto_compressor::{manager, state_saving, LevelInfo};
+
+/// Execution starts here
+fn main() {
+    // setup the logger for the synapse_auto_compressor
+    // The default can be overwritten with RUST_LOG
+    // see the README for more information
+    if env::var("RUST_LOG").is_err() {
+        let mut log_builder = env_logger::builder();
+        // Ensure panics still come through
+        log_builder.filter_module("panic", LevelFilter::Error);
+        // Only output errors from the synapse_compress state library
+        log_builder.filter_module("synapse_compress_state", LevelFilter::Error);
+        // Output log levels info and above from synapse_auto_compressor
+        log_builder.filter_module("synapse_auto_compressor", LevelFilter::Info);
+        log_builder.init();
+    } else {
+        // If RUST_LOG was set then use that
+        let mut log_builder = env_logger::Builder::from_env("RUST_LOG");
+        // Ensure panics still come through
+        log_builder.filter_module("panic", LevelFilter::Error);
+        log_builder.init();
+    }
+    log_panics::init();
+    // Announce the start of the program to the logs
+    log::info!("synapse_auto_compressor started");
+
+    // parse the command line arguments using the clap crate
+    let arguments = App::new(crate_name!())
+        .version(crate_version!())
+        .author(crate_authors!("\n"))
+        .about(crate_description!())
+        .arg(
+            Arg::with_name("postgres-url")
+                .short("p")
+                .value_name("POSTGRES_LOCATION")
+                .help("The configruation for connecting to the postgres database.")
+                .long_help(concat!(
+                    "The configuration for connecting to the Postgres database. This should be of the form ",
+                    r#""postgresql://username:password@mydomain.com/database" or a key-value pair "#,
+                    r#"string: "user=username password=password dbname=database host=mydomain.com" "#,
+                    "See https://docs.rs/tokio-postgres/0.7.2/tokio_postgres/config/struct.Config.html ",
+                    "for the full details."
+                ))
+                .takes_value(true)
+                .required(true),
+        ).arg(
+            Arg::with_name("chunk_size")
+                .short("c")
+                .value_name("COUNT")
+                .help("The maximum number of state groups to load into memroy at once")
+                .long_help(concat!(
+                    "The number of state_groups to work on at once. All of the entries",
+                    " from state_groups_state are requested from the database",
+                    " for state groups that are worked on. Therefore small",
+                    " chunk sizes may be needed on machines with low memory.",
+                    " (Note: if the compressor fails to find space savings on the",
+                    " chunk as a whole (which may well happen in rooms with lots",
+                    " of backfill in) then the entire chunk is skipped.)",
+                ))
+                .takes_value(true)
+                .required(true),
+        ).arg(
+            Arg::with_name("default_levels")
+                .short("l")
+                .value_name("LEVELS")
+                .help("Sizes of each new level in the compression algorithm, as a comma separated list.")
+                .long_help(concat!(
+                    "Sizes of each new level in the compression algorithm, as a comma separated list.",
+                    " The first entry in the list is for the lowest, most granular level,",
+                    " with each subsequent entry being for the next highest level.",
+                    " The number of entries in the list determines the number of levels",
+                    " that will be used.",
+                    "\nThe sum of the sizes of the levels effect the performance of fetching the state",
+                    " from the database, as the sum of the sizes is the upper bound on number of",
+                    " iterations needed to fetch a given set of state.",
+                ))
+                .default_value("100,50,25")
+                .takes_value(true)
+                .required(false),
+        ).arg(
+            Arg::with_name("number_of_chunks")
+                .short("n")
+                .value_name("CHUNKS_TO_COMPRESS")
+                .help("The number of chunks to compress")
+                .long_help(concat!(
+                    "This many chunks of the database will be compressed. The higher this number is set to, ",
+                    "the longer the compressor will run for."
+                ))
+                .takes_value(true)
+                .required(true),
+        ).get_matches();
+
+    // The URL of the database
+    let db_url = arguments
+        .value_of("postgres-url")
+        .expect("A database url is required");
+
+    // The number of state groups to work on at once
+    let chunk_size = arguments
+        .value_of("chunk_size")
+        .map(|s| s.parse().expect("chunk_size must be an integer"))
+        .expect("A chunk size is required");
+
+    // The default structure to use when compressing
+    let default_levels = value_t!(arguments, "default_levels", LevelInfo)
+        .unwrap_or_else(|e| panic!("Unable to parse default levels: {}", e));
+
+    // The number of rooms to compress with this tool
+    let number_of_chunks = arguments
+        .value_of("number_of_chunks")
+        .map(|s| s.parse().expect("number_of_chunks must be an integer"))
+        .expect("number_of_chunks is required");
+
+    // Connect to the database and create the 2 tables this tool needs
+    // (Note: if they already exist then this does nothing)
+    let mut client = state_saving::connect_to_database(db_url)
+        .unwrap_or_else(|e| panic!("Error occured while connecting to {}: {}", db_url, e));
+    state_saving::create_tables_if_needed(&mut client)
+        .unwrap_or_else(|e| panic!("Error occured while creating tables in database: {}", e));
+
+    // call compress_largest_rooms with the arguments supplied
+    // panic if an error is produced
+    manager::compress_chunks_of_database(db_url, chunk_size, &default_levels.0, number_of_chunks)
+        .unwrap();
+
+    log::info!("synapse_auto_compressor finished");
+}

synapse_auto_compressor/src/manager.rs

@@ -0,0 +1,194 @@
+// This module contains functions that carry out diffferent types
+// of compression on the database.
+
+use crate::state_saving::{
+    connect_to_database, create_tables_if_needed, get_next_room_to_compress,
+    read_room_compressor_state, write_room_compressor_state,
+};
+use anyhow::{bail, Context, Result};
+use log::{debug, info, warn};
+use synapse_compress_state::{continue_run, ChunkStats, Level};
+
+/// Runs the compressor on a chunk of the room
+///
+/// Returns `Some(chunk_stats)` if the compressor has progressed
+/// and `None` if it had already got to the end of the room
+///
+/// # Arguments
+///
+/// * `db_url`          -   The URL of the postgres database that synapse is using.
+///                         e.g. "postgresql://user:password@domain.com/synapse"
+///
+/// * `room_id`         -   The id of the room to run the compressor on. Note this
+///                         is the id as stored in the database and will look like
+///                         "!aasdfasdfafdsdsa:matrix.org" instead of the common
+///                         name
+///
+/// * `chunk_size`      -   The number of state_groups to work on. All of the entries
+///                         from state_groups_state are requested from the database
+///                         for state groups that are worked on. Therefore small
+///                         chunk sizes may be needed on machines with low memory.
+///                         (Note: if the compressor fails to find space savings on the
+///                         chunk as a whole (which may well happen in rooms with lots
+///                         of backfill in) then the entire chunk is skipped.)
+///
+/// * `default_levels`  -   If the compressor has never been run on this room before
+///                         then we need to provide the compressor with some information
+///                         on what sort of compression structure we want. The default that
+///                         the library suggests is `vec![Level::new(100), Level::new(50), Level::new(25)]`
+pub fn run_compressor_on_room_chunk(
+    db_url: &str,
+    room_id: &str,
+    chunk_size: i64,
+    default_levels: &[Level],
+) -> Result<Option<ChunkStats>> {
+    // connect to the database
+    let mut client =
+        connect_to_database(db_url).with_context(|| format!("Failed to connect to {}", db_url))?;
+
+    // Access the database to find out where the compressor last got up to
+    let retrieved_state = read_room_compressor_state(&mut client, room_id)
+        .with_context(|| format!("Failed to read compressor state for room {}", room_id,))?;
+
+    // If the database didn't contain any information, then use the default state
+    let (start, level_info) = match retrieved_state {
+        Some((s, l)) => (Some(s), l),
+        None => (None, default_levels.to_vec()),
+    };
+
+    // run the compressor on this chunk
+    let option_chunk_stats = continue_run(start, chunk_size, db_url, room_id, &level_info);
+
+    if option_chunk_stats.is_none() {
+        debug!("No work to do on this room...");
+        return Ok(None);
+    }
+
+    // Ok to unwrap because have checked that it's not None
+    let chunk_stats = option_chunk_stats.unwrap();
+
+    debug!("{:?}", chunk_stats);
+
+    // Check to see whether the compressor sent its changes to the database
+    if !chunk_stats.commited {
+        if chunk_stats.new_num_rows - chunk_stats.original_num_rows != 0 {
+            warn!(
+                "The compressor tried to increase the number of rows in {} between {:?} and {}. Skipping...",
+                room_id, start, chunk_stats.last_compressed_group,
+            );
+        }
+
+        // Skip over the failed chunk and set the level info to the default (empty) state
+        write_room_compressor_state(
+            &mut client,
+            room_id,
+            default_levels,
+            chunk_stats.last_compressed_group,
+        )
+        .with_context(|| {
+            format!(
+                "Failed to skip chunk in room {} between {:?} and {}",
+                room_id, start, chunk_stats.last_compressed_group
+            )
+        })?;
+
+        return Ok(Some(chunk_stats));
+    }
+
+    // Save where we got up to after this successful commit
+    write_room_compressor_state(
+        &mut client,
+        room_id,
+        &chunk_stats.new_level_info,
+        chunk_stats.last_compressed_group,
+    )
+    .with_context(|| {
+        format!(
+            "Failed to save state after compressing chunk in room {} between {:?} and {}",
+            room_id, start, chunk_stats.last_compressed_group
+        )
+    })?;
+
+    Ok(Some(chunk_stats))
+}
+
+/// Runs the compressor in chunks on rooms with the lowest uncompressed state group ids
+///
+/// # Arguments
+///
+/// * `db_url`          -   The URL of the postgres database that synapse is using.
+///                         e.g. "postgresql://user:password@domain.com/synapse"
+///
+/// * `chunk_size`      -   The number of state_groups to work on. All of the entries
+///                         from state_groups_state are requested from the database
+///                         for state groups that are worked on. Therefore small
+///                         chunk sizes may be needed on machines with low memory.
+///                         (Note: if the compressor fails to find space savings on the
+///                         chunk as a whole (which may well happen in rooms with lots
+///                         of backfill in) then the entire chunk is skipped.)
+///
+/// * `default_levels`  -   If the compressor has never been run on this room before
+///                         Then we need to provide the compressor with some information
+///                         on what sort of compression structure we want. The default that
+///                         the library suggests is empty levels with max sizes of 100, 50 and 25
+///
+/// * `number_of_chunks`-   The number of chunks to compress. The larger this number is, the longer
+///                         the compressor will run for.
+pub fn compress_chunks_of_database(
+    db_url: &str,
+    chunk_size: i64,
+    default_levels: &[Level],
+    number_of_chunks: i64,
+) -> Result<()> {
+    // connect to the database
+    let mut client = connect_to_database(db_url)
+        .with_context(|| format!("Failed to connect to database at {}", db_url))?;
+
+    create_tables_if_needed(&mut client).context("Failed to create state compressor tables")?;
+
+    let mut skipped_chunks = 0;
+    let mut rows_saved = 0;
+    let mut chunks_processed = 0;
+
+    while chunks_processed < number_of_chunks {
+        let room_to_compress = get_next_room_to_compress(&mut client)
+            .context("Failed to work out what room to compress next")?;
+
+        if room_to_compress.is_none() {
+            break;
+        }
+
+        let room_to_compress =
+            room_to_compress.expect("Have checked that rooms_to_compress is not None");
+
+        info!(
+            "Running compressor on room {} with chunk size {}",
+            room_to_compress, chunk_size
+        );
+
+        let work_done =
+            run_compressor_on_room_chunk(db_url, &room_to_compress, chunk_size, default_levels)?;
+
+        if let Some(ref chunk_stats) = work_done {
+            if chunk_stats.commited {
+                let savings = chunk_stats.original_num_rows - chunk_stats.new_num_rows;
+                rows_saved += chunk_stats.original_num_rows - chunk_stats.new_num_rows;
+                debug!("Saved {} rows for room {}", savings, room_to_compress);
+            } else {
+                skipped_chunks += 1;
+                debug!(
+                    "Unable to make savings for room {}, skipping chunk",
+                    room_to_compress
+                );
+            }
+            chunks_processed += 1;
+        } else {
+            bail!("Ran the compressor on a room that had no more work to do!")
+        }
+    }
+    info!(
+        "Finished running compressor. Saved {} rows. Skipped {}/{} chunks",
+        rows_saved, skipped_chunks, chunks_processed
+    );
+    Ok(())
+}

synapse_auto_compressor/src/state_saving.rs

@@ -0,0 +1,321 @@
+// This module contains functions to communicate with the database
+
+use anyhow::{bail, Result};
+use log::trace;
+use synapse_compress_state::Level;
+
+use openssl::ssl::{SslConnector, SslMethod, SslVerifyMode};
+use postgres::{fallible_iterator::FallibleIterator, types::ToSql, Client};
+use postgres_openssl::MakeTlsConnector;
+
+/// Connects to the database and returns a postgres client
+///
+/// # Arguments
+///
+/// * `db_url`          -   The URL of the postgres database that synapse is using.
+///                         e.g. "postgresql://user:password@domain.com/synapse"
+pub fn connect_to_database(db_url: &str) -> Result<Client> {
+    let mut builder = SslConnector::builder(SslMethod::tls())?;
+    builder.set_verify(SslVerifyMode::NONE);
+    let connector = MakeTlsConnector::new(builder.build());
+
+    let client = Client::connect(db_url, connector)?;
+    Ok(client)
+}
+
+/// Creates the state_compressor_state and state_compressor progress tables
+///
+/// If these tables already exist then this function does nothing
+///
+/// # Arguments
+///
+/// * `client`        - A postgres client used to send the requests to the database
+pub fn create_tables_if_needed(client: &mut Client) -> Result<()> {
+    let create_state_table = r#"
+        CREATE TABLE IF NOT EXISTS state_compressor_state (
+            room_id TEXT NOT NULL,
+            level_num INT NOT NULL,
+            max_size INT NOT NULL,
+            current_length INT NOT NULL,
+            current_head BIGINT,
+            UNIQUE (room_id, level_num)
+        )"#;
+
+    client.execute(create_state_table, &[])?;
+
+    let create_state_table_indexes = r#"
+        CREATE INDEX IF NOT EXISTS state_compressor_state_index ON state_compressor_state (room_id)"#;
+
+    client.execute(create_state_table_indexes, &[])?;
+
+    let create_progress_table = r#"
+        CREATE TABLE IF NOT EXISTS state_compressor_progress (
+            room_id TEXT PRIMARY KEY,
+            last_compressed BIGINT NOT NULL
+        )"#;
+
+    client.execute(create_progress_table, &[])?;
+
+    let create_compressor_global_progress_table = r#"
+        CREATE TABLE IF NOT EXISTS state_compressor_total_progress(
+            lock CHAR(1) NOT NULL DEFAULT 'X' UNIQUE,
+            lowest_uncompressed_group BIGINT NOT NULL,
+            CHECK (Lock='X')
+        );
+        INSERT INTO state_compressor_total_progress 
+            (lowest_uncompressed_group) 
+            VALUES (0)
+        ON CONFLICT (lock) DO NOTHING;
+    "#;
+
+    client.batch_execute(create_compressor_global_progress_table)?;
+
+    Ok(())
+}
+
+/// Retrieve the level info so we can restart the compressor
+///
+/// # Arguments
+///
+/// * `client`        - A postgres client used to send the requests to the database
+/// * `room_id`       - The room who's saved compressor state we want to load
+pub fn read_room_compressor_state(
+    client: &mut Client,
+    room_id: &str,
+) -> Result<Option<(i64, Vec<Level>)>> {
+    // Query to retrieve all levels from state_compressor_state
+    // Ordered by ascending level_number
+    let sql = r#"
+        SELECT level_num, max_size, current_length, current_head, last_compressed
+        FROM state_compressor_state 
+        LEFT JOIN state_compressor_progress USING (room_id)
+        WHERE room_id = $1
+        ORDER BY level_num ASC
+    "#;
+
+    // send the query to the database
+    let mut levels = client.query_raw(sql, &[room_id])?;
+
+    // Needed to ensure that the rows are for unique consecutive levels
+    // starting from 1 (i.e of form [1,2,3] not [0,1,2] or [1,1,2,2,3])
+    let mut prev_seen = 0;
+
+    // The vector to store the level info from the database in
+    let mut level_info: Vec<Level> = Vec::new();
+
+    // Where the last compressor run stopped
+    let mut last_compressed = None;
+    // Used to only read last_compressed value once
+    let mut first_row = true;
+
+    // Loop through all the rows retrieved by that query
+    while let Some(l) = levels.next()? {
+        // Read out the fields into variables
+        //
+        // Some of these are `usize` as they may be used to index vectors, but stored as Postgres
+        // type `INT` which is the same as`i32`.
+        //
+        // Since usize is unlikely to be ess than 32 bits wide, this conversion should be safe
+        let level_num: usize = l.get::<_, i32>("level_num") as usize;
+        let max_size: usize = l.get::<_, i32>("max_size") as usize;
+        let current_length: usize = l.get::<_, i32>("current_length") as usize;
+        let current_head: Option<i64> = l.get("current_head");
+
+        // Only read the last compressed column once since is the same for each row
+        if first_row {
+            last_compressed = l.get("last_compressed"); // might be NULL if corrupted
+            if last_compressed.is_none() {
+                bail!(
+                    "No entry in state_compressor_progress for room {} but entries in state_compressor_state were found",
+                    room_id
+                )
+            }
+            first_row = false;
+        }
+
+        // Check that there aren't multiple entries for the same level number
+        // in the database. (Should be impossible due to unique key constraint)
+        if prev_seen == level_num {
+            bail!(
+                "The level {} occurs twice in state_compressor_state for room {}",
+                level_num,
+                room_id,
+            );
+        }
+
+        // Check that there is no missing level in the database
+        // e.g. if the previous row retrieved was for level 1 and this
+        // row is for level 3 then since the SQL query orders the results
+        // in ascenting level numbers, there was no level 2 found!
+        if prev_seen != level_num - 1 {
+            bail!("Levels between {} and {} are missing", prev_seen, level_num,);
+        }
+
+        // if the level is not empty, then it must have a head!
+        if current_head.is_none() && current_length != 0 {
+            bail!(
+                "Level {} has no head but current length is {} in room {}",
+                level_num,
+                current_length,
+                room_id,
+            );
+        }
+
+        // If the level has more groups in than the maximum then something is wrong!
+        if current_length > max_size {
+            bail!(
+                "Level {} has length {} but max size {} in room {}",
+                level_num,
+                current_length,
+                max_size,
+                room_id,
+            );
+        }
+
+        // Add this level to the level_info vector
+        level_info.push(Level::restore(max_size, current_length, current_head));
+        // Mark the previous level_number seen as the current one
+        prev_seen = level_num;
+    }
+
+    // If we didn't retrieve anything from the database then there is no saved state
+    // in the database!
+    if level_info.is_empty() {
+        return Ok(None);
+    }
+
+    // Return the compressor state we retrieved
+    // last_compressed cannot be None at this point, so safe to unwrap
+    Ok(Some((last_compressed.unwrap(), level_info)))
+}
+
+/// Save the level info so it can be loaded by the next run of the compressor
+///
+/// # Arguments
+///
+/// * `client`            - A postgres client used to send the requests to the database
+/// * `room_id`           - The room who's saved compressor state we want to save
+/// * `level_info`        - The state that can be used to restore the compressor later
+/// * `last_compressed`   - The last state_group that was compressed. This is needed
+///                         so that the compressor knows where to start from next
+pub fn write_room_compressor_state(
+    client: &mut Client,
+    room_id: &str,
+    level_info: &[Level],
+    last_compressed: i64,
+) -> Result<()> {
+    // Wrap all the changes to the state for this room in a transaction
+    // This prevents accidentally having malformed compressor start info
+    let mut write_transaction = client.transaction()?;
+
+    // Go through every level that the compressor is using
+    for (level_num, level) in level_info.iter().enumerate() {
+        // the 1st level is level 1 not level 0, but enumerate starts at 0
+        // so need to add 1 to get correct number
+        let level_num = level_num + 1;
+
+        // bring the level info out of the Level struct
+        let (max_size, current_len, current_head) = (
+            level.get_max_length(),
+            level.get_current_length(),
+            level.get_head(),
+        );
+
+        // Update the database with this compressor state information
+        //
+        // Some of these are `usize` as they may be used to index vectors, but stored as Postgres
+        // type `INT` which is the same as`i32`.
+        //
+        // Since these values should always be small, this conversion should be safe.
+        let (level_num, max_size, current_len) =
+            (level_num as i32, max_size as i32, current_len as i32);
+        let params: Vec<&(dyn ToSql + Sync)> =
+            vec![&room_id, &level_num, &max_size, &current_len, &current_head];
+
+        write_transaction.execute(
+            r#"
+                INSERT INTO state_compressor_state 
+                    (room_id, level_num, max_size, current_length, current_head) 
+                    VALUES ($1, $2, $3, $4, $5)
+                ON CONFLICT (room_id, level_num) 
+                    DO UPDATE SET 
+                        max_size = excluded.max_size,
+                        current_length = excluded.current_length,
+                        current_head= excluded.current_head;
+            "#,
+            &params,
+        )?;
+    }
+
+    // Update the database with this progress information
+    let params: Vec<&(dyn ToSql + Sync)> = vec![&room_id, &last_compressed];
+    write_transaction.execute(
+        r#"
+            INSERT INTO state_compressor_progress (room_id, last_compressed) 
+                VALUES ($1, $2)
+            ON CONFLICT (room_id)
+                DO UPDATE SET last_compressed = excluded.last_compressed;
+        "#,
+        &params,
+    )?;
+
+    // Commit the transaction (otherwise changes never happen)
+    write_transaction.commit()?;
+
+    Ok(())
+}
+
+/// Returns the room with with the lowest uncompressed state group id
+///
+/// A group is detected as uncompressed if it is greater than the `last_compressed`
+/// entry in `state_compressor_progress` for that room.
+///
+/// The `lowest_uncompressed_group` value stored in `state_compressor_total_progress`
+/// stores where this method last finished, to prevent repeating work
+///
+/// # Arguments
+///
+/// * `client`    -   A postgres client used to send the requests to the database
+pub fn get_next_room_to_compress(client: &mut Client) -> Result<Option<String>> {
+    // Walk the state_groups table until find next uncompressed group
+    let get_next_room = r#"
+        SELECT room_id, id 
+        FROM state_groups
+        LEFT JOIN state_compressor_progress USING (room_id)
+        WHERE
+            id >= (SELECT lowest_uncompressed_group FROM state_compressor_total_progress)
+            AND (
+                id > last_compressed
+                OR last_compressed IS NULL
+            )
+        ORDER BY id ASC
+        LIMIT 1
+    "#;
+
+    let row_opt = client.query_opt(get_next_room, &[])?;
+
+    let next_room_row = if let Some(row) = row_opt {
+        row
+    } else {
+        return Ok(None);
+    };
+
+    let next_room: String = next_room_row.get("room_id");
+    let lowest_uncompressed_group: i64 = next_room_row.get("id");
+
+    // This method has determined where the lowest uncompressesed group is, save that
+    // information so we don't have to redo this work in the future.
+    let update_total_progress = r#"
+        UPDATE state_compressor_total_progress SET lowest_uncompressed_group = $1;
+    "#;
+
+    client.execute(update_total_progress, &[&lowest_uncompressed_group])?;
+
+    trace!(
+        "next_room: {}, lowest_uncompressed: {}",
+        next_room,
+        lowest_uncompressed_group
+    );
+
+    Ok(Some(next_room))
+}