Fmt

(This was fixed via `cargo clippy --fix`)

.gitea/workflows/docker.yaml

@@ -1,46 +0,0 @@
-name: Build and push docker images
-
-on:
-  push:
-    tags: ["v*"]
-    branches: [ main ]
-  workflow_dispatch:
-
-jobs:
-  build:
-    runs-on: ubuntu-latest
-    steps:
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v2
-
-      - name: Log in to Gitea Container Registry
-        uses: docker/login-action@v2
-        with:
-          registry: git.yongyuancv.cn
-          username: ${{ gitea.repository_owner }}
-          password: ${{ secrets.GITEA_TOKEN }}
-
-      - name: Calculate docker image tag
-        id: set-tag
-        uses: docker/metadata-action@master
-        with:
-          images: |
-            git.yongyuancv.cn/${{ gitea.repository }}
-            git.yongyuancv.cn/heimoshuiyu/${{ gitea.event.repository.name }}            
-          flavor: |
-            latest=false            
-          tags: |
-            type=raw,value=latest,enable=${{ gitea.ref == 'refs/heads/main' }}
-            type=sha,prefix=,format=long
-            type=semver,pattern=v{{version}}
-            type=semver,pattern=v{{major}}.{{minor}}            
-
-      - name: Build and push all platforms
-        uses: docker/build-push-action@v4
-        with:
-          push: true
-          labels: "gitsha1=${{ gitea.sha }}"
-          tags: "${{ steps.set-tag.outputs.tags }}"
-          platforms: linux/amd64,linux/arm64
-          cache-from: type=registry,ref=git.yongyuancv.cn/${{ gitea.repository }}:buildcache
-          cache-to: type=registry,ref=git.yongyuancv.cn/${{ gitea.repository }}:buildcache,mode=max

.github/workflows/docker.yaml

@@ -1,58 +0,0 @@
-# GitHub actions workflow which builds and publishes the docker images.
-
-name: Build and push docker images
-
-on:
-  push:
-    tags: ["v*"]
-    branches: [ main ]
-  workflow_dispatch:
-
-permissions:
-  contents: read
-  packages: write
-
-jobs:
-  build:
-    runs-on: ubuntu-latest
-    steps:
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v2
-
-      - name: Log in to DockerHub
-        uses: docker/login-action@v2
-        with:
-          username: ${{ secrets.DOCKER_HUB_USERNAME }}
-          password: ${{ secrets.DOCKER_HUB_TOKEN }}
-
-      - name: Log in to GHCR
-        uses: docker/login-action@v2
-        with:
-          registry: ghcr.io
-          username: ${{ github.repository_owner }}
-          password: ${{ secrets.GITHUB_TOKEN }}
-
-      - name: Calculate docker image tag
-        id: set-tag
-        uses: docker/metadata-action@master
-        with:
-          images: |
-            ghcr.io/${{ github.repository }}
-            docker.io/${{ secrets.DOCKER_HUB_USERNAME }}/${{ github.event.repository.name }}
-          flavor: |
-            latest=false
-          tags: |
-            type=raw,value=latest,enable=${{ github.ref == 'refs/heads/main' }}
-            type=sha,prefix=,format=long
-            type=semver,pattern=v{{version}}
-            type=semver,pattern=v{{major}}.{{minor}}
-
-      - name: Build and push all platforms
-        uses: docker/build-push-action@v4
-        with:
-          push: true
-          labels: "gitsha1=${{ github.sha }}"
-          tags: "${{ steps.set-tag.outputs.tags }}"
-          platforms: linux/amd64,linux/arm64
-          cache-from: type=registry,ref=ghcr.io/${{ github.repository }}:buildcache
-          cache-to: type=registry,ref=ghcr.io/${{ github.repository }}:buildcache,mode=max

Cargo.toml

@@ -8,21 +8,18 @@ name = "synapse_compress_state"
 version = "0.1.0"
 edition = "2018"
 
-[[bin]]
-name = "synapse_compress_state"
-required-features = ["clap"]
-
 [dependencies]
-indicatif = "0.17.6"
+indicatif = "0.17.0"
-openssl = "0.10.60"
+openssl = "0.10.32"
-postgres = "0.19.7"
+postgres = "0.19.0"
 postgres-openssl = "0.5.0"
-rand = "0.8.5"
+rand = "0.8.0"
-rayon = "1.7.0"
+rayon = "1.3.0"
-string_cache = "0.8.7"
+string_cache = "0.8.0"
-env_logger = "0.10.0"
+env_logger = "0.9.0"
-log = "0.4.20"
+log = "0.4.14"
-log-panics = "2.1.0"
+pyo3-log = "0.7.0"
+log-panics = "2.0.0"
 
 [dependencies.state-map]
 git = "https://github.com/matrix-org/rust-matrix-state-map"
@@ -32,25 +29,18 @@ git = "https://github.com/matrix-org/rust-matrix-state-map"
 crate-type = ["cdylib", "rlib"]
 
 [dependencies.clap]
-version = "4.4.2"
+version = "4.0.15"
 features = ["cargo"]
-optional = true
 
 [dependencies.pyo3]
-version = "0.19.2"
+version = "0.17.1"
 features = ["extension-module"]
-optional = true
-
-[dependencies.pyo3-log]
-version = "0.8.3"
-optional = true
 
 [dependencies.tikv-jemallocator]
-version = "0.5.4"
+version = "0.5.0"
 optional = true
 
 [features]
-default = ["clap", "jemalloc"]
+default = ["jemalloc"]
 jemalloc = ["tikv-jemallocator"]
 no-progress-bars = []
-pyo3 = ["dep:pyo3", "dep:pyo3-log"]

Dockerfile

@@ -1,54 +1,22 @@
-# This uses the multi-stage build feature of Docker to build the binaries for multiple architectures without QEMU.
+FROM docker.io/rust:alpine AS builder
-# The first stage is responsible for building binaries for all the supported architectures (amd64 and arm64), and the
-# second stage only copies the binaries for the target architecture.
-# We leverage Zig and cargo-zigbuild for providing a cross-compilation-capable C compiler and linker.
 
-ARG RUSTC_VERSION=1.72.0
+RUN apk add python3 musl-dev pkgconfig openssl-dev make
-ARG ZIG_VERSION=0.11.0
-ARG CARGO_ZIGBUILD_VERSION=0.17.1
 
-FROM --platform=${BUILDPLATFORM} docker.io/rust:${RUSTC_VERSION} AS builder
+ENV RUSTFLAGS="-C target-feature=-crt-static"
-
-# Install cargo-zigbuild for cross-compilation
-ARG CARGO_ZIGBUILD_VERSION
-RUN cargo install --locked cargo-zigbuild@=${CARGO_ZIGBUILD_VERSION}
-
-# Download zig compiler for cross-compilation
-ARG ZIG_VERSION
-RUN curl -L "https://ziglang.org/download/${ZIG_VERSION}/zig-linux-$(uname -m)-${ZIG_VERSION}.tar.xz" | tar -J -x -C /usr/local && \
-  ln -s "/usr/local/zig-linux-$(uname -m)-${ZIG_VERSION}/zig" /usr/local/bin/zig
-
-# Install all cross-compilation targets
-ARG RUSTC_VERSION
-RUN rustup target add  \
-    --toolchain "${RUSTC_VERSION}" \
-    x86_64-unknown-linux-musl \
-    aarch64-unknown-linux-musl
 
 WORKDIR /opt/synapse-compressor/
+
 COPY . .
 
-# Build for all targets
+RUN cargo build
-RUN cargo zigbuild \
-    --release \
-    --workspace \
-    --bins \
-    --features "openssl/vendored" \
-    --target aarch64-unknown-linux-musl \
-    --target x86_64-unknown-linux-musl
 
-# Move the binaries in a separate folder per architecture, so we can copy them using the TARGETARCH build arg
+WORKDIR /opt/synapse-compressor/synapse_auto_compressor/
-RUN mkdir -p /opt/binaries/amd64 /opt/binaries/arm64
-RUN mv target/x86_64-unknown-linux-musl/release/synapse_compress_state \
-       target/x86_64-unknown-linux-musl/release/synapse_auto_compressor \
-       /opt/binaries/amd64
-RUN mv target/aarch64-unknown-linux-musl/release/synapse_compress_state \
-       target/aarch64-unknown-linux-musl/release/synapse_auto_compressor \
-       /opt/binaries/arm64
 
-FROM --platform=${TARGETPLATFORM} docker.io/alpine
+RUN cargo build
 
-ARG TARGETARCH
+FROM docker.io/alpine
 
-COPY --from=builder /opt/binaries/${TARGETARCH}/synapse_compress_state /usr/local/bin/synapse_compress_state
+RUN apk add --no-cache libgcc
-COPY --from=builder /opt/binaries/${TARGETARCH}/synapse_auto_compressor /usr/local/bin/synapse_auto_compressor
+
+COPY --from=builder /opt/synapse-compressor/target/debug/synapse_compress_state /usr/local/bin/synapse_compress_state
+COPY --from=builder /opt/synapse-compressor/target/debug/synapse_auto_compressor /usr/local/bin/synapse_auto_compressor

README.md

@@ -36,17 +36,9 @@ This will create an executable and store it in
 `synapse_auto_compressor/target/debug/synapse_auto_compressor`.
 
 ## Example usage
-
-Compress 100 chunks of size 500 in a remote PostgreSQL database:
 ```
 $ synapse_auto_compressor -p postgresql://user:pass@localhost/synapse -c 500 -n 100
 ```
-
-Compress 100 chunks of size 500 using local PostgreSQL socket:
-```
-$ sudo -u postgres synapse_auto_compressor -p "user=postgres dbname=matrix-synapse host=/var/run/postgresql" -c 500 -n 100
-```
-
 ## Running Options
 
 - -p [POSTGRES_LOCATION] **Required**
@@ -67,7 +59,7 @@ in) then the entire chunk is skipped.
 *CHUNKS_TO_COMPRESS* chunks of size *CHUNK_SIZE* will be compressed. The higher this
 number is set to, the longer the compressor will run for.
 
-- -l [LEVELS]
+- -d [LEVELS]
 Sizes of each new level in the compression algorithm, as a comma-separated list.
 The first entry in the list is for the lowest, most granular level, with each
 subsequent entry being for the next highest level. The number of entries in the

compressor_integration_tests/Cargo.toml

@@ -6,16 +6,16 @@ edition = "2018"
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
-string_cache = "0.8.7"
+string_cache = "0.8.0"
-serial_test = "2.0.0"
+serial_test = "0.9.0"
-openssl = "0.10.60"
+openssl = "0.10.32"
-postgres = "0.19.7"
+postgres = "0.19.0"
 postgres-openssl = "0.5.0"
-rand = "0.8.5"
+rand = "0.8.0"
 synapse_compress_state = { path = "../", features = ["no-progress-bars"] }
 synapse_auto_compressor = { path = "../synapse_auto_compressor/" }
-env_logger = "0.10.0"
+env_logger = "0.9.0"
-log = "0.4.20"
+log = "0.4.14"
 
 [dependencies.state-map]
 git = "https://github.com/matrix-org/rust-matrix-state-map"

compressor_integration_tests/src/lib.rs

@@ -195,7 +195,7 @@ fn collapse_state_with_database(state_group: i64) -> StateMap<Atom> {
 
     while let Some(sg) = next_group {
         // get predecessor from state_group_edges
-        let mut pred = client.query_raw(query_pred, [sg]).unwrap();
+        let mut pred = client.query_raw(query_pred, &[sg]).unwrap();
 
         // set next_group to predecessor
         next_group = match pred.next().unwrap() {
@@ -209,7 +209,7 @@ fn collapse_state_with_database(state_group: i64) -> StateMap<Atom> {
         }
         drop(pred);
 
-        let mut rows = client.query_raw(query_deltas, [sg]).unwrap();
+        let mut rows = client.query_raw(query_deltas, &[sg]).unwrap();
 
         while let Some(row) = rows.next().unwrap() {
             // Copy the single delta from the predecessor stored in this row

pyproject.toml

@@ -1,8 +1,4 @@
 [build-system]
-requires = ["maturin>=1.0,<2.0"]
+requires = ["maturin>=0.11,<0.12"]
 build-backend = "maturin"
-
+cargo-extra-args = "--no-default-features"
-[tool.maturin]
-profile = "release"
-features = ["pyo3"]
-no-default-features = true

src/lib.rs

@@ -20,11 +20,9 @@
 // of arguments - this hopefully doesn't make the code unclear
 // #[allow(clippy::too_many_arguments)] is therefore used around some functions
 
-use log::{info, warn};
+use log::{info, warn, LevelFilter};
-#[cfg(feature = "pyo3")]
 use pyo3::{exceptions, prelude::*};
 
-#[cfg(feature = "clap")]
 use clap::{crate_authors, crate_description, crate_name, crate_version, Arg, Command};
 use indicatif::{ProgressBar, ProgressStyle};
 use rayon::prelude::*;
@@ -119,7 +117,6 @@ pub struct Config {
     verify: bool,
 }
 
-#[cfg(feature = "clap")]
 impl Config {
     /// Build up config from command line arguments
     pub fn parse_arguments() -> Config {
@@ -750,31 +747,23 @@ impl Config {
 /// Default arguments are equivalent to using the command line tool
 /// No default's are provided for db_url or room_id since these arguments
 /// are compulsory (so that new() act's like parse_arguments())
-#[cfg(feature = "pyo3")]
 #[allow(clippy::too_many_arguments)]
-#[pyfunction]
+#[pyfunction(
-#[pyo3(signature = (
     // db_url has no default
-    db_url,
-
     // room_id  has no default
-    room_id,
+    output_file = "None",
-
+    min_state_group = "None",
-    output_file = None,
+    groups_to_compress = "None",
-    min_state_group = None,
+    min_saved_rows = "None",
-    groups_to_compress = None,
+    max_state_group = "None",
-    min_saved_rows = None,
+    level_sizes = "String::from(\"100,50,25\")",
-    max_state_group = None,
-    level_sizes = String::from("100,50,25"),
-
     // have this default to true as is much worse to not have it if you need it
     // than to have it and not need it
     transactions = true,
-
     graphs = false,
     commit_changes = false,
     verify = true,
-))]
+)]
 fn run_compression(
     db_url: String,
     room_id: String,
@@ -813,15 +802,14 @@ fn run_compression(
 }
 
 /// Python module - "import synapse_compress_state" to use
-#[cfg(feature = "pyo3")]
 #[pymodule]
 fn synapse_compress_state(_py: Python, m: &PyModule) -> PyResult<()> {
     let _ = pyo3_log::Logger::default()
         // don't send out anything lower than a warning from other crates
-        .filter(log::LevelFilter::Warn)
+        .filter(LevelFilter::Warn)
         // don't log warnings from synapse_compress_state, the synapse_auto_compressor handles these
         // situations and provides better log messages
-        .filter_target("synapse_compress_state".to_owned(), log::LevelFilter::Debug)
+        .filter_target("synapse_compress_state".to_owned(), LevelFilter::Debug)
         .install();
     // ensure any panics produce error messages in the log
     log_panics::init();

synapse_auto_compressor/Cargo.toml

@@ -4,10 +4,6 @@ authors = ["William Ashton"]
 version = "0.1.3"
 edition = "2018"
 
-[[bin]]
-name = "synapse_auto_compressor"
-required-features = ["clap"]
-
 [package.metadata.maturin]
 requires-python = ">=3.7"
 project-url = {Source = "https://github.com/matrix-org/rust-synapse-compress-state"}
@@ -17,40 +13,34 @@ classifier = [
 ]
 
 [dependencies]
-openssl = { version = "0.10.60", features = ["vendored"] }
+openssl = "0.10.32"
-postgres = "0.19.7"
+postgres = "0.19.0"
 postgres-openssl = "0.5.0"
-rand = "0.8.5"
+rand = "0.8.0"
-serial_test = "2.0.0"
+serial_test = "0.9.0"
-synapse_compress_state = { path = "../", features = ["no-progress-bars"], default-features = false }
+synapse_compress_state = { path = "../", features = ["no-progress-bars"] }
-env_logger = "0.10.0"
+env_logger = "0.9.0"
-log = "0.4.20"
+log = "0.4.14"
-log-panics = "2.1.0"
+log-panics = "2.0.0"
-anyhow = "1.0.75"
+anyhow = "1.0.42"
+pyo3-log = "0.7.0"
 
 # Needed for pyo3 support
 [lib]
 crate-type = ["cdylib", "rlib"]
 
 [dependencies.clap]
-version = "4.4.2"
+version = "4.0.15"
 features = ["cargo"]
-optional = true
 
 [dependencies.pyo3]
-version = "0.19.2"
+version = "0.17.1"
 features = ["extension-module"]
-optional = true
-
-[dependencies.pyo3-log]
-version = "0.8.3"
-optional = true
 
 [dependencies.tikv-jemallocator]
-version = "0.5.4"
+version = "0.5.0"
 optional = true
 
 [features]
-default = ["clap", "jemalloc"]
+default = ["jemalloc"]
-jemalloc = ["tikv-jemallocator", "synapse_compress_state/jemalloc"]
+jemalloc = ["tikv-jemallocator"]
-pyo3 = ["dep:pyo3", "dep:pyo3-log", "synapse_compress_state/pyo3"]

synapse_auto_compressor/pyproject.toml

@@ -1,8 +0,0 @@
-[build-system]
-requires = ["maturin>=1.0,<2.0"]
-build-backend = "maturin"
-
-[tool.maturin]
-profile = "release"
-features = ["pyo3"]
-no-default-features = true

synapse_auto_compressor/src/lib.rs

@@ -7,9 +7,7 @@
 //! on space reductions
 
 use anyhow::Result;
-#[cfg(feature = "pyo3")]
 use log::{error, LevelFilter};
-#[cfg(feature = "pyo3")]
 use pyo3::{
     exceptions::PyRuntimeError, prelude::pymodule, types::PyModule, PyErr, PyResult, Python,
 };
@@ -58,7 +56,6 @@ impl FromStr for LevelInfo {
 }
 
 // PyO3 INTERFACE STARTS HERE
-#[cfg(feature = "pyo3")]
 #[pymodule]
 fn synapse_auto_compressor(_py: Python, m: &PyModule) -> PyResult<()> {
     let _ = pyo3_log::Logger::default()
@@ -74,8 +71,7 @@ fn synapse_auto_compressor(_py: Python, m: &PyModule) -> PyResult<()> {
     // ensure any panics produce error messages in the log
     log_panics::init();
 
-    #[pyfn(m)]
+    #[pyfn(m, compress_largest_rooms)]
-    #[pyo3(name = "compress_largest_rooms")]
     fn compress_state_events_table(
         py: Python,
         db_url: String,