chore: Update ruwuma

Also, don't take read receipts into consideration for lazy loading.
Synapse doesn't do this and they're making initial syncs very large.

.forgejo/actions/create-docker-manifest/action.yml

@@ -0,0 +1,108 @@
+name: create-manifest
+description: |
+  Create and push a multi-platform Docker manifest from individual platform digests.
+  Handles downloading digests, creating manifest lists, and pushing to registry.
+
+inputs:
+  digest_pattern:
+    description: Glob pattern to match digest artifacts (e.g. "digests-linux-{amd64,arm64}")
+    required: true
+  tag_suffix:
+    description: Suffix to add to all Docker tags (e.g. "-maxperf")
+    required: false
+    default: ""
+  images:
+    description: Container registry images (newline-separated)
+    required: true
+  registry_user:
+    description: Registry username for authentication
+    required: false
+  registry_password:
+    description: Registry password for authentication
+    required: false
+
+outputs:
+  version:
+    description: The version tag created for the manifest
+    value: ${{ steps.meta.outputs.version }}
+  tags:
+    description: All tags created for the manifest
+    value: ${{ steps.meta.outputs.tags }}
+
+runs:
+  using: composite
+  steps:
+    - name: Download digests
+      if: ${{ env.BUILTIN_REGISTRY_ENABLED == 'true' }}
+      uses: forgejo/download-artifact@v4
+      with:
+        path: /tmp/digests
+        pattern: ${{ inputs.digest_pattern }}
+        merge-multiple: true
+
+    - name: Login to builtin registry
+      if: ${{ env.BUILTIN_REGISTRY_ENABLED == 'true' }}
+      uses: docker/login-action@v3
+      with:
+        registry: ${{ env.BUILTIN_REGISTRY }}
+        username: ${{ inputs.registry_user }}
+        password: ${{ inputs.registry_password }}
+
+    - name: Set up Docker Buildx
+      if: ${{ env.BUILTIN_REGISTRY_ENABLED == 'true' }}
+      uses: docker/setup-buildx-action@v3
+      with:
+        # Use persistent BuildKit if BUILDKIT_ENDPOINT is set (e.g. tcp://buildkit:8125)
+        driver: ${{ env.BUILDKIT_ENDPOINT != '' && 'remote' || 'docker-container' }}
+        endpoint: ${{ env.BUILDKIT_ENDPOINT || '' }}
+
+    - name: Extract metadata (tags) for Docker
+      if: ${{ env.BUILTIN_REGISTRY_ENABLED == 'true' }}
+      id: meta
+      uses: docker/metadata-action@v5
+      with:
+        flavor: |
+          suffix=${{ inputs.tag_suffix }},onlatest=true
+        tags: |
+          type=semver,pattern={{version}},prefix=v
+          type=semver,pattern={{major}}.{{minor}},enable=${{ !startsWith(github.ref, 'refs/tags/v0.0.') }},prefix=v
+          type=semver,pattern={{major}},enable=${{ !startsWith(github.ref, 'refs/tags/v0.') }},prefix=v
+          type=ref,event=branch,prefix=${{ format('refs/heads/{0}', github.event.repository.default_branch) != github.ref && 'branch-' || '' }},
+          type=ref,event=pr
+          type=sha,format=short
+          type=raw,value=latest${{ inputs.tag_suffix }},enable=${{ startsWith(github.ref, 'refs/tags/v') }},priority=1100
+        images: ${{ inputs.images }}
+        # default labels & annotations: https://github.com/docker/metadata-action/blob/master/src/meta.ts#L509
+      env:
+        DOCKER_METADATA_ANNOTATIONS_LEVELS: index
+
+    - name: Create manifest list and push
+      if: ${{ env.BUILTIN_REGISTRY_ENABLED == 'true' }}
+      working-directory: /tmp/digests
+      shell: bash
+      env:
+        IMAGES: ${{ inputs.images }}
+      run: |
+        set -o xtrace
+        IFS=$'\n'
+        IMAGES_LIST=($IMAGES)
+        ANNOTATIONS_LIST=($DOCKER_METADATA_OUTPUT_ANNOTATIONS)
+        TAGS_LIST=($DOCKER_METADATA_OUTPUT_TAGS)
+        for REPO in "${IMAGES_LIST[@]}"; do
+            docker buildx imagetools create \
+              $(for tag in "${TAGS_LIST[@]}"; do echo "--tag"; echo "$tag"; done) \
+              $(for annotation in "${ANNOTATIONS_LIST[@]}"; do echo "--annotation"; echo "$annotation"; done) \
+              $(for reference in *; do printf "$REPO@sha256:%s\n" $reference; done)
+        done
+
+    - name: Inspect image
+      if: ${{ env.BUILTIN_REGISTRY_ENABLED == 'true' }}
+      shell: bash
+      env:
+        IMAGES: ${{ inputs.images }}
+      run: |
+        set -o xtrace
+        IMAGES_LIST=($IMAGES)
+        for REPO in "${IMAGES_LIST[@]}"; do
+          docker buildx imagetools inspect $REPO:${{ steps.meta.outputs.version }}
+        done

.forgejo/actions/detect-runner-os/action.yml

@@ -1,58 +0,0 @@
-name: detect-runner-os
-description: |
-  Detect the actual OS name and version of the runner.
-  Provides separate outputs for name, version, and a combined slug.
-
-outputs:
-  name:
-    description: 'OS name (e.g. Ubuntu, Debian)'
-    value: ${{ steps.detect.outputs.name }}
-  version:
-    description: 'OS version (e.g. 22.04, 11)'
-    value: ${{ steps.detect.outputs.version }}
-  slug:
-    description: 'Combined OS slug (e.g. Ubuntu-22.04)'
-    value: ${{ steps.detect.outputs.slug }}
-  node_major:
-    description: 'Major version of Node.js if available (e.g. 22)'
-    value: ${{ steps.detect.outputs.node_major }}
-  node_version:
-    description: 'Full Node.js version if available (e.g. 22.19.0)'
-    value: ${{ steps.detect.outputs.node_version }}
-
-runs:
-  using: composite
-  steps:
-    - name: Detect runner OS
-      id: detect
-      shell: bash
-      run: |
-        # Detect OS version (try lsb_release first, fall back to /etc/os-release)
-        OS_VERSION=$(lsb_release -rs 2>/dev/null || grep VERSION_ID /etc/os-release | cut -d'"' -f2)
-
-        # Detect OS name and capitalise (try lsb_release first, fall back to /etc/os-release)
-        OS_NAME=$(lsb_release -is 2>/dev/null || grep "^ID=" /etc/os-release | cut -d'=' -f2 | tr -d '"' | sed 's/\b\(.\)/\u\1/g')
-
-        # Create combined slug
-        OS_SLUG="${OS_NAME}-${OS_VERSION}"
-
-        # Detect Node.js version if available
-        if command -v node >/dev/null 2>&1; then
-          NODE_VERSION=$(node --version | sed 's/v//')
-          NODE_MAJOR=$(echo $NODE_VERSION | cut -d. -f1)
-          echo "node_version=${NODE_VERSION}" >> $GITHUB_OUTPUT
-          echo "node_major=${NODE_MAJOR}" >> $GITHUB_OUTPUT
-          echo "🔍 Detected Node.js: v${NODE_VERSION}"
-        else
-          echo "node_version=" >> $GITHUB_OUTPUT
-          echo "node_major=" >> $GITHUB_OUTPUT
-          echo "🔍 Node.js not found"
-        fi
-
-        # Set OS outputs
-        echo "name=${OS_NAME}" >> $GITHUB_OUTPUT
-        echo "version=${OS_VERSION}" >> $GITHUB_OUTPUT
-        echo "slug=${OS_SLUG}" >> $GITHUB_OUTPUT
-
-        # Log detection results
-        echo "🔍 Detected Runner OS: ${OS_NAME} ${OS_VERSION}"

.forgejo/actions/prepare-docker-build/action.yml

@@ -0,0 +1,169 @@
+name: prepare-docker-build
+description: |
+  Prepare the Docker build environment for Continuwuity builds.
+  Sets up Rust toolchain, Docker Buildx, caching, and extracts metadata for Docker builds.
+
+inputs:
+  platform:
+    description: Target platform (e.g. linux/amd64, linux/arm64)
+    required: true
+  slug:
+    description: Platform slug for artifact naming (e.g. linux-amd64, linux-arm64)
+    required: true
+  target_cpu:
+    description: Target CPU architecture (e.g. haswell, empty for base)
+    required: false
+    default: ""
+  profile:
+    description: Cargo build profile (release or release-max-perf)
+    required: true
+  images:
+    description: Container registry images (newline-separated)
+    required: true
+  registry_user:
+    description: Registry username for authentication
+    required: false
+  registry_password:
+    description: Registry password for authentication
+    required: false
+
+outputs:
+  cpu_suffix:
+    description: CPU suffix for artifact naming
+    value: ${{ steps.cpu-suffix.outputs.suffix }}
+  metadata_labels:
+    description: Docker labels for the image
+    value: ${{ steps.meta.outputs.labels }}
+  metadata_annotations:
+    description: Docker annotations for the image
+    value: ${{ steps.meta.outputs.annotations }}
+
+runs:
+  using: composite
+  steps:
+    - name: Set CPU suffix variable
+      id: cpu-suffix
+      shell: bash
+      run: |
+        if [[ -n "${{ inputs.target_cpu }}" ]]; then
+          echo "suffix=-${{ inputs.target_cpu }}" >> $GITHUB_OUTPUT
+          echo "CPU_SUFFIX=-${{ inputs.target_cpu }}" >> $GITHUB_ENV
+        else
+          echo "suffix=" >> $GITHUB_OUTPUT
+          echo "CPU_SUFFIX=" >> $GITHUB_ENV
+        fi
+
+    - name: Echo matrix configuration
+      shell: bash
+      run: |
+        echo "Platform: ${{ inputs.platform }}"
+        echo "Slug: ${{ inputs.slug }}"
+        echo "Target CPU: ${{ inputs.target_cpu }}"
+        echo "Profile: ${{ inputs.profile }}"
+
+    - name: Install rust
+      if: ${{ env.BUILDKIT_ENDPOINT == '' }}
+      id: rust-toolchain
+      uses: ./.forgejo/actions/rust-toolchain
+
+    - name: Set up Docker Buildx
+      uses: docker/setup-buildx-action@v3
+      with:
+        # Use persistent BuildKit if BUILDKIT_ENDPOINT is set (e.g. tcp://buildkit:8125)
+        driver: ${{ env.BUILDKIT_ENDPOINT != '' && 'remote' || 'docker-container' }}
+        endpoint: ${{ env.BUILDKIT_ENDPOINT || '' }}
+
+    - name: Set up QEMU
+      if: ${{ env.BUILDKIT_ENDPOINT == '' }}
+      uses: docker/setup-qemu-action@v3
+
+    - name: Login to builtin registry
+      if: ${{ env.BUILTIN_REGISTRY_ENABLED == 'true' }}
+      uses: docker/login-action@v3
+      with:
+        registry: ${{ env.BUILTIN_REGISTRY }}
+        username: ${{ inputs.registry_user }}
+        password: ${{ inputs.registry_password }}
+
+    - name: Extract metadata (labels, annotations) for Docker
+      id: meta
+      uses: docker/metadata-action@v5
+      with:
+        images: ${{ inputs.images }}
+        # default labels & annotations: https://github.com/docker/metadata-action/blob/master/src/meta.ts#L509
+      env:
+        DOCKER_METADATA_ANNOTATIONS_LEVELS: manifest,index
+
+    - name: Get short git commit SHA
+      id: sha
+      shell: bash
+      run: |
+        calculatedSha=$(git rev-parse --short ${{ github.sha }})
+        echo "COMMIT_SHORT_SHA=$calculatedSha" >> $GITHUB_ENV
+        echo "Short SHA: $calculatedSha"
+
+    - name: Get Git commit timestamps
+      shell: bash
+      run: |
+        timestamp=$(git log -1 --pretty=%ct)
+        echo "TIMESTAMP=$timestamp" >> $GITHUB_ENV
+        echo "Commit timestamp: $timestamp"
+
+    - uses: ./.forgejo/actions/timelord
+      id: timelord
+
+    - name: Cache Rust registry
+      if: ${{ env.BUILDKIT_ENDPOINT == '' }}
+      uses: actions/cache@v3
+      with:
+        path: |
+          .cargo/git
+          .cargo/git/checkouts
+          .cargo/registry
+          .cargo/registry/src
+        key: continuwuity-rust-registry-image-${{hashFiles('**/Cargo.lock') }}
+
+    - name: Cache cargo target
+      if: ${{ env.BUILDKIT_ENDPOINT == '' }}
+      id: cache-cargo-target
+      uses: actions/cache@v3
+      with:
+        path: |
+          cargo-target${{ env.CPU_SUFFIX }}-${{ inputs.slug }}-${{ inputs.profile }}
+        key: continuwuity-cargo-target${{ env.CPU_SUFFIX }}-${{ inputs.slug }}-${{ inputs.profile }}-${{hashFiles('**/Cargo.lock') }}-${{steps.rust-toolchain.outputs.rustc_version}}
+
+    - name: Cache apt cache
+      if: ${{ env.BUILDKIT_ENDPOINT == '' }}
+      id: cache-apt
+      uses: actions/cache@v3
+      with:
+        path: |
+          var-cache-apt-${{ inputs.slug }}
+        key: continuwuity-var-cache-apt-${{ inputs.slug }}
+
+    - name: Cache apt lib
+      if: ${{ env.BUILDKIT_ENDPOINT == '' }}
+      id: cache-apt-lib
+      uses: actions/cache@v3
+      with:
+        path: |
+          var-lib-apt-${{ inputs.slug }}
+        key: continuwuity-var-lib-apt-${{ inputs.slug }}
+
+    - name: inject cache into docker
+      if: ${{ env.BUILDKIT_ENDPOINT == '' }}
+      uses: https://github.com/reproducible-containers/buildkit-cache-dance@v3.3.0
+      with:
+        cache-map: |
+          {
+            ".cargo/registry": "/usr/local/cargo/registry",
+            ".cargo/git/db": "/usr/local/cargo/git/db",
+            "cargo-target${{ env.CPU_SUFFIX }}-${{ inputs.slug }}-${{ inputs.profile }}": {
+              "target": "/app/target",
+              "id": "cargo-target${{ env.CPU_SUFFIX }}-${{ inputs.slug }}-${{ inputs.profile }}"
+            },
+            "var-cache-apt-${{ inputs.slug }}": "/var/cache/apt",
+            "var-lib-apt-${{ inputs.slug }}": "/var/lib/apt",
+            "${{ steps.timelord.outputs.database-path }}":"/timelord"
+          }
+        skip-extraction: ${{ steps.cache.outputs.cache-hit }}

.forgejo/actions/rust-toolchain/action.yml

@@ -40,7 +40,7 @@ runs:
           !~/.rustup/tmp
           !~/.rustup/downloads
         # Requires repo to be cloned if toolchain is not specified
-        key: ${{ runner.os }}-rustup-${{ inputs.toolchain || hashFiles('**/rust-toolchain.toml') }}
+        key: continuwuity-${{ runner.os }}-rustup-${{ inputs.toolchain || hashFiles('**/rust-toolchain.toml') }}
     - name: Install Rust toolchain
       if: steps.rustup-version.outputs.version == ''
       shell: bash

.forgejo/actions/sccache/action.yml

@@ -9,7 +9,7 @@ runs:
   - name: Install sccache
     uses: https://git.tomfos.tr/tom/sccache-action@v1
   - name: Configure sccache
-    uses: https://github.com/actions/github-script@v7
+    uses: https://github.com/actions/github-script@v8
     with:
       script: |
         core.exportVariable('ACTIONS_RESULTS_URL', process.env.ACTIONS_RESULTS_URL || '');

.forgejo/actions/setup-llvm-with-apt/action.yml

@@ -29,7 +29,7 @@ runs:
   steps:
     - name: Detect runner OS
       id: runner-os
-      uses: ./.forgejo/actions/detect-runner-os
+      uses: https://git.tomfos.tr/actions/detect-versions@v1
 
     - name: Configure cross-compilation architecture
       if: inputs.dpkg-arch != ''
@@ -57,7 +57,7 @@ runs:
 
     - name: Check for LLVM cache
       id: cache
-      uses: https://github.com/actions/cache@v4
+      uses: actions/cache@v4
       with:
         path: |
           /usr/bin/clang-*
@@ -69,7 +69,7 @@ runs:
           /usr/lib/x86_64-linux-gnu/libclang*.so*
           /etc/apt/sources.list.d/archive_uri-*
           /etc/apt/trusted.gpg.d/apt.llvm.org.asc
-        key: llvm-${{ steps.runner-os.outputs.slug }}-v${{ inputs.llvm-version }}-v3-${{ hashFiles('**/Cargo.lock', 'rust-toolchain.toml') }}
+        key: continuwuity-llvm-${{ steps.runner-os.outputs.slug }}-${{ steps.runner-os.outputs.arch }}-v${{ inputs.llvm-version }}-${{ hashFiles('**/Cargo.lock', 'rust-toolchain.toml') }}
 
     - name: End LLVM cache group
       shell: bash

.forgejo/actions/setup-rust/action.yml

@@ -17,9 +17,9 @@ inputs:
     required: false
     default: ''
   rust-version:
-    description: 'Rust version to install (e.g. nightly). Defaults to 1.87.0'
+    description: 'Rust version to install (e.g. nightly).'
     required: false
-    default: '1.87.0'
+    default: ''
   sccache-cache-limit:
     description: 'Maximum size limit for sccache local cache (e.g. 2G, 500M)'
     required: false
@@ -39,7 +39,7 @@ runs:
   steps:
     - name: Detect runner OS
       id: runner-os
-      uses: ./.forgejo/actions/detect-runner-os
+      uses: https://git.tomfos.tr/actions/detect-versions@v1
 
     - name: Configure Cargo environment
       shell: bash
@@ -59,60 +59,18 @@ runs:
         mkdir -p "${{ github.workspace }}/target"
         mkdir -p "${{ github.workspace }}/.rustup"
 
-    - name: Start cache restore group
-      shell: bash
-      run: echo "::group::📦 Restoring caches (registry, toolchain, build artifacts)"
 
-    - name: Cache Cargo registry and git
-      id: registry-cache
-      uses: https://github.com/actions/cache@v4
-      with:
-        path: |
-          .cargo/registry/index
-          .cargo/registry/cache
-          .cargo/git/db
-        # Registry cache saved per workflow, restored from any workflow's cache
-        # Each workflow maintains its own registry that accumulates its needed crates
-        key: cargo-registry-${{ steps.runner-os.outputs.slug }}-${{ github.workflow }}
-        restore-keys: |
-          cargo-registry-${{ steps.runner-os.outputs.slug }}-
 
     - name: Cache toolchain binaries
       id: toolchain-cache
-      uses: https://github.com/actions/cache@v4
+      uses: actions/cache@v4
       with:
         path: |
           .cargo/bin
           .rustup/toolchains
           .rustup/update-hashes
         # Shared toolchain cache across all Rust versions
-        key: toolchain-${{ steps.runner-os.outputs.slug }}
-
-
-    - name: Setup sccache
-      uses: https://git.tomfos.tr/tom/sccache-action@v1
-
-    - name: Cache build artifacts
-      id: build-cache
-      uses: https://github.com/actions/cache@v4
-      with:
-        path: |
-          target/**/deps
-          !target/**/deps/*.rlib
-          target/**/build
-          target/**/.fingerprint
-          target/**/incremental
-          target/**/*.d
-          /timelord/
-        # Build artifacts - cache per code change, restore from deps when code changes
-        key: >-
-          build-${{ steps.runner-os.outputs.slug }}-${{ inputs.rust-version }}${{ inputs.cache-key-suffix && format('-{0}', inputs.cache-key-suffix) || '' }}-${{ hashFiles('rust-toolchain.toml', '**/Cargo.lock') }}-${{ hashFiles('**/*.rs', '**/Cargo.toml') }}
-        restore-keys: |
-          build-${{ steps.runner-os.outputs.slug }}-${{ inputs.rust-version }}${{ inputs.cache-key-suffix && format('-{0}', inputs.cache-key-suffix) || '' }}-${{ hashFiles('rust-toolchain.toml', '**/Cargo.lock') }}-
-
-    - name: End cache restore group
-      shell: bash
-      run: echo "::endgroup::"
+        key: continuwuity-toolchain-${{ steps.runner-os.outputs.slug }}-${{ steps.runner-os.outputs.arch }}
 
     - name: Setup Rust toolchain
       shell: bash
@@ -145,6 +103,71 @@ runs:
         fi
         echo "::endgroup::"
 
+    - name: Output Rust version
+      id: rust-setup
+      shell: bash
+      run: |
+        RUST_VERSION=$(rustc --version | cut -d' ' -f2)
+        echo "version=$RUST_VERSION" >> $GITHUB_OUTPUT
+        echo "📋 Rust version: $(rustc --version)"
+
+    - name: Setup sccache
+      uses: https://git.tomfos.tr/tom/sccache-action@v1
+
+
+    - name: Start cache restore group
+      shell: bash
+      run: echo "::group::📦 Restoring caches (registry, toolchain, build artifacts)"
+
+    - name: Cache Cargo registry and git
+      id: registry-cache
+      uses: actions/cache@v4
+      with:
+        path: |
+          .cargo/registry/index
+          .cargo/registry/cache
+          .cargo/git/db
+        # Registry cache saved per workflow, restored from any workflow's cache
+        # Each workflow maintains its own registry that accumulates its needed crates
+        key: continuwuity-cargo-registry-${{ steps.runner-os.outputs.slug }}-${{ steps.runner-os.outputs.arch }}-${{ github.workflow }}
+        restore-keys: |
+          continuwuity-cargo-registry-${{ steps.runner-os.outputs.slug }}-${{ steps.runner-os.outputs.arch }}-
+
+    - name: Cache dependencies
+      id: deps-cache
+      uses: actions/cache@v4
+      with:
+        path: |
+          target/**/.fingerprint
+          target/**/deps
+          target/**/*.d
+          target/**/.cargo-lock
+          target/**/CACHEDIR.TAG
+          target/**/.rustc_info.json
+          /timelord/
+        # Dependencies cache - based on Cargo.lock, survives source code changes
+        key: >-
+          continuwuity-deps-${{ steps.runner-os.outputs.slug }}-${{ steps.runner-os.outputs.arch }}-${{ steps.rust-setup.outputs.version }}${{ inputs.cache-key-suffix && format('-{0}', inputs.cache-key-suffix) || '' }}-${{ hashFiles('rust-toolchain.toml', '**/Cargo.lock') }}
+        restore-keys: |
+          continuwuity-deps-${{ steps.runner-os.outputs.slug }}-${{ steps.runner-os.outputs.arch }}-${{ steps.rust-setup.outputs.version }}${{ inputs.cache-key-suffix && format('-{0}', inputs.cache-key-suffix) || '' }}-
+
+    - name: Cache incremental compilation
+      id: incremental-cache
+      uses: actions/cache@v4
+      with:
+        path: |
+          target/**/incremental
+        # Incremental cache - based on source code changes
+        key: >-
+          continuwuity-incremental-${{ steps.runner-os.outputs.slug }}-${{ steps.runner-os.outputs.arch }}-${{ steps.rust-setup.outputs.version }}${{ inputs.cache-key-suffix && format('-{0}', inputs.cache-key-suffix) || '' }}-${{ hashFiles('rust-toolchain.toml', '**/Cargo.lock') }}-${{ hashFiles('**/*.rs', '**/Cargo.toml') }}
+        restore-keys: |
+          continuwuity-incremental-${{ steps.runner-os.outputs.slug }}-${{ steps.runner-os.outputs.arch }}-${{ steps.rust-setup.outputs.version }}${{ inputs.cache-key-suffix && format('-{0}', inputs.cache-key-suffix) || '' }}-${{ hashFiles('rust-toolchain.toml', '**/Cargo.lock') }}-
+          continuwuity-incremental-${{ steps.runner-os.outputs.slug }}-${{ steps.runner-os.outputs.arch }}-${{ steps.rust-setup.outputs.version }}${{ inputs.cache-key-suffix && format('-{0}', inputs.cache-key-suffix) || '' }}-
+
+    - name: End cache restore group
+      shell: bash
+      run: echo "::endgroup::"
+
     - name: Configure PATH and install tools
       shell: bash
       env:
@@ -212,13 +235,9 @@ runs:
         echo "📦 Installing target: ${{ inputs.rust-target }}"
         rustup target add ${{ inputs.rust-target }}
 
-    - name: Output version and summary
-      id: rust-setup
+    - name: Output summary
       shell: bash
       run: |
-        RUST_VERSION=$(rustc --version | cut -d' ' -f2)
-        echo "version=$RUST_VERSION" >> $GITHUB_OUTPUT
-
         echo "📋 Setup complete:"
         echo "  Rust: $(rustc --version)"
         echo "  Cargo: $(cargo --version)"

.forgejo/actions/timelord/action.yml

@@ -1,46 +1,120 @@
 name: timelord
 description: |
-  Use timelord to set file timestamps
+  Use timelord to set file timestamps with git-warp-time fallback for cache misses
 inputs:
   key:
     description: |
       The key to use for caching the timelord data.
-      This should be unique to the repository and the runner.
-    required: true
-    default: timelord-v0
+    required: false
+    default: ''
   path:
     description: |
       The path to the directory to be timestamped.
-      This should be the root of the repository.
-    required: true
-    default: .
+    required: false
+    default: ''
+
+outputs:
+  database-path:
+    description: Path to timelord database
+    value: '${{ env.TIMELORD_CACHE_PATH }}'
 
 runs:
   using: composite
   steps:
-    - name: Cache timelord-cli installation
-      id: cache-timelord-bin
-      uses: actions/cache@v3
-      with:
-        path: ~/.cargo/bin/timelord
-        key: timelord-cli-v3.0.1
-    - name: Install timelord-cli
-      uses: https://github.com/cargo-bins/cargo-binstall@main
-      if: steps.cache-timelord-bin.outputs.cache-hit != 'true'
-    - run: cargo binstall timelord-cli@3.0.1
+    - name: Set defaults
       shell: bash
-      if: steps.cache-timelord-bin.outputs.cache-hit != 'true'
+      run: |
+        echo "TIMELORD_KEY=${{ inputs.key || format('timelord-v1-{0}-{1}', github.repository, hashFiles('**/*.rs', '**/Cargo.toml', '**/Cargo.lock')) }}" >> $GITHUB_ENV
+        echo "TIMELORD_PATH=${{ inputs.path || '.' }}" >> $GITHUB_ENV
+        echo "TIMELORD_CACHE_PATH=$HOME/.cache/timelord" >> $GITHUB_ENV
+        echo "PATH=$HOME/.cargo/bin:/usr/share/rust/.cargo/bin:$PATH" >> $GITHUB_ENV
 
-    - name: Load timelord files
-      uses: actions/cache/restore@v3
+    - name: Restore binary cache
+      id: binary-cache
+      uses: actions/cache/restore@v4
       with:
-        path: /timelord/
-        key: ${{ inputs.key }}
-    - name: Run timelord to set timestamps
+        path: |
+          /usr/share/rust/.cargo/bin
+          ~/.cargo/bin
+        key: continuwuity-timelord-binaries
+
+    - name: Check if binaries need installation
       shell: bash
-      run: timelord sync --source-dir ${{ inputs.path }} --cache-dir /timelord/
-    - name: Save timelord
-      uses: actions/cache/save@v3
+      id: check-binaries
+      run: |
+        NEED_INSTALL=false
+
+        # Ensure ~/.cargo/bin exists
+        mkdir -p ~/.cargo/bin
+
+        # Check and move timelord if needed
+        if [ -f /usr/share/rust/.cargo/bin/timelord ] && [ ! -f ~/.cargo/bin/timelord ]; then
+          echo "Moving timelord from /usr/share/rust/.cargo/bin to ~/.cargo/bin"
+          mv /usr/share/rust/.cargo/bin/timelord ~/.cargo/bin/
+        fi
+        if [ ! -f ~/.cargo/bin/timelord ]; then
+          echo "timelord-cli not found, needs installation"
+          NEED_INSTALL=true
+        fi
+
+        # Check and move git-warp-time if needed
+        if [ -f /usr/share/rust/.cargo/bin/git-warp-time ] && [ ! -f ~/.cargo/bin/git-warp-time ]; then
+          echo "Moving git-warp-time from /usr/share/rust/.cargo/bin to ~/.cargo/bin"
+          mv /usr/share/rust/.cargo/bin/git-warp-time ~/.cargo/bin/
+        fi
+        if [ ! -f ~/.cargo/bin/git-warp-time ]; then
+          echo "git-warp-time not found, needs installation"
+          NEED_INSTALL=true
+        fi
+
+        echo "need-install=$NEED_INSTALL" >> $GITHUB_OUTPUT
+
+    - name: Install timelord-cli and git-warp-time
+      if: steps.check-binaries.outputs.need-install == 'true'
+      uses: https://github.com/taiki-e/install-action@v2
       with:
-        path: /timelord/
-        key: ${{ inputs.key }}
+        tool: git-warp-time,timelord-cli@3.0.1
+
+    - name: Save binary cache
+      if: steps.check-binaries.outputs.need-install == 'true'
+      uses: actions/cache/save@v4
+      with:
+        path: |
+          /usr/share/rust/.cargo/bin
+          ~/.cargo/bin
+        key: continuwuity-timelord-binaries
+
+
+    - name: Restore timelord cache with fallbacks
+      id: timelord-restore
+      uses: actions/cache/restore@v4
+      with:
+        path: ${{ env.TIMELORD_CACHE_PATH }}
+        key: ${{ env.TIMELORD_KEY }}
+        restore-keys: |
+          continuwuity-timelord-${{ github.repository }}-
+
+    - name: Initialize timestamps on complete cache miss
+      if: steps.timelord-restore.outputs.cache-hit != 'true'
+      shell: bash
+      run: |
+        echo "Complete timelord cache miss - running git-warp-time"
+        git fetch --unshallow
+        if [ "${{ env.TIMELORD_PATH }}" = "." ]; then
+          git-warp-time --quiet
+        else
+          git-warp-time --quiet ${{ env.TIMELORD_PATH }}
+        fi
+        echo "Git timestamps restored"
+
+    - name: Run timelord sync
+      shell: bash
+      run: |
+        mkdir -p ${{ env.TIMELORD_CACHE_PATH }}
+        timelord sync --source-dir ${{ env.TIMELORD_PATH }} --cache-dir ${{ env.TIMELORD_CACHE_PATH }}
+
+    - name: Save updated timelord cache immediately
+      uses: actions/cache/save@v4
+      with:
+        path: ${{ env.TIMELORD_CACHE_PATH }}
+        key: ${{ env.TIMELORD_KEY }}

.forgejo/actions/upload-docker-artifacts/action.yml

@@ -0,0 +1,70 @@
+name: upload-docker-artifacts
+description: |
+  Upload Docker build artifacts including binary and digest files.
+  Handles artifact naming and conditional digest uploads for registry publishing.
+
+inputs:
+  slug:
+    description: Platform slug for artifact naming (e.g. linux-amd64, linux-arm64)
+    required: true
+  cpu_suffix:
+    description: CPU suffix for artifact naming (e.g. -haswell)
+    required: false
+    default: ""
+  artifact_suffix:
+    description: Suffix for binary artifacts (e.g. -maxperf)
+    required: false
+    default: ""
+  digest_suffix:
+    description: Suffix for digest artifacts (e.g. -maxperf)
+    required: false
+    default: ""
+  digest:
+    description: The digest of the built Docker image
+    required: true
+
+outputs:
+  binary_artifact_name:
+    description: The name of the uploaded binary artifact
+    value: conduwuit${{ inputs.cpu_suffix }}-${{ inputs.slug }}${{ inputs.artifact_suffix }}
+
+runs:
+  using: composite
+  steps:
+    - name: Export digest
+      if: ${{ env.BUILTIN_REGISTRY_ENABLED == 'true' }}
+      shell: bash
+      run: |
+        mkdir -p /tmp/digests
+        digest="${{ inputs.digest }}"
+        echo "🔍 Build step digest output: '$digest'"
+        if [[ -z "$digest" ]]; then
+          echo "❌ ERROR: No digest found from build step"
+          exit 1
+        fi
+        digest_file="/tmp/digests/${digest#sha256:}"
+        echo "📁 Creating digest file: $digest_file"
+        touch "$digest_file"
+        echo "✅ Digest file created successfully"
+        echo "📋 Contents of /tmp/digests:"
+        ls -la /tmp/digests/
+
+    - name: Rename extracted binary
+      shell: bash
+      run: mv /tmp/binaries/sbin/conduwuit /tmp/binaries/conduwuit${{ inputs.cpu_suffix }}-${{ inputs.slug }}${{ inputs.artifact_suffix }}
+
+    - name: Upload binary artifact
+      uses: forgejo/upload-artifact@v4
+      with:
+        name: conduwuit${{ inputs.cpu_suffix }}-${{ inputs.slug }}${{ inputs.artifact_suffix }}
+        path: /tmp/binaries/conduwuit${{ inputs.cpu_suffix }}-${{ inputs.slug }}${{ inputs.artifact_suffix }}
+        if-no-files-found: error
+
+    - name: Upload digest
+      if: ${{ env.BUILTIN_REGISTRY_ENABLED == 'true' }}
+      uses: forgejo/upload-artifact@v4
+      with:
+        name: digests${{ inputs.digest_suffix }}-${{ inputs.slug }}${{ inputs.cpu_suffix }}
+        path: /tmp/digests/*
+        if-no-files-found: error
+        retention-days: 5

.forgejo/regsync/regsync.yml

@@ -40,6 +40,15 @@ creds:
   - registry: registry.gitlab.com
     user: "{{env \"GITLAB_USERNAME\"}}"
     pass: "{{env \"GITLAB_TOKEN\"}}"
+  - registry: git.nexy7574.co.uk
+    user: "{{env \"N7574_GIT_USERNAME\"}}"
+    pass: "{{env \"N7574_GIT_TOKEN\"}}"
+  - registry: ghcr.io
+    user: "{{env \"GH_PACKAGES_USER\"}}"
+    pass: "{{env \"GH_PACKAGES_TOKEN\"}}"
+  - registry: docker.io
+    user: "{{env \"DOCKER_MIRROR_USER\"}}"
+    pass: "{{env \"DOCKER_MIRROR_TOKEN\"}}"
 
 # Global defaults
 defaults:
@@ -53,3 +62,15 @@ sync:
     target: registry.gitlab.com/continuwuity/continuwuity
     type: repository
     <<: *tags-main
+  - source: *source
+    target: git.nexy7574.co.uk/mirrored/continuwuity
+    type: repository
+    <<: *tags-releases
+  - source: *source
+    target: ghcr.io/continuwuity/continuwuity
+    type: repository
+    <<: *tags-main
+  - source: *source
+    target: docker.io/jadedblueeyes/continuwuity
+    type: repository
+    <<: *tags-main

.forgejo/workflows/build-debian.yml

@@ -6,17 +6,11 @@ concurrency:
 
 on:
   push:
-    branches:
-      - '**'
     tags:
-      - 'v*'
-    paths:
-      - 'pkg/debian/**'
-      - 'src/**'
-      - 'Cargo.toml'
-      - 'Cargo.lock'
-      - '.forgejo/workflows/build-debian.yml'
+      - "v*.*.*"
   workflow_dispatch:
+  schedule:
+    - cron: '30 0 * * *'
 
 jobs:
   build:
@@ -38,7 +32,7 @@ jobs:
           echo "Debian distribution: $DISTRIBUTION ($VERSION)"
 
       - name: Checkout repository with full history
-        uses: https://code.forgejo.org/actions/checkout@v4
+        uses: https://code.forgejo.org/actions/checkout@v5
         with:
           fetch-depth: 0
 
@@ -138,7 +132,7 @@ jobs:
           path: ${{ steps.cargo-deb.outputs.path }}
 
       - name: Publish to Forgejo package registry
-        if: ${{ forge.event_name == 'push' || forge.event_name == 'workflow_dispatch' }}
+        if: ${{ forge.event_name == 'push' || forge.event_name == 'workflow_dispatch' || forge.event_name == 'schedule' }}
         run: |
           OWNER="continuwuation"
           DISTRIBUTION=${{ steps.debian-version.outputs.distribution }}

.forgejo/workflows/build-fedora.yml

@@ -0,0 +1,389 @@
+name: Build / Fedora RPM
+
+concurrency:
+  group: "build-fedora-${{ github.ref }}"
+  cancel-in-progress: true
+
+on:
+  push:
+    tags:
+      - "v*.*.*"
+    # paths:
+    #   - 'pkg/fedora/**'
+    #   - 'src/**'
+    #   - 'Cargo.toml'
+    #   - 'Cargo.lock'
+    #   - '.forgejo/workflows/build-fedora.yml'
+  workflow_dispatch:
+  schedule:
+    - cron: '30 0 * * *'
+
+jobs:
+  build:
+    runs-on: fedora-latest
+    steps:
+      - name: Detect Fedora version
+        id: fedora
+        run: |
+          VERSION=$(rpm -E %fedora)
+          echo "version=$VERSION" >> $GITHUB_OUTPUT
+          echo "Fedora version: $VERSION"
+
+      - name: Checkout repository with full history
+        uses: https://code.forgejo.org/actions/checkout@v5
+        with:
+          fetch-depth: 0
+
+
+      - name: Cache DNF packages
+        uses: https://code.forgejo.org/actions/cache@v4
+        with:
+          path: |
+            /var/cache/dnf
+            /var/cache/yum
+          key: dnf-fedora${{ steps.fedora.outputs.version }}-${{ hashFiles('pkg/fedora/continuwuity.spec.rpkg') }}-v1
+          restore-keys: |
+            dnf-fedora${{ steps.fedora.outputs.version }}-
+
+      - name: Cache Cargo registry
+        uses: https://code.forgejo.org/actions/cache@v4
+        with:
+          path: |
+            ~/.cargo/registry
+            ~/.cargo/git
+          key: cargo-fedora${{ steps.fedora.outputs.version }}-${{ hashFiles('**/Cargo.lock') }}
+          restore-keys: |
+            cargo-fedora${{ steps.fedora.outputs.version }}-
+
+      - name: Cache Rust build dependencies
+        uses: https://code.forgejo.org/actions/cache@v4
+        with:
+          path: |
+            ~/rpmbuild/BUILD/*/target/release/deps
+            ~/rpmbuild/BUILD/*/target/release/build
+            ~/rpmbuild/BUILD/*/target/release/.fingerprint
+            ~/rpmbuild/BUILD/*/target/release/incremental
+          key: rust-deps-fedora${{ steps.fedora.outputs.version }}-${{ hashFiles('**/Cargo.lock') }}
+          restore-keys: |
+            rust-deps-fedora${{ steps.fedora.outputs.version }}-
+
+      - name: Setup sccache
+        uses: https://git.tomfos.tr/tom/sccache-action@v1
+
+      - name: Configure sccache environment
+        run: |
+          echo "RUSTC_WRAPPER=sccache" >> $GITHUB_ENV
+          echo "CMAKE_C_COMPILER_LAUNCHER=sccache" >> $GITHUB_ENV
+          echo "CMAKE_CXX_COMPILER_LAUNCHER=sccache" >> $GITHUB_ENV
+          echo "SCCACHE_CACHE_SIZE=10G" >> $GITHUB_ENV
+          # Aggressive GC since cache restores don't increment counter
+          echo "CARGO_INCREMENTAL_GC_TRIGGER=5" >> $GITHUB_ENV
+
+      - name: Install base RPM tools
+        run: |
+          dnf install -y --setopt=keepcache=1 \
+            fedora-packager \
+            python3-pip \
+            rpm-sign \
+            rpkg \
+            wget
+
+      - name: Setup build environment and build SRPM
+        run: |
+          git config --global --add safe.directory "$GITHUB_WORKSPACE"
+          git config --global user.email "ci@continuwuity.org"
+          git config --global user.name "Continuwuity"
+
+          rpmdev-setuptree
+
+          cd "$GITHUB_WORKSPACE"
+
+          # Determine release suffix and version based on ref type and branch
+          if [[ "${{ github.ref }}" == "refs/tags/"* ]]; then
+            # Tags get clean version numbers for stable releases
+            RELEASE_SUFFIX=""
+            TAG_NAME="${{ github.ref_name }}"
+            # Extract version from tag (remove v prefix if present)
+            TAG_VERSION=$(echo "$TAG_NAME" | sed 's/^v//')
+
+            # Create spec file with tag version
+            sed -e "s/^Version:.*$/Version:        $TAG_VERSION/" \
+                -e "s/^Release:.*$/Release:        1%{?dist}/" \
+                pkg/fedora/continuwuity.spec.rpkg > continuwuity.spec.rpkg
+          elif [ "${{ github.ref_name }}" = "main" ]; then
+            # Main branch gets .dev suffix
+            RELEASE_SUFFIX=".dev"
+
+            # Replace the Release line to include our suffix
+            sed "s/^Release:.*$/Release:        1${RELEASE_SUFFIX}%{?dist}/" \
+              pkg/fedora/continuwuity.spec.rpkg > continuwuity.spec.rpkg
+          else
+            # Other branches get sanitized branch name as suffix
+            SAFE_BRANCH=$(echo "${{ github.ref_name }}" | sed 's/[^a-zA-Z0-9]/_/g' | cut -c1-20)
+            RELEASE_SUFFIX=".${SAFE_BRANCH}"
+
+            # Replace the Release line to include our suffix
+            sed "s/^Release:.*$/Release:        1${RELEASE_SUFFIX}%{?dist}/" \
+              pkg/fedora/continuwuity.spec.rpkg > continuwuity.spec.rpkg
+          fi
+
+          rpkg srpm --outdir "$HOME/rpmbuild/SRPMS"
+
+          ls -la $HOME/rpmbuild/SRPMS/
+
+
+      - name: Install build dependencies from SRPM
+        run: |
+          SRPM=$(find "$HOME/rpmbuild/SRPMS" -name "*.src.rpm" | head -1)
+
+          if [ -z "$SRPM" ]; then
+            echo "Error: No SRPM file found"
+            exit 1
+          fi
+
+          echo "Installing build dependencies from: $(basename $SRPM)"
+          dnf builddep -y "$SRPM"
+
+      - name: Build RPM from SRPM
+        run: |
+          SRPM=$(find "$HOME/rpmbuild/SRPMS" -name "*.src.rpm" | head -1)
+
+          if [ -z "$SRPM" ]; then
+            echo "Error: No SRPM file found"
+            exit 1
+          fi
+
+          echo "Building from SRPM: $SRPM"
+
+          rpmbuild --rebuild "$SRPM" \
+            --define "_topdir $HOME/rpmbuild" \
+            --define "_sourcedir $GITHUB_WORKSPACE" \
+            --nocheck  # Skip %check section to avoid test dependencies
+
+
+      - name: Test RPM installation
+        run: |
+          # Find the main binary RPM (exclude debug and source RPMs)
+          RPM=$(find "$HOME/rpmbuild/RPMS" -name "continuwuity-*.rpm" \
+            ! -name "*debuginfo*" \
+            ! -name "*debugsource*" \
+            ! -name "*.src.rpm" | head -1)
+
+          if [ -z "$RPM" ]; then
+            echo "Error: No binary RPM file found"
+            exit 1
+          fi
+
+          echo "Testing installation of: $RPM"
+
+          # Dry run first
+          rpm -qpi "$RPM"
+          echo ""
+          rpm -qpl "$RPM"
+
+          # Actually install it
+          dnf install -y "$RPM"
+
+          # Verify installation
+          rpm -qa | grep continuwuity
+
+          # Check that the binary exists
+          [ -f /usr/bin/conduwuit ] && echo "✅ Binary installed successfully"
+          [ -f /usr/lib/systemd/system/conduwuit.service ] && echo "✅ Systemd service installed"
+          [ -f /etc/conduwuit/conduwuit.toml ] && echo "✅ Config file installed"
+
+      - name: List built packages
+        run: |
+          echo "Binary RPMs:"
+          find "$HOME/rpmbuild/RPMS" -name "*.rpm" -type f -exec ls -la {} \;
+
+          echo ""
+          echo "Source RPMs:"
+          find "$HOME/rpmbuild/SRPMS" -name "*.rpm" -type f -exec ls -la {} \;
+
+      - name: Collect artifacts
+        run: |
+          mkdir -p artifacts
+
+          find "$HOME/rpmbuild/RPMS" -name "*.rpm" -type f -exec cp {} artifacts/ \;
+          find "$HOME/rpmbuild/SRPMS" -name "*.rpm" -type f -exec cp {} artifacts/ \;
+
+          cd artifacts
+          echo "Build Information:" > BUILD_INFO.txt
+          echo "==================" >> BUILD_INFO.txt
+          echo "Git commit: ${{ github.sha }}" >> BUILD_INFO.txt
+          echo "Git branch: ${{ github.ref_name }}" >> BUILD_INFO.txt
+          echo "Build date: $(date -u +%Y-%m-%d_%H:%M:%S_UTC)" >> BUILD_INFO.txt
+          echo "" >> BUILD_INFO.txt
+          echo "Package contents:" >> BUILD_INFO.txt
+          echo "-----------------" >> BUILD_INFO.txt
+          for rpm in *.rpm; do
+            echo "" >> BUILD_INFO.txt
+            echo "File: $rpm" >> BUILD_INFO.txt
+            rpm -qpi "$rpm" 2>/dev/null | grep -E "^(Name|Version|Release|Architecture|Size)" >> BUILD_INFO.txt
+          done
+
+          ls -la
+
+      - name: Upload binary RPM artifact
+        run: |
+          # Find the main binary RPM (exclude debug and source RPMs)
+          BIN_RPM=$(find artifacts -name "continuwuity-*.rpm" \
+            ! -name "*debuginfo*" \
+            ! -name "*debugsource*" \
+            ! -name "*.src.rpm" \
+            -type f)
+
+          mkdir -p upload-bin
+          cp $BIN_RPM upload-bin/
+
+      - name: Upload binary RPM
+        uses: https://code.forgejo.org/actions/upload-artifact@v3
+        with:
+          name: continuwuity
+          path: upload-bin/
+
+      - name: Upload debug RPM artifact
+        uses: https://code.forgejo.org/actions/upload-artifact@v3
+        with:
+          name: continuwuity-debug
+          path: artifacts/*debuginfo*.rpm
+
+      - name: Publish to RPM Package Registry
+        if: ${{ github.event_name == 'push' || github.event_name == 'workflow_dispatch' || github.event_name == 'schedule' }}
+        run: |
+          # Find the main binary RPM (exclude debug and source RPMs)
+          RPM=$(find artifacts -name "continuwuity-*.rpm" \
+            ! -name "*debuginfo*" \
+            ! -name "*debugsource*" \
+            ! -name "*.src.rpm" \
+            -type f | head -1)
+
+          if [ -z "$RPM" ]; then
+            echo "No binary RPM found to publish"
+            exit 0
+          fi
+
+          RPM_BASENAME=$(basename "$RPM")
+          echo "Publishing: $RPM_BASENAME"
+
+          # Determine the group based on ref type and branch
+          if [[ "${{ github.ref }}" == "refs/tags/"* ]]; then
+            GROUP="stable"
+            # For tags, extract the tag name for version info
+            TAG_NAME="${{ github.ref_name }}"
+          elif [ "${{ github.ref_name }}" = "main" ]; then
+            GROUP="dev"
+          else
+            # Use sanitized branch name as group for feature branches
+            GROUP=$(echo "${{ github.ref_name }}" | sed 's/[^a-zA-Z0-9]/-/g' | tr '[:upper:]' '[:lower:]' | cut -c1-30)
+          fi
+
+          PACKAGE_INFO=$(rpm -qpi "$RPM" 2>/dev/null)
+          PACKAGE_NAME=$(echo "$PACKAGE_INFO" | grep "^Name" | awk '{print $3}')
+          PACKAGE_VERSION=$(echo "$PACKAGE_INFO" | grep "^Version" | awk '{print $3}')
+          PACKAGE_RELEASE=$(echo "$PACKAGE_INFO" | grep "^Release" | awk '{print $3}')
+          PACKAGE_ARCH=$(echo "$PACKAGE_INFO" | grep "^Architecture" | awk '{print $2}')
+
+          # Full version includes release
+          FULL_VERSION="${PACKAGE_VERSION}-${PACKAGE_RELEASE}"
+
+          # Forgejo's RPM registry cannot overwrite existing packages, so we must delete first
+          # 404 is OK if package doesn't exist yet
+          echo "Removing any existing package: $PACKAGE_NAME-$FULL_VERSION.$PACKAGE_ARCH"
+          RESPONSE=$(curl -s -w "\n%{http_code}" -X DELETE \
+            -H "Authorization: token ${{ secrets.BUILTIN_REGISTRY_PASSWORD || secrets.GITHUB_TOKEN }}" \
+            "https://forgejo.ellis.link/api/packages/continuwuation/rpm/$GROUP/package/$PACKAGE_NAME/$FULL_VERSION/$PACKAGE_ARCH")
+          HTTP_CODE=$(echo "$RESPONSE" | tail -n1)
+
+          if [ "$HTTP_CODE" != "204" ] && [ "$HTTP_CODE" != "404" ]; then
+            echo "ERROR: Failed to delete package (HTTP $HTTP_CODE)"
+            echo "$RESPONSE" | head -n -1
+            exit 1
+          fi
+
+          curl --fail-with-body \
+            -X PUT \
+            -H "Authorization: token ${{ secrets.BUILTIN_REGISTRY_PASSWORD || secrets.GITHUB_TOKEN }}" \
+            -H "Content-Type: application/x-rpm" \
+            -T "$RPM" \
+            "https://forgejo.ellis.link/api/packages/continuwuation/rpm/$GROUP/upload?sign=true"
+
+          echo ""
+          echo "✅ Published binary RPM to: https://forgejo.ellis.link/continuwuation/-/packages/rpm/continuwuity/"
+          echo "Group: $GROUP"
+
+          # Upload debug RPMs to separate group
+          DEBUG_RPMS=$(find artifacts -name "*debuginfo*.rpm")
+          if [ -n "$DEBUG_RPMS" ]; then
+            echo ""
+            echo "Publishing debug RPMs to group: ${GROUP}-debug"
+
+            for DEBUG_RPM in $DEBUG_RPMS; do
+              echo "Publishing: $(basename "$DEBUG_RPM")"
+
+              DEBUG_INFO=$(rpm -qpi "$DEBUG_RPM" 2>/dev/null)
+              DEBUG_NAME=$(echo "$DEBUG_INFO" | grep "^Name" | awk '{print $3}')
+              DEBUG_VERSION=$(echo "$DEBUG_INFO" | grep "^Version" | awk '{print $3}')
+              DEBUG_RELEASE=$(echo "$DEBUG_INFO" | grep "^Release" | awk '{print $3}')
+              DEBUG_ARCH=$(echo "$DEBUG_INFO" | grep "^Architecture" | awk '{print $2}')
+              DEBUG_FULL_VERSION="${DEBUG_VERSION}-${DEBUG_RELEASE}"
+
+              # Must delete existing package first (Forgejo limitation)
+              RESPONSE=$(curl -s -w "\n%{http_code}" -X DELETE \
+                -H "Authorization: token ${{ secrets.BUILTIN_REGISTRY_PASSWORD || secrets.GITHUB_TOKEN }}" \
+                "https://forgejo.ellis.link/api/packages/continuwuation/rpm/${GROUP}-debug/package/$DEBUG_NAME/$DEBUG_FULL_VERSION/$DEBUG_ARCH")
+              HTTP_CODE=$(echo "$RESPONSE" | tail -n1)
+
+              if [ "$HTTP_CODE" != "204" ] && [ "$HTTP_CODE" != "404" ]; then
+                echo "ERROR: Failed to delete debug package (HTTP $HTTP_CODE)"
+                echo "$RESPONSE" | head -n -1
+                exit 1
+              fi
+
+              curl --fail-with-body \
+                -X PUT \
+                -H "Authorization: token ${{ secrets.BUILTIN_REGISTRY_PASSWORD || secrets.GITHUB_TOKEN }}" \
+                -H "Content-Type: application/x-rpm" \
+                -T "$DEBUG_RPM" \
+                "https://forgejo.ellis.link/api/packages/continuwuation/rpm/${GROUP}-debug/upload?sign=true"
+            done
+
+            echo "✅ Published debug RPMs to group: ${GROUP}-debug"
+          fi
+
+          # Also upload the SRPM to separate group
+          SRPM=$(find artifacts -name "*.src.rpm" | head -1)
+          if [ -n "$SRPM" ]; then
+            echo ""
+            echo "Publishing source RPM: $(basename "$SRPM")"
+            echo "Publishing to group: ${GROUP}-src"
+
+            SRPM_INFO=$(rpm -qpi "$SRPM" 2>/dev/null)
+            SRPM_NAME=$(echo "$SRPM_INFO" | grep "^Name" | awk '{print $3}')
+            SRPM_VERSION=$(echo "$SRPM_INFO" | grep "^Version" | awk '{print $3}')
+            SRPM_RELEASE=$(echo "$SRPM_INFO" | grep "^Release" | awk '{print $3}')
+            SRPM_FULL_VERSION="${SRPM_VERSION}-${SRPM_RELEASE}"
+
+            # Must delete existing SRPM first (Forgejo limitation)
+            echo "Removing any existing SRPM: $SRPM_NAME-$SRPM_FULL_VERSION.src"
+            RESPONSE=$(curl -s -w "\n%{http_code}" -X DELETE \
+              -H "Authorization: token ${{ secrets.BUILTIN_REGISTRY_PASSWORD || secrets.GITHUB_TOKEN }}" \
+              "https://forgejo.ellis.link/api/packages/continuwuation/rpm/${GROUP}-src/package/$SRPM_NAME/$SRPM_FULL_VERSION/src")
+            HTTP_CODE=$(echo "$RESPONSE" | tail -n1)
+
+            if [ "$HTTP_CODE" != "204" ] && [ "$HTTP_CODE" != "404" ]; then
+              echo "ERROR: Failed to delete SRPM (HTTP $HTTP_CODE)"
+              echo "$RESPONSE" | head -n -1
+              exit 1
+            fi
+
+            curl --fail-with-body \
+              -X PUT \
+              -H "Authorization: token ${{ secrets.BUILTIN_REGISTRY_PASSWORD || secrets.GITHUB_TOKEN }}" \
+              -H "Content-Type: application/x-rpm" \
+              -T "$SRPM" \
+              "https://forgejo.ellis.link/api/packages/continuwuation/rpm/${GROUP}-src/upload?sign=true"
+
+            echo "✅ Published source RPM to group: ${GROUP}-src"
+          fi

.forgejo/workflows/documentation.yml

@@ -21,7 +21,7 @@ jobs:
 
     steps:
       - name: Sync repository
-        uses: https://github.com/actions/checkout@v4
+        uses: actions/checkout@v5
         with:
           persist-credentials: false
           fetch-depth: 0
@@ -51,11 +51,11 @@ jobs:
 
       - name: Detect runner environment
         id: runner-env
-        uses: ./.forgejo/actions/detect-runner-os
+        uses: https://git.tomfos.tr/actions/detect-versions@v1
 
       - name: Setup Node.js
         if: steps.runner-env.outputs.node_major == '' || steps.runner-env.outputs.node_major < '20'
-        uses: https://github.com/actions/setup-node@v4
+        uses: https://github.com/actions/setup-node@v6
         with:
           node-version: 22
 
@@ -63,9 +63,7 @@ jobs:
         uses: actions/cache@v3
         with:
           path: ~/.npm
-          key: ${{ steps.runner-env.outputs.slug }}-node-${{ hashFiles('**/package-lock.json') }}
-          restore-keys: |
-            ${{ steps.runner-env.outputs.slug }}-node-
+          key: continuwuity-${{ steps.runner-env.outputs.slug }}-${{ steps.runner-env.outputs.arch }}-node-${{ steps.runner-env.outputs.node_version }}
 
       - name: Install dependencies
         run: npm install --save-dev wrangler@latest

.forgejo/workflows/element.yml

@@ -4,6 +4,14 @@ on:
   schedule:
     - cron: "0 0 * * *"
   workflow_dispatch:
+  pull_request:
+    paths:
+      - ".forgejo/workflows/element.yml"
+  push:
+    branches:
+      - main
+    paths:
+      - ".forgejo/workflows/element.yml"
 
 concurrency:
   group: "element-${{ github.ref }}"
@@ -16,7 +24,7 @@ jobs:
 
     steps:
       - name: 📦 Setup Node.js
-        uses: https://github.com/actions/setup-node@v4
+        uses: https://github.com/actions/setup-node@v6
         with:
           node-version: "22"
 
@@ -101,7 +109,7 @@ jobs:
           cat ./element-web/webapp/config.json
 
       - name: 📤 Upload Artifact
-        uses: https://code.forgejo.org/actions/upload-artifact@v3
+        uses: forgejo/upload-artifact@v4
         with:
           name: element-web
           path: ./element-web/webapp/

.forgejo/workflows/mirror-images.yml

@@ -11,7 +11,13 @@ on:
         required: false
         default: false
         type: boolean
-
+  push:
+    branches:
+      - main
+    paths:
+      # Re-run when config changes
+      - '.forgejo/regsync/regsync.yml'
+      - '.forgejo/workflows/mirror-images.yml'
 concurrency:
   group: "mirror-images"
   cancel-in-progress: true
@@ -24,12 +30,27 @@ jobs:
       BUILTIN_REGISTRY_PASSWORD: ${{ secrets.BUILTIN_REGISTRY_PASSWORD }}
       GITLAB_USERNAME: ${{ vars.GITLAB_USERNAME }}
       GITLAB_TOKEN: ${{ secrets.GITLAB_TOKEN }}
+      N7574_GIT_USERNAME: ${{ vars.N7574_GIT_USERNAME }}
+      N7574_GIT_TOKEN: ${{ secrets.N7574_GIT_TOKEN }}
+      GH_PACKAGES_USER: ${{ vars.GH_PACKAGES_USER }}
+      GH_PACKAGES_TOKEN: ${{ secrets.GH_PACKAGES_TOKEN }}
+      DOCKER_MIRROR_USER: ${{ vars.DOCKER_MIRROR_USER }}
+      DOCKER_MIRROR_TOKEN: ${{ secrets.DOCKER_MIRROR_TOKEN }}
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v4
+        uses: actions/checkout@v5
         with:
           persist-credentials: false
 
+      # - uses: https://github.com/actions/create-github-app-token@v2
+      #   id: app-token
+      #   with:
+      #     app-id: ${{ vars.GH_APP_ID }}
+      #     private-key: ${{ secrets.GH_APP_PRIVATE_KEY }}
+      #     github-api-url: https://api.github.com
+      #     owner: continuwuity
+      #     repositories: continuwuity
+
       - name: Install regctl
         uses: https://forgejo.ellis.link/continuwuation/regclient-actions/regctl-installer@main
         with:

.forgejo/workflows/prek-checks.yml

@@ -16,7 +16,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v4
+        uses: actions/checkout@v5
         with:
           persist-credentials: false
 
@@ -47,7 +47,7 @@ jobs:
 
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v4
+        uses: actions/checkout@v5
         with:
           persist-credentials: false
 

.forgejo/workflows/release-image.yml

@@ -23,55 +23,20 @@ on:
       - "renovate.json"
       - "pkg/**"
       - "docs/**"
+    tags:
+      - "v*.*.*"
   # Allows you to run this workflow manually from the Actions tab
   workflow_dispatch:
 
 env:
   BUILTIN_REGISTRY: forgejo.ellis.link
   BUILTIN_REGISTRY_ENABLED: "${{ ((vars.BUILTIN_REGISTRY_USER && secrets.BUILTIN_REGISTRY_PASSWORD) || (github.event_name != 'pull_request' || github.event.pull_request.head.repo.fork == false)) && 'true' || 'false' }}"
+  IMAGE_PATH: forgejo.ellis.link/continuwuation/continuwuity
 
 jobs:
-  define-variables:
-    runs-on: ubuntu-latest
-
-    outputs:
-      images: ${{ steps.var.outputs.images }}
-      images_list: ${{ steps.var.outputs.images_list }}
-      build_matrix: ${{ steps.var.outputs.build_matrix }}
-
-    steps:
-      - name: Setting variables
-        uses: https://github.com/actions/github-script@v7
-        id: var
-        with:
-          script: |
-            const githubRepo = '${{ github.repository }}'.toLowerCase()
-            const repoId = githubRepo.split('/')[1]
-
-            core.setOutput('github_repository', githubRepo)
-            const builtinImage = '${{ env.BUILTIN_REGISTRY }}/' + githubRepo
-            let images = []
-            if (process.env.BUILTIN_REGISTRY_ENABLED === "true") {
-              images.push(builtinImage)
-            } else {
-              // Fallback to official registry for forks/PRs without credentials
-              images.push('forgejo.ellis.link/continuwuation/continuwuity')
-            }
-            core.setOutput('images', images.join("\n"))
-            core.setOutput('images_list', images.join(","))
-            const platforms = ['linux/amd64', 'linux/arm64']
-            core.setOutput('build_matrix', JSON.stringify({
-              platform: platforms,
-              target_cpu: ['base'],
-              include: platforms.map(platform => { return {
-                platform,
-                slug: platform.replace('/', '-')
-              }})
-            }))
-
-  build-image:
+  build-release:
+    name: "Build ${{ matrix.slug }} (release)"
     runs-on: dind
-    needs: define-variables
     permissions:
       contents: read
       packages: write
@@ -79,133 +44,28 @@ jobs:
       id-token: write
     strategy:
       matrix:
-        {
-          "target_cpu": ["base"],
-          "profile": ["release"],
-          "include":
-            [
-              { "platform": "linux/amd64", "slug": "linux-amd64" },
-              { "platform": "linux/arm64", "slug": "linux-arm64" },
-            ],
-          "platform": ["linux/amd64", "linux/arm64"],
-        }
+        include:
+          - platform: "linux/amd64"
+            slug: "linux-amd64"
+          - platform: "linux/arm64"
+            slug: "linux-arm64"
 
     steps:
-      - name: Echo strategy
-        run: echo '${{ toJSON(fromJSON(needs.define-variables.outputs.build_matrix)) }}'
-      - name: Echo matrix
-        run: echo '${{ toJSON(matrix) }}'
-
       - name: Checkout repository
-        uses: actions/checkout@v4
+        uses: actions/checkout@v5
         with:
           persist-credentials: false
-      - name: Install rust
-        if: ${{ env.BUILDKIT_ENDPOINT == '' }}
-        id: rust-toolchain
-        uses: ./.forgejo/actions/rust-toolchain
-
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v3
+      - name: Prepare Docker build environment
+        id: prepare
+        uses: ./.forgejo/actions/prepare-docker-build
         with:
-          # Use persistent BuildKit if BUILDKIT_ENDPOINT is set (e.g. tcp://buildkit:8125)
-          driver: ${{ env.BUILDKIT_ENDPOINT != '' && 'remote' || 'docker-container' }}
-          endpoint: ${{ env.BUILDKIT_ENDPOINT || '' }}
-      - name: Set up QEMU
-        if: ${{ env.BUILDKIT_ENDPOINT == '' }}
-        uses: docker/setup-qemu-action@v3
-      # Uses the `docker/login-action` action to log in to the Container registry registry using the account and password that will publish the packages. Once published, the packages are scoped to the account defined here.
-      - name: Login to builtin registry
-        if: ${{ env.BUILTIN_REGISTRY_ENABLED == 'true' }}
-        uses: docker/login-action@v3
-        with:
-          registry: ${{ env.BUILTIN_REGISTRY }}
-          username: ${{ vars.BUILTIN_REGISTRY_USER || github.actor }}
-          password: ${{ secrets.BUILTIN_REGISTRY_PASSWORD || secrets.GITHUB_TOKEN }}
-
-      # This step uses [docker/metadata-action](https://github.com/docker/metadata-action#about) to extract tags and labels that will be applied to the specified image. The `id` "meta" allows the output of this step to be referenced in a subsequent step. The `images` value provides the base name for the tags and labels.
-      - name: Extract metadata (labels, annotations) for Docker
-        id: meta
-        uses: docker/metadata-action@v5
-        with:
-          images: ${{needs.define-variables.outputs.images}}
-          # default labels & annotations: https://github.com/docker/metadata-action/blob/master/src/meta.ts#L509
-        env:
-          DOCKER_METADATA_ANNOTATIONS_LEVELS: manifest,index
-
-      # This step uses the `docker/build-push-action` action to build the image, based on your repository's `Dockerfile`. If the build succeeds, it pushes the image to GitHub Packages.
-      # It uses the `context` parameter to define the build's context as the set of files located in the specified path. For more information, see "[Usage](https://github.com/docker/build-push-action#usage)" in the README of the `docker/build-push-action` repository.
-      # It uses the `tags` and `labels` parameters to tag and label the image with the output from the "meta" step.
-      # It will not push images generated from a pull request
-      - name: Get short git commit SHA
-        id: sha
-        run: |
-          calculatedSha=$(git rev-parse --short ${{ github.sha }})
-          echo "COMMIT_SHORT_SHA=$calculatedSha" >> $GITHUB_ENV
-          echo "Short SHA: $calculatedSha"
-      - name: Get Git commit timestamps
-        run: |
-          timestamp=$(git log -1 --pretty=%ct)
-          echo "TIMESTAMP=$timestamp" >> $GITHUB_ENV
-          echo "Commit timestamp: $timestamp"
-
-      - uses: ./.forgejo/actions/timelord
-        if: ${{ env.BUILDKIT_ENDPOINT == '' }}
-        with:
-          key: timelord-v0
-          path: .
-
-      - name: Cache Rust registry
-        if: ${{ env.BUILDKIT_ENDPOINT == '' }}
-        uses: actions/cache@v3
-        with:
-          path: |
-            .cargo/git
-            .cargo/git/checkouts
-            .cargo/registry
-            .cargo/registry/src
-          key: rust-registry-image-${{hashFiles('**/Cargo.lock') }}
-      - name: Cache cargo target
-        if: ${{ env.BUILDKIT_ENDPOINT == '' }}
-        id: cache-cargo-target
-        uses: actions/cache@v3
-        with:
-          path: |
-            cargo-target-${{ matrix.target_cpu }}-${{ matrix.slug }}-${{ matrix.profile }}
-          key: cargo-target-${{ matrix.target_cpu }}-${{ matrix.slug }}-${{ matrix.profile }}-${{hashFiles('**/Cargo.lock') }}-${{steps.rust-toolchain.outputs.rustc_version}}
-      - name: Cache apt cache
-        if: ${{ env.BUILDKIT_ENDPOINT == '' }}
-        id: cache-apt
-        uses: actions/cache@v3
-        with:
-          path: |
-            var-cache-apt-${{ matrix.slug }}
-          key: var-cache-apt-${{ matrix.slug }}
-      - name: Cache apt lib
-        if: ${{ env.BUILDKIT_ENDPOINT == '' }}
-        id: cache-apt-lib
-        uses: actions/cache@v3
-        with:
-          path: |
-            var-lib-apt-${{ matrix.slug }}
-          key: var-lib-apt-${{ matrix.slug }}
-      - name: inject cache into docker
-        if: ${{ env.BUILDKIT_ENDPOINT == '' }}
-        uses: https://github.com/reproducible-containers/buildkit-cache-dance@v3.3.0
-        with:
-          cache-map: |
-            {
-              ".cargo/registry": "/usr/local/cargo/registry",
-              ".cargo/git/db": "/usr/local/cargo/git/db",
-              "cargo-target-${{ matrix.target_cpu }}-${{ matrix.slug }}-${{ matrix.profile }}": {
-                "target": "/app/target",
-                "id": "cargo-target-${{ matrix.target_cpu }}-${{ matrix.slug }}-${{ matrix.profile }}"
-              },
-              "var-cache-apt-${{ matrix.slug }}": "/var/cache/apt",
-              "var-lib-apt-${{ matrix.slug }}": "/var/lib/apt"
-            }
-          skip-extraction: ${{ steps.cache.outputs.cache-hit }}
-
+          platform: ${{ matrix.platform }}
+          slug: ${{ matrix.slug }}
+          target_cpu: ""
+          profile: "release"
+          images: ${{ env.IMAGE_PATH }}
+          registry_user: ${{ vars.BUILTIN_REGISTRY_USER || github.actor }}
+          registry_password: ${{ secrets.BUILTIN_REGISTRY_PASSWORD || secrets.GITHUB_TOKEN }}
       - name: Build and push Docker image by digest
         id: build
         uses: docker/build-push-action@v6
@@ -217,117 +77,130 @@ jobs:
             GIT_COMMIT_HASH_SHORT=${{ env.COMMIT_SHORT_SHA }}
             GIT_REMOTE_URL=${{github.event.repository.html_url }}
             GIT_REMOTE_COMMIT_URL=${{github.event.head_commit.url }}
+            CARGO_INCREMENTAL=${{ env.BUILDKIT_ENDPOINT != '' && '1' || '0' }}
+            TARGET_CPU=
+            RUST_PROFILE=release
           platforms: ${{ matrix.platform }}
-          labels: ${{ steps.meta.outputs.labels }}
-          annotations: ${{ steps.meta.outputs.annotations }}
+          labels: ${{ steps.prepare.outputs.metadata_labels }}
+          annotations: ${{ steps.prepare.outputs.metadata_annotations }}
           cache-from: type=gha
           # cache-to: type=gha,mode=max
           sbom: true
           outputs: |
-            ${{ env.BUILTIN_REGISTRY_ENABLED == 'true' && format('type=image,"name={0}",push-by-digest=true,name-canonical=true,push=true', needs.define-variables.outputs.images_list) || format('type=image,"name={0}",push=false', needs.define-variables.outputs.images_list) }}
+            ${{ env.BUILTIN_REGISTRY_ENABLED == 'true' && format('type=image,"name={0}",push-by-digest=true,name-canonical=true,push=true', env.IMAGE_PATH) || format('type=image,"name={0}",push=false', env.IMAGE_PATH) }}
             type=local,dest=/tmp/binaries
         env:
           SOURCE_DATE_EPOCH: ${{ env.TIMESTAMP }}
-
-      # For publishing multi-platform manifests
-      - name: Export digest
-        if: ${{ env.BUILTIN_REGISTRY_ENABLED == 'true' }}
-        run: |
-          mkdir -p /tmp/digests
-          digest="${{ steps.build.outputs.digest }}"
-          touch "/tmp/digests/${digest#sha256:}"
-
-      # Binary extracted via local output for all builds
-      - name: Rename extracted binary
-        run: mv /tmp/binaries/sbin/conduwuit /tmp/binaries/conduwuit-${{ matrix.target_cpu }}-${{ matrix.slug }}-${{ matrix.profile }}
-
-      - name: Upload binary artifact
-        uses: forgejo/upload-artifact@v4
+      - name: Upload Docker artifacts
+        uses: ./.forgejo/actions/upload-docker-artifacts
         with:
-          name: conduwuit-${{ matrix.target_cpu }}-${{ matrix.slug }}-${{ matrix.profile }}
-          path: /tmp/binaries/conduwuit-${{ matrix.target_cpu }}-${{ matrix.slug }}-${{ matrix.profile }}
-          if-no-files-found: error
+          slug: ${{ matrix.slug }}
+          cpu_suffix: ${{ steps.prepare.outputs.cpu_suffix }}
+          artifact_suffix: ""
+          digest_suffix: ""
+          digest: ${{ steps.build.outputs.digest }}
 
-      - name: Upload digest
-        if: ${{ env.BUILTIN_REGISTRY_ENABLED == 'true' }}
-        uses: forgejo/upload-artifact@v4
-        with:
-          name: digests-${{ matrix.slug }}
-          path: /tmp/digests/*
-          if-no-files-found: error
-          retention-days: 5
-
-  merge:
+  merge-release:
+    name: "Create Multi-arch Release Manifest"
     runs-on: dind
-    needs: [define-variables, build-image]
+    needs: build-release
     steps:
-      - name: Download digests
-        if: ${{ env.BUILTIN_REGISTRY_ENABLED == 'true' }}
-        uses: forgejo/download-artifact@v4
+      - name: Checkout repository
+        uses: actions/checkout@v5
         with:
-          path: /tmp/digests
-          pattern: digests-*
-          merge-multiple: true
-      # Uses the `docker/login-action` action to log in to the Container registry registry using the account and password that will publish the packages. Once published, the packages are scoped to the account defined here.
-      - name: Login to builtin registry
-        if: ${{ env.BUILTIN_REGISTRY_ENABLED == 'true' }}
-        uses: docker/login-action@v3
+          persist-credentials: false
+      - name: Create multi-platform manifest
+        uses: ./.forgejo/actions/create-docker-manifest
         with:
-          registry: ${{ env.BUILTIN_REGISTRY }}
-          username: ${{ vars.BUILTIN_REGISTRY_USER || github.actor }}
-          password: ${{ secrets.BUILTIN_REGISTRY_PASSWORD || secrets.GITHUB_TOKEN }}
+          digest_pattern: "digests-linux-{amd64,arm64}"
+          tag_suffix: ""
+          images: ${{ env.IMAGE_PATH }}
+          registry_user: ${{ vars.BUILTIN_REGISTRY_USER || github.actor }}
+          registry_password: ${{ secrets.BUILTIN_REGISTRY_PASSWORD || secrets.GITHUB_TOKEN }}
 
-      - name: Set up Docker Buildx
-        if: ${{ env.BUILTIN_REGISTRY_ENABLED == 'true' }}
-        uses: docker/setup-buildx-action@v3
-        with:
-          # Use persistent BuildKit if BUILDKIT_ENDPOINT is set (e.g. tcp://buildkit:8125)
-          driver: ${{ env.BUILDKIT_ENDPOINT != '' && 'remote' || 'docker-container' }}
-          endpoint: ${{ env.BUILDKIT_ENDPOINT || '' }}
+  build-maxperf:
+    name: "Build ${{ matrix.slug }} (max-perf)"
+    runs-on: dind
+    needs: build-release
+    permissions:
+      contents: read
+      packages: write
+      attestations: write
+      id-token: write
+    strategy:
+      matrix:
+        include:
+          - platform: "linux/amd64"
+            slug: "linux-amd64"
+            target_cpu: "haswell"
+          - platform: "linux/arm64"
+            slug: "linux-arm64"
+            target_cpu: ""
 
-      - name: Extract metadata (tags) for Docker
-        if: ${{ env.BUILTIN_REGISTRY_ENABLED == 'true' }}
-        id: meta
-        uses: docker/metadata-action@v5
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v5
         with:
-          tags: |
-            type=semver,pattern={{version}},prefix=v
-            type=semver,pattern={{major}}.{{minor}},enable=${{ !startsWith(github.ref, 'refs/tags/v0.0.') }},prefix=v
-            type=semver,pattern={{major}},enable=${{ !startsWith(github.ref, 'refs/tags/v0.') }},prefix=v
-            type=ref,event=branch,prefix=${{ format('refs/heads/{0}', github.event.repository.default_branch) != github.ref && 'branch-' || '' }}
-            type=ref,event=pr
-            type=sha,format=long
-            type=raw,value=latest,enable=${{ startsWith(github.ref, 'refs/tags/v') }}
-          images: ${{needs.define-variables.outputs.images}}
-          # default labels & annotations: https://github.com/docker/metadata-action/blob/master/src/meta.ts#L509
+          persist-credentials: false
+      - name: Prepare max-perf Docker build environment
+        id: prepare
+        uses: ./.forgejo/actions/prepare-docker-build
+        with:
+          platform: ${{ matrix.platform }}
+          slug: ${{ matrix.slug }}
+          target_cpu: ${{ matrix.target_cpu }}
+          profile: "release-max-perf"
+          images: ${{ env.IMAGE_PATH }}
+          registry_user: ${{ vars.BUILTIN_REGISTRY_USER || github.actor }}
+          registry_password: ${{ secrets.BUILTIN_REGISTRY_PASSWORD || secrets.GITHUB_TOKEN }}
+      - name: Build and push max-perf Docker image by digest
+        id: build
+        uses: docker/build-push-action@v6
+        with:
+          context: .
+          file: "docker/Dockerfile"
+          build-args: |
+            GIT_COMMIT_HASH=${{ github.sha }}
+            GIT_COMMIT_HASH_SHORT=${{ env.COMMIT_SHORT_SHA }}
+            GIT_REMOTE_URL=${{github.event.repository.html_url }}
+            GIT_REMOTE_COMMIT_URL=${{github.event.head_commit.url }}
+            CARGO_INCREMENTAL=${{ env.BUILDKIT_ENDPOINT != '' && '1' || '0' }}
+            TARGET_CPU=${{ matrix.target_cpu }}
+            RUST_PROFILE=release-max-perf
+          platforms: ${{ matrix.platform }}
+          labels: ${{ steps.prepare.outputs.metadata_labels }}
+          annotations: ${{ steps.prepare.outputs.metadata_annotations }}
+          cache-from: type=gha
+          # cache-to: type=gha,mode=max
+          sbom: true
+          outputs: |
+            ${{ env.BUILTIN_REGISTRY_ENABLED == 'true' && format('type=image,"name={0}",push-by-digest=true,name-canonical=true,push=true', env.IMAGE_PATH) || format('type=image,"name={0}",push=false', env.IMAGE_PATH) }}
+            type=local,dest=/tmp/binaries
         env:
-          DOCKER_METADATA_ANNOTATIONS_LEVELS: index
+          SOURCE_DATE_EPOCH: ${{ env.TIMESTAMP }}
+      - name: Upload max-perf Docker artifacts
+        uses: ./.forgejo/actions/upload-docker-artifacts
+        with:
+          slug: ${{ matrix.slug }}
+          cpu_suffix: ${{ steps.prepare.outputs.cpu_suffix }}
+          artifact_suffix: "-maxperf"
+          digest_suffix: "-maxperf"
+          digest: ${{ steps.build.outputs.digest }}
 
-      - name: Create manifest list and push
-        if: ${{ env.BUILTIN_REGISTRY_ENABLED == 'true' }}
-        working-directory: /tmp/digests
-        env:
-          IMAGES: ${{needs.define-variables.outputs.images}}
-        shell: bash
-        run: |
-          IFS=$'\n'
-          IMAGES_LIST=($IMAGES)
-          ANNOTATIONS_LIST=($DOCKER_METADATA_OUTPUT_ANNOTATIONS)
-          TAGS_LIST=($DOCKER_METADATA_OUTPUT_TAGS)
-          for REPO in "${IMAGES_LIST[@]}"; do
-              docker buildx imagetools create \
-                $(for tag in "${TAGS_LIST[@]}"; do echo "--tag"; echo "$tag"; done) \
-                $(for annotation in "${ANNOTATIONS_LIST[@]}"; do echo "--annotation"; echo "$annotation"; done) \
-                $(for reference in *; do printf "$REPO@sha256:%s\n" $reference; done)
-          done
-
-      - name: Inspect image
-        if: ${{ env.BUILTIN_REGISTRY_ENABLED == 'true' }}
-        env:
-          IMAGES: ${{needs.define-variables.outputs.images}}
-        shell: bash
-        run: |
-          IMAGES_LIST=($IMAGES)
-          for REPO in "${IMAGES_LIST[@]}"; do
-            docker buildx imagetools inspect $REPO:${{ steps.meta.outputs.version }}
-          done
+  merge-maxperf:
+    name: "Create Max-Perf Manifest"
+    runs-on: dind
+    needs: build-maxperf
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v5
+        with:
+          persist-credentials: false
+      - name: Create max-perf manifest
+        uses: ./.forgejo/actions/create-docker-manifest
+        with:
+          digest_pattern: "digests-maxperf-linux-{amd64-haswell,arm64}"
+          tag_suffix: "-maxperf"
+          images: ${{ env.IMAGE_PATH }}
+          registry_user: ${{ vars.BUILTIN_REGISTRY_USER || github.actor }}
+          registry_password: ${{ secrets.BUILTIN_REGISTRY_PASSWORD || secrets.GITHUB_TOKEN }}

.forgejo/workflows/renovate.yml

@@ -1,5 +1,7 @@
 name: Maintenance / Renovate
 
+enable-email-notifications: true
+
 on:
   schedule:
     # Run at 5am UTC daily to avoid late-night dev
@@ -10,10 +12,10 @@ on:
       dryRun:
         description: 'Dry run mode'
         required: false
-        default: null
+        default: ''
         type: choice
         options:
-          - null
+          - ''
           - 'extract'
           - 'lookup'
           - 'full'
@@ -23,6 +25,7 @@ on:
         default: 'info'
         type: choice
         options:
+          - 'debug'
           - 'info'
           - 'warning'
           - 'critical'
@@ -40,11 +43,11 @@ jobs:
     name: Renovate
     runs-on: ubuntu-latest
     container:
-      image: ghcr.io/renovatebot/renovate:41
+      image: ghcr.io/renovatebot/renovate:41.146.4@sha256:bb70194b7405faf10a6f279b60caa10403a440ba37d158c5a4ef0ae7b67a0f92
       options: --tmpfs /tmp:exec
     steps:
       - name: Checkout
-        uses: actions/checkout@v4
+        uses: actions/checkout@v5
         with:
           show-progress: false
 
@@ -52,25 +55,34 @@ jobs:
         run: /usr/local/renovate/node -e 'console.log(`node heap limit = ${require("v8").getHeapStatistics().heap_size_limit / (1024 * 1024)} Mb`)'
 
       - name: Restore renovate repo cache
-        uses: https://github.com/actions/cache@v4
+        uses: actions/cache/restore@v4
         with:
           path: |
             /tmp/renovate/cache/renovate/repository
-          key: repo-cache-${{ github.run_id }}
+          key: renovate-repo-cache-${{ github.run_id }}
           restore-keys: |
-            repo-cache-
+            renovate-repo-cache-
 
       - name: Restore renovate package cache
-        uses: https://github.com/actions/cache@v4
+        uses: actions/cache/restore@v4
         with:
           path: |
             /tmp/renovate/cache/renovate/renovate-cache-sqlite
-          key: package-cache-${{ github.run_id }}
+          key: renovate-package-cache-${{ github.run_id }}
           restore-keys: |
-            package-cache-
+            renovate-package-cache-
+
+      - name: Restore renovate OSV cache
+        uses: actions/cache/restore@v4
+        with:
+          path: |
+            /tmp/osv
+          key: renovate-osv-cache-${{ github.run_id }}
+          restore-keys: |
+            renovate-osv-cache-
 
       - name: Self-hosted Renovate
-        uses: https://github.com/renovatebot/github-action@v43.0.11
+        run: renovate
         env:
           LOG_LEVEL: ${{ inputs.logLevel || 'info' }}
           RENOVATE_DRY_RUN: ${{ inputs.dryRun || 'false' }}
@@ -84,28 +96,37 @@ jobs:
 
           RENOVATE_REQUIRE_CONFIG: 'required'
           RENOVATE_ONBOARDING: 'false'
-
-          RENOVATE_PR_COMMITS_PER_RUN_LIMIT: 3
+          RENOVATE_INHERIT_CONFIG: 'true'
 
           RENOVATE_GITHUB_TOKEN_WARN: 'false'
           RENOVATE_TOKEN: ${{ secrets.RENOVATE_TOKEN }}
-          GITHUB_COM_TOKEN: ${{ secrets.GH_PUBLIC_RO }}
+          GITHUB_COM_TOKEN: ${{ secrets.GH_PUBLIC_RO || secrets.GH_TOKEN }}
 
           RENOVATE_REPOSITORY_CACHE: 'enabled'
-          RENOVATE_X_SQLITE_PACKAGE_CACHE: true
+          RENOVATE_X_SQLITE_PACKAGE_CACHE: 'true'
+          OSV_OFFLINE_ROOT_DIR: /tmp/osv
 
       - name: Save renovate repo cache
-        if: always() && env.RENOVATE_DRY_RUN != 'full'
-        uses: https://github.com/actions/cache@v4
+        if: always()
+        uses:
+          actions/cache/save@v4
         with:
           path: |
             /tmp/renovate/cache/renovate/repository
-          key: repo-cache-${{ github.run_id }}
+          key: renovate-repo-cache-${{ github.run_id }}
 
       - name: Save renovate package cache
-        if: always() && env.RENOVATE_DRY_RUN != 'full'
-        uses: https://github.com/actions/cache@v4
+        if: always()
+        uses: actions/cache/save@v4
         with:
           path: |
             /tmp/renovate/cache/renovate/renovate-cache-sqlite
-          key: package-cache-${{ github.run_id }}
+          key: renovate-package-cache-${{ github.run_id }}
+
+      - name: Save renovate OSV cache
+        if: always()
+        uses: actions/cache/save@v4
+        with:
+          path: |
+            /tmp/osv
+          key: renovate-osv-cache-${{ github.run_id }}

.forgejo/workflows/update-flake-hashes.yml

@@ -0,0 +1,121 @@
+name: Update flake hashes
+
+on:
+  workflow_dispatch:
+  pull_request:
+    paths:
+      - "Cargo.lock"
+      - "Cargo.toml"
+      - "rust-toolchain.toml"
+      - "nix/**/*"
+      - ".forgejo/workflows/update-flake-hashes.yml"
+
+jobs:
+  update-flake-hashes:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: https://code.forgejo.org/actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5
+        with:
+          fetch-depth: 0
+          fetch-tags: false
+          fetch-single-branch: true
+          submodules: false
+          persist-credentials: true
+          token: ${{ secrets.FORGEJO_TOKEN }}
+
+      - uses: https://github.com/cachix/install-nix-action@7ab6e7fd29da88e74b1e314a4ae9ac6b5cda3801 # v31.8.0
+        with:
+          nix_path: nixpkgs=channel:nixos-unstable
+
+        # We can skip getting a toolchain hash if this was ran as a dispatch with the intent
+        # to update just the rocksdb hash. If this was ran as a dispatch and the toolchain
+        # files are changed, we still update them, as well as the rocksdb import.
+      - name: Detect changed files
+        id: changes
+        run: |
+          git fetch origin ${{ github.base_ref }} --depth=1 || true
+          if [ -n "${{ github.event.pull_request.base.sha }}" ]; then
+            base=${{ github.event.pull_request.base.sha }}
+          else
+            base=$(git rev-parse HEAD~1)
+          fi
+          echo "Base: $base"
+          echo "HEAD: $(git rev-parse HEAD)"
+          git diff --name-only $base HEAD > changed_files.txt
+          echo "detected changes in $(cat changed_files.txt)"
+          # Join files with commas
+          files=$(paste -sd, changed_files.txt)
+          echo "files=$files" >> $FORGEJO_OUTPUT
+
+      - name: Debug output
+        run: |
+          echo "State of output"
+          echo "Changed files: ${{ steps.changes.outputs.files }}"
+
+      - name: Get new toolchain hash
+        if: contains(steps.changes.outputs.files, 'Cargo.toml') || contains(steps.changes.outputs.files, 'Cargo.lock') || contains(steps.changes.outputs.files, 'rust-toolchain.toml')
+        run: |
+          # Set the current sha256 to an empty hash to make `nix build` calculate a new one
+          awk '/fromToolchainFile *\{/{found=1; print; next} found && /sha256 =/{sub(/sha256 = .*/, "sha256 = lib.fakeSha256;"); found=0} 1' nix/packages/rust.nix > temp.nix
+          mv temp.nix nix/packages/rust.nix
+
+          # Build continuwuity and filter for the new hash
+          # We do `|| true` because we want this to fail without stopping the workflow
+          nix build .#default 2>&1 | tee >(grep 'got:' | awk '{print $2}' > new_toolchain_hash.txt) || true
+
+          # Place the new hash in place of the empty hash
+          new_hash=$(cat new_toolchain_hash.txt)
+          sed -i "s|lib.fakeSha256|\"$new_hash\"|" nix/packages/rust.nix
+
+          echo "New hash:"
+          awk -F'"' '/fromToolchainFile/{found=1; next} found && /sha256 =/{print $2; found=0}' nix/packages/rust.nix
+          echo "Expected new hash:"
+          cat new_toolchain_hash.txt
+
+          rm new_toolchain_hash.txt
+
+      - name: Get new rocksdb hash
+        if: contains(steps.changes.outputs.files, '.nix') || contains(steps.changes.outputs.files, 'flake.lock')
+        run: |
+          # Set the current sha256 to an empty hash to make `nix build` calculate a new one
+          awk '/repo = "rocksdb";/{found=1; print; next} found && /sha256 =/{sub(/sha256 = .*/, "sha256 = lib.fakeSha256;"); found=0} 1' nix/packages/rocksdb/package.nix > temp.nix
+          mv temp.nix nix/packages/rocksdb/package.nix
+
+          # Build continuwuity and filter for the new hash
+          # We do `|| true` because we want this to fail without stopping the workflow
+          nix build .#default 2>&1 | tee >(grep 'got:' | awk '{print $2}' > new_rocksdb_hash.txt) || true
+
+          # Place the new hash in place of the empty hash
+          new_hash=$(cat new_rocksdb_hash.txt)
+          sed -i "s|lib.fakeSha256|\"$new_hash\"|" nix/packages/rocksdb/package.nix
+
+          echo "New hash:"
+          awk -F'"' '/repo = "rocksdb";/{found=1; next} found && /sha256 =/{print $2; found=0}' nix/packages/rocksdb/package.nix
+          echo "Expected new hash:"
+          cat new_rocksdb_hash.txt
+
+          rm new_rocksdb_hash.txt
+
+      - name: Show diff
+        run: git diff flake.nix nix
+
+      - name: Push changes
+        run: |
+          set -euo pipefail
+
+          if git diff --quiet --exit-code; then
+            echo "No changes to commit."
+            exit 0
+          fi
+
+          git config user.email "renovate@mail.ellis.link"
+          git config user.name "renovate"
+
+          REF="${{ github.head_ref }}"
+
+          git fetch origin "$REF"
+          git checkout "$REF"
+
+          git commit -a -m "chore(Nix): Updated flake hashes"
+
+          git push origin HEAD:refs/heads/"$REF"

.pre-commit-config.yaml

@@ -7,7 +7,7 @@ default_stages:
 
 repos:
   - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v5.0.0
+    rev: v6.0.0
     hooks:
         - id: fix-byte-order-marker
         - id: check-case-conflict
@@ -23,7 +23,7 @@ repos:
         - id: check-added-large-files
 
   - repo: https://github.com/crate-ci/typos
-    rev: v1.26.0
+    rev: v1.39.0
     hooks:
       - id: typos
       - id: typos

.typos.toml

@@ -13,6 +13,9 @@ extend-ignore-re = [
     "[0-9+][A-Za-z0-9+]{30,}[a-z0-9+]",
     "\\$[A-Z0-9+][A-Za-z0-9+]{6,}[a-z0-9+]",
     "\\b[a-z0-9+/=][A-Za-z0-9+/=]{7,}[a-z0-9+/=][A-Z]\\b",
+
+    # In the renovate config
+    ".ontainer"
 ]
 
 [default.extend-words]

Cargo.toml

@@ -21,7 +21,7 @@ license = "Apache-2.0"
 readme = "README.md"
 repository = "https://forgejo.ellis.link/continuwuation/continuwuity"
 rust-version = "1.86.0"
-version = "0.5.0-rc.7"
+version = "0.5.0-rc.8"
 
 [workspace.metadata.crane]
 name = "conduwuit"
@@ -45,7 +45,7 @@ version = "0.3"
 features = ["ffi", "std", "union"]
 
 [workspace.dependencies.const-str]
-version = "0.6.2"
+version = "0.7.0"
 
 [workspace.dependencies.ctor]
 version = "0.5.0"
@@ -166,8 +166,8 @@ default-features = false
 features = ["raw_value"]
 
 # Used for appservice registration files
-[workspace.dependencies.serde_yaml]
-version = "0.9.34"
+[workspace.dependencies.serde-saphyr]
+version = "0.0.7"
 
 # Used to load forbidden room/user regex from config
 [workspace.dependencies.serde_regex]
@@ -210,13 +210,13 @@ default-features = false
 version = "0.1.41"
 default-features = false
 [workspace.dependencies.tracing-subscriber]
-version = "0.3.19"
+version = "0.3.20"
 default-features = false
 features = ["env-filter", "std", "tracing", "tracing-log", "ansi", "fmt"]
 [workspace.dependencies.tracing-journald]
 version = "0.3.1"
 [workspace.dependencies.tracing-core]
-version = "0.1.33"
+version = "0.1.34"
 default-features = false
 
 # for URL previews
@@ -286,7 +286,7 @@ features = [
 ]
 
 [workspace.dependencies.hyper-util]
-version = "0.1.11"
+version = "=0.1.17"
 default-features = false
 features = [
 	"server-auto",
@@ -351,8 +351,7 @@ version = "0.1.2"
 # Used for matrix spec type definitions and helpers
 [workspace.dependencies.ruma]
 git = "https://forgejo.ellis.link/continuwuation/ruwuma"
-#branch = "conduwuit-changes"
-rev = "8fb268fa2771dfc3a1c8075ef1246e7c9a0a53fd"
+rev = "b96a99ac85264aa6bf102c12e201090c6474c99d"
 features = [
     "compat",
     "rand",
@@ -382,16 +381,18 @@ features = [
     "unstable-msc4095",
     "unstable-msc4121",
     "unstable-msc4125",
+	"unstable-msc4155",
     "unstable-msc4186",
     "unstable-msc4203", # sending to-device events to appservices
     "unstable-msc4210", # remove legacy mentions
     "unstable-extensible-events",
     "unstable-pdu",
+	"unstable-msc4155"
 ]
 
 [workspace.dependencies.rust-rocksdb]
 git = "https://forgejo.ellis.link/continuwuation/rust-rocksdb-zaidoon1"
-rev = "99b0319416b64830dd6f8943e1f65e15aeef18bc"
+rev = "61d9d23872197e9ace4a477f2617d5c9f50ecb23"
 default-features = false
 features = [
 	"multi-threaded-cf",
@@ -411,28 +412,27 @@ default-features = false
 
 # optional opentelemetry, performance measurements, flamegraphs, etc for performance measurements and monitoring
 [workspace.dependencies.opentelemetry]
-version = "0.30.0"
+version = "0.31.0"
 
 [workspace.dependencies.tracing-flame]
 version = "0.2.0"
 
 [workspace.dependencies.tracing-opentelemetry]
-version = "0.31.0"
+version = "0.32.0"
 
 [workspace.dependencies.opentelemetry_sdk]
-version = "0.30.0"
+version = "0.31.0"
 features = ["rt-tokio"]
 
 [workspace.dependencies.opentelemetry-otlp]
-version = "0.30.0"
+version = "0.31.0"
 features = ["http", "trace", "logs", "metrics"]
 
-[workspace.dependencies.opentelemetry-jaeger-propagator]
-version = "0.30.0"
+
 
 # optional sentry metrics for crash/panic reporting
 [workspace.dependencies.sentry]
-version = "0.42.0"
+version = "0.45.0"
 default-features = false
 features = [
     "backtrace",
@@ -448,9 +448,9 @@ features = [
 ]
 
 [workspace.dependencies.sentry-tracing]
-version = "0.42.0"
+version = "0.45.0"
 [workspace.dependencies.sentry-tower]
-version = "0.42.0"
+version = "0.45.0"
 
 # jemalloc usage
 [workspace.dependencies.tikv-jemalloc-sys]
@@ -476,7 +476,7 @@ default-features = false
 features = ["use_std"]
 
 [workspace.dependencies.console-subscriber]
-version = "0.4"
+version = "0.5"
 
 [workspace.dependencies.nix]
 version = "0.30.1"
@@ -550,9 +550,12 @@ features = ["std"]
 version = "1.0.2"
 
 [workspace.dependencies.ldap3]
-version = "0.11.5"
+version = "0.12.0"
 default-features = false
-features = ["sync", "tls-rustls"]
+features = ["sync", "tls-rustls", "rustls-provider"]
+
+[workspace.dependencies.resolv-conf]
+version = "0.7.5"
 
 #
 # Patches
@@ -560,19 +563,7 @@ features = ["sync", "tls-rustls"]
 
 # backport of [https://github.com/tokio-rs/tracing/pull/2956] to the 0.1.x branch of tracing.
 # we can switch back to upstream if #2956 is merged and backported in the upstream repo.
-# https://forgejo.ellis.link/continuwuation/tracing/commit/b348dca742af641c47bc390261f60711c2af573c
-[patch.crates-io.tracing-subscriber]
-git = "https://forgejo.ellis.link/continuwuation/tracing"
-rev = "1e64095a8051a1adf0d1faa307f9f030889ec2aa"
-[patch.crates-io.tracing]
-git = "https://forgejo.ellis.link/continuwuation/tracing"
-rev = "1e64095a8051a1adf0d1faa307f9f030889ec2aa"
-[patch.crates-io.tracing-core]
-git = "https://forgejo.ellis.link/continuwuation/tracing"
-rev = "1e64095a8051a1adf0d1faa307f9f030889ec2aa"
-[patch.crates-io.tracing-log]
-git = "https://forgejo.ellis.link/continuwuation/tracing"
-rev = "1e64095a8051a1adf0d1faa307f9f030889ec2aa"
+
 
 # adds a tab completion callback: https://forgejo.ellis.link/continuwuation/rustyline-async/src/branch/main/.patchy/0002-add-tab-completion-callback.patch
 # adds event for CTRL+\: https://forgejo.ellis.link/continuwuation/rustyline-async/src/branch/main/.patchy/0001-add-event-for-ctrl.patch
@@ -596,13 +587,7 @@ rev = "9c8e51510c35077df888ee72a36b4b05637147da"
 # reverts hyperium#148 conflicting with our delicate federation resolver hooks
 [patch.crates-io.hyper-util]
 git = "https://forgejo.ellis.link/continuwuation/hyper-util"
-rev = "e4ae7628fe4fcdacef9788c4c8415317a4489941"
-
-# Allows no-aaaa option in resolv.conf
-# Use 1-indexed line numbers when displaying parse error messages
-[patch.crates-io.resolv-conf]
-git = "https://forgejo.ellis.link/continuwuation/resolv-conf"
-rev = "56251316cc4127bcbf36e68ce5e2093f4d33e227"
+rev = "5886d5292bf704c246206ad72d010d674a7b77d0"
 
 #
 # Our crates
@@ -962,7 +947,7 @@ semicolon_outside_block = "warn"
 str_to_string = "warn"
 string_lit_chars_any = "warn"
 string_slice = "warn"
-string_to_string = "warn"
+
 suspicious_xor_used_as_pow = "warn"
 tests_outside_test_module = "warn"
 try_err = "warn"

conduwuit-example.toml

@@ -957,6 +957,21 @@
 #
 #rocksdb_bottommost_compression = true
 
+# Compression algorithm for RocksDB's Write-Ahead-Log (WAL).
+#
+# At present, only ZSTD compression is supported by RocksDB for WAL
+# compression. Enabling this can reduce WAL size at the expense of some
+# CPU usage during writes.
+#
+# The options are:
+# - "none" = No compression
+# - "zstd" = ZSTD compression
+#
+# For more information on WAL compression, see:
+# https://github.com/facebook/rocksdb/wiki/WAL-Compression
+#
+#rocksdb_wal_compression = "zstd"
+
 # Database recovery mode (for RocksDB WAL corruption).
 #
 # Use this option when the server reports corruption and refuses to start.
@@ -1497,6 +1512,19 @@
 #
 #block_non_admin_invites = false
 
+# Enable or disable making requests to MSC4284 Policy Servers.
+# It is recommended you keep this enabled unless you experience frequent
+# connectivity issues, such as in a restricted networking environment.
+#
+#enable_msc4284_policy_servers = true
+
+# Enable running locally generated events through configured MSC4284
+# policy servers. You may wish to disable this if your server is
+# single-user for a slight speed benefit in some rooms, but otherwise
+# should leave it enabled.
+#
+#policy_server_check_own_events = true
+
 # Allow admins to enter commands in rooms other than "#admins" (admin
 # room) by prefixing your message with "\!admin" or "\\!admin" followed up
 # a normal continuwuity admin command. The reply will be publicly visible

docker/Dockerfile

@@ -48,11 +48,13 @@ EOF
 
 # Developer tool versions
 # renovate: datasource=github-releases depName=cargo-bins/cargo-binstall
-ENV BINSTALL_VERSION=1.13.0
+ENV BINSTALL_VERSION=1.15.11
 # renovate: datasource=github-releases depName=psastras/sbom-rs
 ENV CARGO_SBOM_VERSION=0.9.1
 # renovate: datasource=crate depName=lddtree
 ENV LDDTREE_VERSION=0.3.7
+# renovate: datasource=crate depName=timelord-cli
+ENV TIMELORD_VERSION=3.0.1
 
 # Install unpackaged tools
 RUN <<EOF
@@ -60,6 +62,7 @@ RUN <<EOF
     curl --retry 5 -L --proto '=https' --tlsv1.2 -sSf https://raw.githubusercontent.com/cargo-bins/cargo-binstall/main/install-from-binstall-release.sh | bash
     cargo binstall --no-confirm cargo-sbom --version $CARGO_SBOM_VERSION
     cargo binstall --no-confirm lddtree --version $LDDTREE_VERSION
+    cargo binstall --no-confirm timelord-cli --version $TIMELORD_VERSION
 EOF
 
 # Set up xx (cross-compilation scripts)
@@ -81,8 +84,9 @@ RUN rustc --version \
     && xx-cargo --setup-target-triple
 
 # Build binary
-# We disable incremental compilation to save disk space, as it only produces a minimal speedup for this case.
-RUN echo "CARGO_INCREMENTAL=0" >> /etc/environment
+# Configure incremental compilation based on build context
+ARG CARGO_INCREMENTAL=0
+RUN echo "CARGO_INCREMENTAL=${CARGO_INCREMENTAL}" >> /etc/environment
 
 # Configure pkg-config
 RUN <<EOF
@@ -133,6 +137,11 @@ FROM toolchain AS builder
 # Get source
 COPY . .
 
+# Restore timestamps from timelord cache if available
+RUN --mount=type=cache,target=/timelord/ \
+    echo "Restoring timestamps from timelord cache"; \
+    timelord sync --source-dir /app --cache-dir /timelord;
+
 ARG TARGETPLATFORM
 
 # Verify environment configuration
@@ -157,7 +166,7 @@ ARG RUST_PROFILE=release
 # Build the binary
 RUN --mount=type=cache,target=/usr/local/cargo/registry \
     --mount=type=cache,target=/usr/local/cargo/git/db \
-    --mount=type=cache,target=/app/target,id=cargo-target-${TARGET_CPU}-${TARGETPLATFORM}-${RUST_PROFILE} \
+    --mount=type=cache,target=/app/target,id=continuwuity-cargo-target-${TARGET_CPU}-${TARGETPLATFORM}-${RUST_PROFILE} \
     bash <<'EOF'
     set -o allexport
     set -o xtrace
@@ -172,8 +181,8 @@ RUN --mount=type=cache,target=/usr/local/cargo/registry \
         jq -r ".packages[] | select(.name == \"$PACKAGE\") | .targets[] | select( .kind | map(. == \"bin\") | any ) | .name"))
     for BINARY in "${BINARIES[@]}"; do
         echo $BINARY
-        xx-verify $TARGET_DIR/$(xx-cargo --print-target-triple)/release/$BINARY
-        cp $TARGET_DIR/$(xx-cargo --print-target-triple)/release/$BINARY /out/sbin/$BINARY
+        xx-verify $TARGET_DIR/$(xx-cargo --print-target-triple)/${RUST_PROFILE}/$BINARY
+        cp $TARGET_DIR/$(xx-cargo --print-target-triple)/${RUST_PROFILE}/$BINARY /out/sbin/$BINARY
     done
 EOF
 

docker/musl.Dockerfile

@@ -18,7 +18,7 @@ RUN --mount=type=cache,target=/etc/apk/cache apk add \
 
 # Developer tool versions
 # renovate: datasource=github-releases depName=cargo-bins/cargo-binstall
-ENV BINSTALL_VERSION=1.13.0
+ENV BINSTALL_VERSION=1.15.11
 # renovate: datasource=github-releases depName=psastras/sbom-rs
 ENV CARGO_SBOM_VERSION=0.9.1
 # renovate: datasource=crate depName=lddtree
@@ -122,7 +122,7 @@ ARG RUST_PROFILE=release
 # Build the binary
 RUN --mount=type=cache,target=/usr/local/cargo/registry \
     --mount=type=cache,target=/usr/local/cargo/git/db \
-    --mount=type=cache,target=/app/target,id=cargo-target-${TARGET_CPU}-${TARGETPLATFORM}-musl-${RUST_PROFILE} \
+    --mount=type=cache,target=/app/target,id=continuwuity-cargo-target-${TARGET_CPU}-${TARGETPLATFORM}-musl-${RUST_PROFILE} \
     bash <<'EOF'
     set -o allexport
     set -o xtrace

docs/SUMMARY.md

@@ -10,6 +10,7 @@ # Summary
   - [Kubernetes](deploying/kubernetes.md)
   - [Arch Linux](deploying/arch-linux.md)
   - [Debian](deploying/debian.md)
+  - [Fedora](deploying/fedora.md)
   - [FreeBSD](deploying/freebsd.md)
 - [TURN](turn.md)
 - [Appservices](appservices.md)

docs/admin_reference.md

@@ -1078,7 +1078,10 @@ ###### **Subcommands:**
 
 * `delete` — - Deletes a single media file from our database and on the filesystem via a single MXC URL or event ID (not redacted)
 * `delete-list` — - Deletes a codeblock list of MXC URLs from our database and on the filesystem. This will always ignore errors
-* `delete-past-remote-media` — - Deletes all remote (and optionally local) media created before or after [duration] time using filesystem metadata first created at date, or fallback to last modified date. This will always ignore errors by default
+* `delete-past-remote-media` — Deletes all remote (and optionally local) media created before/after
+[duration] ago, using filesystem metadata first created at date, or
+fallback to last modified date. This will always ignore errors by
+default.
 * `delete-all-from-user` — - Deletes all the local media from a local user on our server. This will always ignore errors by default
 * `delete-all-from-server` — - Deletes all remote media from the specified remote server. This will always ignore errors by default
 * `get-file-info` —
@@ -1110,13 +1113,25 @@ ## `admin media delete-list`
 
 ## `admin media delete-past-remote-media`
 
-- Deletes all remote (and optionally local) media created before or after [duration] time using filesystem metadata first created at date, or fallback to last modified date. This will always ignore errors by default
+Deletes all remote (and optionally local) media created before/after
+[duration] ago, using filesystem metadata first created at date, or
+fallback to last modified date. This will always ignore errors by
+default.
+
+* Examples:
+  * Delete all remote media older than a year:
+
+    `!admin media delete-past-remote-media -b 1y`
+
+  * Delete all remote and local media from 3 days ago, up until now:
+
+    `!admin media delete-past-remote-media -a 3d --yes-i-want-to-delete-local-media`
 
 **Usage:** `admin media delete-past-remote-media [OPTIONS] <DURATION>`
 
 ###### **Arguments:**
 
-* `<DURATION>` — - The relative time (e.g. 30s, 5m, 7d) within which to search
+* `<DURATION>` — - The relative time (e.g. 30s, 5m, 7d) from now within which to search
 
 ###### **Options:**
 

docs/deploying/fedora.md

@@ -0,0 +1,201 @@
+# RPM Installation Guide
+
+Continuwuity is available as RPM packages for Fedora, RHEL, and compatible distributions.
+
+The RPM packaging files are maintained in the `fedora/` directory:
+- `continuwuity.spec.rpkg` - RPM spec file using rpkg macros for building from git
+- `continuwuity.service` - Systemd service file for the server
+- `RPM-GPG-KEY-continuwuity.asc` - GPG public key for verifying signed packages
+
+RPM packages built by CI are signed with our GPG key (Ed25519, ID: `5E0FF73F411AAFCA`).
+
+```bash
+# Import the signing key
+sudo rpm --import https://forgejo.ellis.link/continuwuation/continuwuity/raw/branch/main/fedora/RPM-GPG-KEY-continuwuity.asc
+
+# Verify a downloaded package
+rpm --checksig continuwuity-*.rpm
+```
+
+## Installation methods
+
+**Stable releases** (recommended)
+
+```bash
+# Add the repository and install
+sudo dnf config-manager addrepo --from-repofile=https://forgejo.ellis.link/api/packages/continuwuation/rpm/stable/continuwuation.repo
+sudo dnf install continuwuity
+```
+
+**Development builds** from main branch
+
+```bash
+# Add the dev repository and install
+sudo dnf config-manager addrepo --from-repofile=https://forgejo.ellis.link/api/packages/continuwuation/rpm/dev/continuwuation.repo
+sudo dnf install continuwuity
+```
+
+**Feature branch builds** (example: `tom/new-feature`)
+
+```bash
+# Branch names are sanitized (slashes become hyphens, lowercase only)
+sudo dnf config-manager addrepo --from-repofile=https://forgejo.ellis.link/api/packages/continuwuation/rpm/tom-new-feature/continuwuation.repo
+sudo dnf install continuwuity
+```
+
+**Direct installation** without adding repository
+
+```bash
+# Latest stable release
+sudo dnf install https://forgejo.ellis.link/api/packages/continuwuation/rpm/stable/continuwuity
+
+# Latest development build
+sudo dnf install https://forgejo.ellis.link/api/packages/continuwuation/rpm/dev/continuwuity
+
+# Specific feature branch
+sudo dnf install https://forgejo.ellis.link/api/packages/continuwuation/rpm/branch-name/continuwuity
+```
+
+**Manual repository configuration** (alternative method)
+
+```bash
+cat << 'EOF' | sudo tee /etc/yum.repos.d/continuwuity.repo
+[continuwuity]
+name=Continuwuity - Matrix homeserver
+baseurl=https://forgejo.ellis.link/api/packages/continuwuation/rpm/stable
+enabled=1
+gpgcheck=1
+gpgkey=https://forgejo.ellis.link/continuwuation/continuwuity/raw/branch/main/fedora/RPM-GPG-KEY-continuwuity.asc
+EOF
+
+sudo dnf install continuwuity
+```
+
+## Package management
+
+**Automatic updates** with DNF Automatic
+
+```bash
+# Install and configure
+sudo dnf install dnf-automatic
+sudo nano /etc/dnf/automatic.conf  # Set: apply_updates = yes
+sudo systemctl enable --now dnf-automatic.timer
+```
+
+**Manual updates**
+
+```bash
+# Check for updates
+sudo dnf check-update continuwuity
+
+# Update to latest version
+sudo dnf update continuwuity
+```
+
+**Switching channels** (stable/dev/feature branches)
+
+```bash
+# List enabled repositories
+dnf repolist | grep continuwuation
+
+# Disable current repository
+sudo dnf config-manager --set-disabled continuwuation-stable  # or -dev, or branch name
+
+# Enable desired repository
+sudo dnf config-manager --set-enabled continuwuation-dev  # or -stable, or branch name
+
+# Update to the new channel's version
+sudo dnf update continuwuity
+```
+
+**Verifying installation**
+
+```bash
+# Check installed version
+rpm -q continuwuity
+
+# View package information
+rpm -qi continuwuity
+
+# List installed files
+rpm -ql continuwuity
+
+# Verify package integrity
+rpm -V continuwuity
+```
+
+## Service management and removal
+
+**Systemd service commands**
+
+```bash
+# Start the service
+sudo systemctl start conduwuit
+
+# Enable on boot
+sudo systemctl enable conduwuit
+
+# Check status
+sudo systemctl status conduwuit
+
+# View logs
+sudo journalctl -u conduwuit -f
+```
+
+**Uninstallation**
+
+```bash
+# Stop and disable the service
+sudo systemctl stop conduwuit
+sudo systemctl disable conduwuit
+
+# Remove the package
+sudo dnf remove continuwuity
+
+# Remove the repository (optional)
+sudo rm /etc/yum.repos.d/continuwuation-*.repo
+```
+
+## Troubleshooting
+
+**GPG key errors**: Temporarily disable GPG checking
+
+```bash
+sudo dnf --nogpgcheck install continuwuity
+```
+
+**Repository metadata issues**: Clear and rebuild cache
+
+```bash
+sudo dnf clean all
+sudo dnf makecache
+```
+
+**Finding specific versions**
+
+```bash
+# List all available versions
+dnf --showduplicates list continuwuity
+
+# Install a specific version
+sudo dnf install continuwuity-<version>
+```
+
+## Building locally
+
+Build the RPM locally using rpkg:
+
+```bash
+# Install dependencies
+sudo dnf install rpkg rpm-build cargo-rpm-macros systemd-rpm-macros
+
+# Clone the repository
+git clone https://forgejo.ellis.link/continuwuation/continuwuity.git
+cd continuwuity
+
+# Build SRPM
+rpkg srpm
+
+# Build RPM
+rpmbuild --rebuild *.src.rpm
+```

docs/development/code_style.md

@@ -241,7 +241,7 @@ ## Documentation
 ### Code Comments
 
 - Reference related documentation or parts of the specification
-- When a task has multiple ways of being acheved, explain your reasoning for your decision
+- When a task has multiple ways of being achieved, explain your reasoning for your decision
 - Update comments when code changes
 
 ```rs

flake.lock

@@ -1,95 +1,28 @@
 {
   "nodes": {
-    "attic": {
-      "inputs": {
-        "crane": "crane",
-        "flake-compat": "flake-compat",
-        "flake-parts": "flake-parts",
-        "nix-github-actions": "nix-github-actions",
-        "nixpkgs": "nixpkgs",
-        "nixpkgs-stable": "nixpkgs-stable"
-      },
+    "advisory-db": {
+      "flake": false,
       "locked": {
-        "lastModified": 1751403276,
-        "narHash": "sha256-V0EPQNsQko1a8OqIWc2lLviLnMpR1m08Ej00z5RVTfs=",
-        "owner": "zhaofengli",
-        "repo": "attic",
-        "rev": "896ad88fa57ad5dbcd267c0ac51f1b71ccfcb4dd",
+        "lastModified": 1761112158,
+        "narHash": "sha256-RIXu/7eyKpQHjsPuAUODO81I4ni8f+WYSb7K4mTG6+0=",
+        "owner": "rustsec",
+        "repo": "advisory-db",
+        "rev": "58f3aaec0e1776f4a900737be8cd7cb00972210d",
         "type": "github"
       },
       "original": {
-        "owner": "zhaofengli",
-        "ref": "main",
-        "repo": "attic",
-        "type": "github"
-      }
-    },
-    "cachix": {
-      "inputs": {
-        "devenv": "devenv",
-        "flake-compat": "flake-compat_2",
-        "git-hooks": "git-hooks",
-        "nixpkgs": "nixpkgs_4"
-      },
-      "locked": {
-        "lastModified": 1748883665,
-        "narHash": "sha256-R0W7uAg+BLoHjMRMQ8+oiSbTq8nkGz5RDpQ+ZfxxP3A=",
-        "owner": "cachix",
-        "repo": "cachix",
-        "rev": "f707778d902af4d62d8dd92c269f8e70de09acbe",
-        "type": "github"
-      },
-      "original": {
-        "owner": "cachix",
-        "ref": "master",
-        "repo": "cachix",
-        "type": "github"
-      }
-    },
-    "cachix_2": {
-      "inputs": {
-        "devenv": [
-          "cachix",
-          "devenv"
-        ],
-        "flake-compat": [
-          "cachix",
-          "devenv"
-        ],
-        "git-hooks": [
-          "cachix",
-          "devenv"
-        ],
-        "nixpkgs": "nixpkgs_2"
-      },
-      "locked": {
-        "lastModified": 1744206633,
-        "narHash": "sha256-pb5aYkE8FOoa4n123slgHiOf1UbNSnKe5pEZC+xXD5g=",
-        "owner": "cachix",
-        "repo": "cachix",
-        "rev": "8a60090640b96f9df95d1ab99e5763a586be1404",
-        "type": "github"
-      },
-      "original": {
-        "owner": "cachix",
-        "ref": "latest",
-        "repo": "cachix",
+        "owner": "rustsec",
+        "repo": "advisory-db",
         "type": "github"
       }
     },
     "crane": {
-      "inputs": {
-        "nixpkgs": [
-          "attic",
-          "nixpkgs"
-        ]
-      },
       "locked": {
-        "lastModified": 1722960479,
-        "narHash": "sha256-NhCkJJQhD5GUib8zN9JrmYGMwt4lCRp6ZVNzIiYCl0Y=",
+        "lastModified": 1760924934,
+        "narHash": "sha256-tuuqY5aU7cUkR71sO2TraVKK2boYrdW3gCSXUkF4i44=",
         "owner": "ipetkov",
         "repo": "crane",
-        "rev": "4c6c77920b8d44cd6660c1621dea6b3fc4b4c4f4",
+        "rev": "c6b4d5308293d0d04fcfeee92705017537cad02f",
         "type": "github"
       },
       "original": {
@@ -98,53 +31,6 @@
         "type": "github"
       }
     },
-    "crane_2": {
-      "locked": {
-        "lastModified": 1750266157,
-        "narHash": "sha256-tL42YoNg9y30u7zAqtoGDNdTyXTi8EALDeCB13FtbQA=",
-        "owner": "ipetkov",
-        "repo": "crane",
-        "rev": "e37c943371b73ed87faf33f7583860f81f1d5a48",
-        "type": "github"
-      },
-      "original": {
-        "owner": "ipetkov",
-        "ref": "master",
-        "repo": "crane",
-        "type": "github"
-      }
-    },
-    "devenv": {
-      "inputs": {
-        "cachix": "cachix_2",
-        "flake-compat": [
-          "cachix",
-          "flake-compat"
-        ],
-        "git-hooks": [
-          "cachix",
-          "git-hooks"
-        ],
-        "nix": "nix",
-        "nixpkgs": [
-          "cachix",
-          "nixpkgs"
-        ]
-      },
-      "locked": {
-        "lastModified": 1748273445,
-        "narHash": "sha256-5V0dzpNgQM0CHDsMzh+ludYeu1S+Y+IMjbaskSSdFh0=",
-        "owner": "cachix",
-        "repo": "devenv",
-        "rev": "668a50d8b7bdb19a0131f53c9f6c25c9071e1ffb",
-        "type": "github"
-      },
-      "original": {
-        "owner": "cachix",
-        "repo": "devenv",
-        "type": "github"
-      }
-    },
     "fenix": {
       "inputs": {
         "nixpkgs": [
@@ -153,53 +39,20 @@
         "rust-analyzer-src": "rust-analyzer-src"
       },
       "locked": {
-        "lastModified": 1755585599,
-        "narHash": "sha256-tl/0cnsqB/Yt7DbaGMel2RLa7QG5elA8lkaOXli6VdY=",
+        "lastModified": 1761115517,
+        "narHash": "sha256-Fev/ag/c3Fp3JBwHfup3lpA5FlNXfkoshnQ7dssBgJ0=",
         "owner": "nix-community",
         "repo": "fenix",
-        "rev": "6ed03ef4c8ec36d193c18e06b9ecddde78fb7e42",
+        "rev": "320433651636186ea32b387cff05d6bbfa30cea7",
         "type": "github"
       },
       "original": {
         "owner": "nix-community",
-        "ref": "main",
         "repo": "fenix",
         "type": "github"
       }
     },
     "flake-compat": {
-      "flake": false,
-      "locked": {
-        "lastModified": 1696426674,
-        "narHash": "sha256-kvjfFW7WAETZlt09AgDn1MrtKzP7t90Vf7vypd3OL1U=",
-        "owner": "edolstra",
-        "repo": "flake-compat",
-        "rev": "0f9255e01c2351cc7d116c072cb317785dd33b33",
-        "type": "github"
-      },
-      "original": {
-        "owner": "edolstra",
-        "repo": "flake-compat",
-        "type": "github"
-      }
-    },
-    "flake-compat_2": {
-      "flake": false,
-      "locked": {
-        "lastModified": 1747046372,
-        "narHash": "sha256-CIVLLkVgvHYbgI2UpXvIIBJ12HWgX+fjA8Xf8PUmqCY=",
-        "owner": "edolstra",
-        "repo": "flake-compat",
-        "rev": "9100a0f413b0c601e0533d1d94ffd501ce2e7885",
-        "type": "github"
-      },
-      "original": {
-        "owner": "edolstra",
-        "repo": "flake-compat",
-        "type": "github"
-      }
-    },
-    "flake-compat_3": {
       "flake": false,
       "locked": {
         "lastModified": 1747046372,
@@ -218,17 +71,14 @@
     },
     "flake-parts": {
       "inputs": {
-        "nixpkgs-lib": [
-          "attic",
-          "nixpkgs"
-        ]
+        "nixpkgs-lib": "nixpkgs-lib"
       },
       "locked": {
-        "lastModified": 1722555600,
-        "narHash": "sha256-XOQkdLafnb/p9ij77byFQjDf5m5QYl9b2REiVClC+x4=",
+        "lastModified": 1760948891,
+        "narHash": "sha256-TmWcdiUUaWk8J4lpjzu4gCGxWY6/Ok7mOK4fIFfBuU4=",
         "owner": "hercules-ci",
         "repo": "flake-parts",
-        "rev": "8471fe90ad337a8074e957b69ca4d0089218391d",
+        "rev": "864599284fc7c0ba6357ed89ed5e2cd5040f0c04",
         "type": "github"
       },
       "original": {
@@ -237,225 +87,13 @@
         "type": "github"
       }
     },
-    "flake-parts_2": {
-      "inputs": {
-        "nixpkgs-lib": [
-          "cachix",
-          "devenv",
-          "nix",
-          "nixpkgs"
-        ]
-      },
-      "locked": {
-        "lastModified": 1712014858,
-        "narHash": "sha256-sB4SWl2lX95bExY2gMFG5HIzvva5AVMJd4Igm+GpZNw=",
-        "owner": "hercules-ci",
-        "repo": "flake-parts",
-        "rev": "9126214d0a59633752a136528f5f3b9aa8565b7d",
-        "type": "github"
-      },
-      "original": {
-        "owner": "hercules-ci",
-        "repo": "flake-parts",
-        "type": "github"
-      }
-    },
-    "flake-utils": {
-      "inputs": {
-        "systems": "systems"
-      },
-      "locked": {
-        "lastModified": 1731533236,
-        "narHash": "sha256-l0KFg5HjrsfsO/JpG+r7fRrqm12kzFHyUHqHCVpMMbI=",
-        "owner": "numtide",
-        "repo": "flake-utils",
-        "rev": "11707dc2f618dd54ca8739b309ec4fc024de578b",
-        "type": "github"
-      },
-      "original": {
-        "owner": "numtide",
-        "ref": "main",
-        "repo": "flake-utils",
-        "type": "github"
-      }
-    },
-    "git-hooks": {
-      "inputs": {
-        "flake-compat": [
-          "cachix",
-          "flake-compat"
-        ],
-        "gitignore": "gitignore",
-        "nixpkgs": [
-          "cachix",
-          "nixpkgs"
-        ]
-      },
-      "locked": {
-        "lastModified": 1747372754,
-        "narHash": "sha256-2Y53NGIX2vxfie1rOW0Qb86vjRZ7ngizoo+bnXU9D9k=",
-        "owner": "cachix",
-        "repo": "git-hooks.nix",
-        "rev": "80479b6ec16fefd9c1db3ea13aeb038c60530f46",
-        "type": "github"
-      },
-      "original": {
-        "owner": "cachix",
-        "repo": "git-hooks.nix",
-        "type": "github"
-      }
-    },
-    "gitignore": {
-      "inputs": {
-        "nixpkgs": [
-          "cachix",
-          "git-hooks",
-          "nixpkgs"
-        ]
-      },
-      "locked": {
-        "lastModified": 1709087332,
-        "narHash": "sha256-HG2cCnktfHsKV0s4XW83gU3F57gaTljL9KNSuG6bnQs=",
-        "owner": "hercules-ci",
-        "repo": "gitignore.nix",
-        "rev": "637db329424fd7e46cf4185293b9cc8c88c95394",
-        "type": "github"
-      },
-      "original": {
-        "owner": "hercules-ci",
-        "repo": "gitignore.nix",
-        "type": "github"
-      }
-    },
-    "libgit2": {
-      "flake": false,
-      "locked": {
-        "lastModified": 1697646580,
-        "narHash": "sha256-oX4Z3S9WtJlwvj0uH9HlYcWv+x1hqp8mhXl7HsLu2f0=",
-        "owner": "libgit2",
-        "repo": "libgit2",
-        "rev": "45fd9ed7ae1a9b74b957ef4f337bc3c8b3df01b5",
-        "type": "github"
-      },
-      "original": {
-        "owner": "libgit2",
-        "repo": "libgit2",
-        "type": "github"
-      }
-    },
-    "nix": {
-      "inputs": {
-        "flake-compat": [
-          "cachix",
-          "devenv"
-        ],
-        "flake-parts": "flake-parts_2",
-        "libgit2": "libgit2",
-        "nixpkgs": "nixpkgs_3",
-        "nixpkgs-23-11": [
-          "cachix",
-          "devenv"
-        ],
-        "nixpkgs-regression": [
-          "cachix",
-          "devenv"
-        ],
-        "pre-commit-hooks": [
-          "cachix",
-          "devenv"
-        ]
-      },
-      "locked": {
-        "lastModified": 1745930071,
-        "narHash": "sha256-bYyjarS3qSNqxfgc89IoVz8cAFDkF9yPE63EJr+h50s=",
-        "owner": "domenkozar",
-        "repo": "nix",
-        "rev": "b455edf3505f1bf0172b39a735caef94687d0d9c",
-        "type": "github"
-      },
-      "original": {
-        "owner": "domenkozar",
-        "ref": "devenv-2.24",
-        "repo": "nix",
-        "type": "github"
-      }
-    },
-    "nix-filter": {
-      "locked": {
-        "lastModified": 1731533336,
-        "narHash": "sha256-oRam5PS1vcrr5UPgALW0eo1m/5/pls27Z/pabHNy2Ms=",
-        "owner": "numtide",
-        "repo": "nix-filter",
-        "rev": "f7653272fd234696ae94229839a99b73c9ab7de0",
-        "type": "github"
-      },
-      "original": {
-        "owner": "numtide",
-        "ref": "main",
-        "repo": "nix-filter",
-        "type": "github"
-      }
-    },
-    "nix-github-actions": {
-      "inputs": {
-        "nixpkgs": [
-          "attic",
-          "nixpkgs"
-        ]
-      },
-      "locked": {
-        "lastModified": 1729742964,
-        "narHash": "sha256-B4mzTcQ0FZHdpeWcpDYPERtyjJd/NIuaQ9+BV1h+MpA=",
-        "owner": "nix-community",
-        "repo": "nix-github-actions",
-        "rev": "e04df33f62cdcf93d73e9a04142464753a16db67",
-        "type": "github"
-      },
-      "original": {
-        "owner": "nix-community",
-        "repo": "nix-github-actions",
-        "type": "github"
-      }
-    },
     "nixpkgs": {
       "locked": {
-        "lastModified": 1726042813,
-        "narHash": "sha256-LnNKCCxnwgF+575y0pxUdlGZBO/ru1CtGHIqQVfvjlA=",
+        "lastModified": 1760878510,
+        "narHash": "sha256-K5Osef2qexezUfs0alLvZ7nQFTGS9DL2oTVsIXsqLgs=",
         "owner": "NixOS",
         "repo": "nixpkgs",
-        "rev": "159be5db480d1df880a0135ca0bfed84c2f88353",
-        "type": "github"
-      },
-      "original": {
-        "owner": "NixOS",
-        "ref": "nixpkgs-unstable",
-        "repo": "nixpkgs",
-        "type": "github"
-      }
-    },
-    "nixpkgs-stable": {
-      "locked": {
-        "lastModified": 1724316499,
-        "narHash": "sha256-Qb9MhKBUTCfWg/wqqaxt89Xfi6qTD3XpTzQ9eXi3JmE=",
-        "owner": "NixOS",
-        "repo": "nixpkgs",
-        "rev": "797f7dc49e0bc7fab4b57c021cdf68f595e47841",
-        "type": "github"
-      },
-      "original": {
-        "owner": "NixOS",
-        "ref": "nixos-24.05",
-        "repo": "nixpkgs",
-        "type": "github"
-      }
-    },
-    "nixpkgs_2": {
-      "locked": {
-        "lastModified": 1733212471,
-        "narHash": "sha256-M1+uCoV5igihRfcUKrr1riygbe73/dzNnzPsmaLCmpo=",
-        "owner": "NixOS",
-        "repo": "nixpkgs",
-        "rev": "55d15ad12a74eb7d4646254e13638ad0c4128776",
+        "rev": "5e2a59a5b1a82f89f2c7e598302a9cacebb72a67",
         "type": "github"
       },
       "original": {
@@ -465,74 +103,40 @@
         "type": "github"
       }
     },
-    "nixpkgs_3": {
+    "nixpkgs-lib": {
       "locked": {
-        "lastModified": 1717432640,
-        "narHash": "sha256-+f9c4/ZX5MWDOuB1rKoWj+lBNm0z0rs4CK47HBLxy1o=",
-        "owner": "NixOS",
-        "repo": "nixpkgs",
-        "rev": "88269ab3044128b7c2f4c7d68448b2fb50456870",
+        "lastModified": 1754788789,
+        "narHash": "sha256-x2rJ+Ovzq0sCMpgfgGaaqgBSwY+LST+WbZ6TytnT9Rk=",
+        "owner": "nix-community",
+        "repo": "nixpkgs.lib",
+        "rev": "a73b9c743612e4244d865a2fdee11865283c04e6",
         "type": "github"
       },
       "original": {
-        "owner": "NixOS",
-        "ref": "release-24.05",
-        "repo": "nixpkgs",
-        "type": "github"
-      }
-    },
-    "nixpkgs_4": {
-      "locked": {
-        "lastModified": 1748190013,
-        "narHash": "sha256-R5HJFflOfsP5FBtk+zE8FpL8uqE7n62jqOsADvVshhE=",
-        "owner": "NixOS",
-        "repo": "nixpkgs",
-        "rev": "62b852f6c6742134ade1abdd2a21685fd617a291",
-        "type": "github"
-      },
-      "original": {
-        "owner": "NixOS",
-        "ref": "nixos-unstable",
-        "repo": "nixpkgs",
-        "type": "github"
-      }
-    },
-    "nixpkgs_5": {
-      "locked": {
-        "lastModified": 1751498133,
-        "narHash": "sha256-QWJ+NQbMU+NcU2xiyo7SNox1fAuwksGlQhpzBl76g1I=",
-        "owner": "NixOS",
-        "repo": "nixpkgs",
-        "rev": "d55716bb59b91ae9d1ced4b1ccdea7a442ecbfdb",
-        "type": "github"
-      },
-      "original": {
-        "owner": "NixOS",
-        "ref": "nixpkgs-unstable",
-        "repo": "nixpkgs",
+        "owner": "nix-community",
+        "repo": "nixpkgs.lib",
         "type": "github"
       }
     },
     "root": {
       "inputs": {
-        "attic": "attic",
-        "cachix": "cachix",
-        "crane": "crane_2",
+        "advisory-db": "advisory-db",
+        "crane": "crane",
         "fenix": "fenix",
-        "flake-compat": "flake-compat_3",
-        "flake-utils": "flake-utils",
-        "nix-filter": "nix-filter",
-        "nixpkgs": "nixpkgs_5"
+        "flake-compat": "flake-compat",
+        "flake-parts": "flake-parts",
+        "nixpkgs": "nixpkgs",
+        "treefmt-nix": "treefmt-nix"
       }
     },
     "rust-analyzer-src": {
       "flake": false,
       "locked": {
-        "lastModified": 1755504847,
-        "narHash": "sha256-VX0B9hwhJypCGqncVVLC+SmeMVd/GAYbJZ0MiiUn2Pk=",
+        "lastModified": 1761077270,
+        "narHash": "sha256-O1uTuvI/rUlubJ8AXKyzh1WSWV3qCZX0huTFUvWLN4E=",
         "owner": "rust-lang",
         "repo": "rust-analyzer",
-        "rev": "a905e3b21b144d77e1b304e49f3264f6f8d4db75",
+        "rev": "39990a923c8bca38f5bd29dc4c96e20ee7808d5d",
         "type": "github"
       },
       "original": {
@@ -542,18 +146,23 @@
         "type": "github"
       }
     },
-    "systems": {
+    "treefmt-nix": {
+      "inputs": {
+        "nixpkgs": [
+          "nixpkgs"
+        ]
+      },
       "locked": {
-        "lastModified": 1681028828,
-        "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=",
-        "owner": "nix-systems",
-        "repo": "default",
-        "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e",
+        "lastModified": 1760945191,
+        "narHash": "sha256-ZRVs8UqikBa4Ki3X4KCnMBtBW0ux1DaT35tgsnB1jM4=",
+        "owner": "numtide",
+        "repo": "treefmt-nix",
+        "rev": "f56b1934f5f8fcab8deb5d38d42fd692632b47c2",
         "type": "github"
       },
       "original": {
-        "owner": "nix-systems",
-        "repo": "default",
+        "owner": "numtide",
+        "repo": "treefmt-nix",
         "type": "github"
       }
     }

flake.nix

@@ -1,351 +1,46 @@
 {
+  description = "A nix flake for the continuwuity project";
+
   inputs = {
-    attic.url = "github:zhaofengli/attic?ref=main";
-    cachix.url = "github:cachix/cachix?ref=master";
-    crane = {
-      url = "github:ipetkov/crane?ref=master";
-    };
+    # basics
+    flake-parts.url = "github:hercules-ci/flake-parts";
+    nixpkgs.url = "github:NixOS/nixpkgs/nixos-unstable";
+
+    # for rust via nix
+    crane.url = "github:ipetkov/crane";
     fenix = {
-      url = "github:nix-community/fenix?ref=main";
+      url = "github:nix-community/fenix";
       inputs.nixpkgs.follows = "nixpkgs";
     };
+
+    # for vuln checks
+    advisory-db = {
+      url = "github:rustsec/advisory-db";
+      flake = false;
+    };
+
+    treefmt-nix = {
+      url = "github:numtide/treefmt-nix";
+      inputs.nixpkgs.follows = "nixpkgs";
+    };
+
+    # for default.nix
     flake-compat = {
       url = "github:edolstra/flake-compat?ref=master";
       flake = false;
     };
-    flake-utils.url = "github:numtide/flake-utils?ref=main";
-    nix-filter.url = "github:numtide/nix-filter?ref=main";
-    nixpkgs.url = "github:NixOS/nixpkgs?ref=nixpkgs-unstable";
+
   };
 
   outputs =
-    inputs:
-    inputs.flake-utils.lib.eachDefaultSystem (
-      system:
-      let
-        pkgsHost = import inputs.nixpkgs {
-          inherit system;
-        };
-
-        fnx = inputs.fenix.packages.${system};
-        # The Rust toolchain to use
-        toolchain = fnx.combine [
-          (fnx.fromToolchainFile {
-            file = ./rust-toolchain.toml;
-
-            # See also `rust-toolchain.toml`
-            sha256 = "sha256-+9FmLhAOezBZCOziO0Qct1NOrfpjNsXxc/8I0c7BdKE=";
-          })
-          fnx.complete.rustfmt
-        ];
-
-        mkScope =
-          pkgs:
-          pkgs.lib.makeScope pkgs.newScope (self: {
-            inherit pkgs inputs;
-            craneLib = (inputs.crane.mkLib pkgs).overrideToolchain (_: toolchain);
-            main = self.callPackage ./pkg/nix/pkgs/main { };
-            liburing = pkgs.liburing.overrideAttrs {
-              # Tests weren't building
-              outputs = [
-                "out"
-                "dev"
-                "man"
-              ];
-              buildFlags = [ "library" ];
-            };
-            rocksdb =
-              (pkgs.rocksdb_9_10.override {
-                # Override the liburing input for the build with our own so
-                # we have it built with the library flag
-                inherit (self) liburing;
-              }).overrideAttrs
-                (old: {
-                  src = pkgsHost.fetchFromGitea {
-                    domain = "forgejo.ellis.link";
-                    owner = "continuwuation";
-                    repo = "rocksdb";
-                    rev = "10.4.fb";
-                    sha256 = "sha256-/Hvy1yTH/0D5aa7bc+/uqFugCQq4InTdwlRw88vA5IY=";
-                  };
-                  version = "v10.4.fb";
-                  cmakeFlags =
-                    pkgs.lib.subtractLists [
-                      # No real reason to have snappy or zlib, no one uses this
-                      "-DWITH_SNAPPY=1"
-                      "-DZLIB=1"
-                      "-DWITH_ZLIB=1"
-                      # We don't need to use ldb or sst_dump (core_tools)
-                      "-DWITH_CORE_TOOLS=1"
-                      # We don't need to build rocksdb tests
-                      "-DWITH_TESTS=1"
-                      # We use rust-rocksdb via C interface and don't need C++ RTTI
-                      "-DUSE_RTTI=1"
-                      # This doesn't exist in RocksDB, and USE_SSE is deprecated for
-                      # PORTABLE=$(march)
-                      "-DFORCE_SSE42=1"
-                      # PORTABLE will get set in main/default.nix
-                      "-DPORTABLE=1"
-                    ] old.cmakeFlags
-                    ++ [
-                      # No real reason to have snappy, no one uses this
-                      "-DWITH_SNAPPY=0"
-                      "-DZLIB=0"
-                      "-DWITH_ZLIB=0"
-                      # We don't need to use ldb or sst_dump (core_tools)
-                      "-DWITH_CORE_TOOLS=0"
-                      # We don't need trace tools
-                      "-DWITH_TRACE_TOOLS=0"
-                      # We don't need to build rocksdb tests
-                      "-DWITH_TESTS=0"
-                      # We use rust-rocksdb via C interface and don't need C++ RTTI
-                      "-DUSE_RTTI=0"
-                    ];
-
-                  # outputs has "tools" which we don't need or use
-                  outputs = [ "out" ];
-
-                  # preInstall hooks has stuff for messing with ldb/sst_dump which we don't need or use
-                  preInstall = "";
-
-                  # We have this already at https://forgejo.ellis.link/continuwuation/rocksdb/commit/a935c0273e1ba44eacf88ce3685a9b9831486155
-                  # Unsetting this so we don't have to revert it and make this nix exclusive
-                  patches = [ ];
-
-                  postPatch = ''
-                    # Fix gcc-13 build failures due to missing <cstdint> and
-                    # <system_error> includes, fixed upstream since 8.x
-                    sed -e '1i #include <cstdint>' -i db/compaction/compaction_iteration_stats.h
-                    sed -e '1i #include <cstdint>' -i table/block_based/data_block_hash_index.h
-                    sed -e '1i #include <cstdint>' -i util/string_util.h
-                    sed -e '1i #include <cstdint>' -i include/rocksdb/utilities/checkpoint.h
-                  '';
-                });
-          });
-
-        scopeHost = mkScope pkgsHost;
-        mkCrossScope =
-          crossSystem:
-          let
-            pkgsCrossStatic =
-              (import inputs.nixpkgs {
-                inherit system;
-                crossSystem = {
-                  config = crossSystem;
-                };
-              }).pkgsStatic;
-          in
-          mkScope pkgsCrossStatic;
-
-      in
-      {
-        packages =
-          {
-            default = scopeHost.main.override {
-              disable_features = [
-                # Don't include experimental features
-                "experimental"
-                # jemalloc profiling/stats features are expensive and shouldn't
-                # be expected on non-debug builds.
-                "jemalloc_prof"
-                "jemalloc_stats"
-                # This is non-functional on nix for some reason
-                "hardened_malloc"
-                # conduwuit_mods is a development-only hot reload feature
-                "conduwuit_mods"
-              ];
-            };
-            default-debug = scopeHost.main.override {
-              profile = "dev";
-              # Debug build users expect full logs
-              disable_release_max_log_level = true;
-              disable_features = [
-                # Don't include experimental features
-                "experimental"
-                # This is non-functional on nix for some reason
-                "hardened_malloc"
-                # conduwuit_mods is a development-only hot reload feature
-                "conduwuit_mods"
-              ];
-            };
-            # Just a test profile used for things like CI and complement
-            default-test = scopeHost.main.override {
-              profile = "test";
-              disable_release_max_log_level = true;
-              disable_features = [
-                # Don't include experimental features
-                "experimental"
-                # this is non-functional on nix for some reason
-                "hardened_malloc"
-                # conduwuit_mods is a development-only hot reload feature
-                "conduwuit_mods"
-              ];
-            };
-            all-features = scopeHost.main.override {
-              all_features = true;
-              disable_features = [
-                # Don't include experimental features
-                "experimental"
-                # jemalloc profiling/stats features are expensive and shouldn't
-                # be expected on non-debug builds.
-                "jemalloc_prof"
-                "jemalloc_stats"
-                # This is non-functional on nix for some reason
-                "hardened_malloc"
-                # conduwuit_mods is a development-only hot reload feature
-                "conduwuit_mods"
-              ];
-            };
-            all-features-debug = scopeHost.main.override {
-              profile = "dev";
-              all_features = true;
-              # Debug build users expect full logs
-              disable_release_max_log_level = true;
-              disable_features = [
-                # Don't include experimental features
-                "experimental"
-                # This is non-functional on nix for some reason
-                "hardened_malloc"
-                # conduwuit_mods is a development-only hot reload feature
-                "conduwuit_mods"
-              ];
-            };
-            hmalloc = scopeHost.main.override { features = [ "hardened_malloc" ]; };
-          }
-          // builtins.listToAttrs (
-            builtins.concatLists (
-              builtins.map
-                (
-                  crossSystem:
-                  let
-                    binaryName = "static-${crossSystem}";
-                    scopeCrossStatic = mkCrossScope crossSystem;
-                  in
-                  [
-                    # An output for a statically-linked binary
-                    {
-                      name = binaryName;
-                      value = scopeCrossStatic.main;
-                    }
-
-                    # An output for a statically-linked binary with x86_64 haswell
-                    # target optimisations
-                    {
-                      name = "${binaryName}-x86_64-haswell-optimised";
-                      value = scopeCrossStatic.main.override {
-                        x86_64_haswell_target_optimised =
-                          if (crossSystem == "x86_64-linux-gnu" || crossSystem == "x86_64-linux-musl") then true else false;
-                      };
-                    }
-
-                    # An output for a statically-linked unstripped debug ("dev") binary
-                    {
-                      name = "${binaryName}-debug";
-                      value = scopeCrossStatic.main.override {
-                        profile = "dev";
-                        # debug build users expect full logs
-                        disable_release_max_log_level = true;
-                      };
-                    }
-
-                    # An output for a statically-linked unstripped debug binary with the
-                    # "test" profile (for CI usage only)
-                    {
-                      name = "${binaryName}-test";
-                      value = scopeCrossStatic.main.override {
-                        profile = "test";
-                        disable_release_max_log_level = true;
-                        disable_features = [
-                          # dont include experimental features
-                          "experimental"
-                          # this is non-functional on nix for some reason
-                          "hardened_malloc"
-                          # conduwuit_mods is a development-only hot reload feature
-                          "conduwuit_mods"
-                        ];
-                      };
-                    }
-
-                    # An output for a statically-linked binary with `--all-features`
-                    {
-                      name = "${binaryName}-all-features";
-                      value = scopeCrossStatic.main.override {
-                        all_features = true;
-                        disable_features = [
-                          # dont include experimental features
-                          "experimental"
-                          # jemalloc profiling/stats features are expensive and shouldn't
-                          # be expected on non-debug builds.
-                          "jemalloc_prof"
-                          "jemalloc_stats"
-                          # this is non-functional on nix for some reason
-                          "hardened_malloc"
-                          # conduwuit_mods is a development-only hot reload feature
-                          "conduwuit_mods"
-                        ];
-                      };
-                    }
-
-                    # An output for a statically-linked binary with `--all-features` and with x86_64 haswell
-                    # target optimisations
-                    {
-                      name = "${binaryName}-all-features-x86_64-haswell-optimised";
-                      value = scopeCrossStatic.main.override {
-                        all_features = true;
-                        disable_features = [
-                          # dont include experimental features
-                          "experimental"
-                          # jemalloc profiling/stats features are expensive and shouldn't
-                          # be expected on non-debug builds.
-                          "jemalloc_prof"
-                          "jemalloc_stats"
-                          # this is non-functional on nix for some reason
-                          "hardened_malloc"
-                          # conduwuit_mods is a development-only hot reload feature
-                          "conduwuit_mods"
-                        ];
-                        x86_64_haswell_target_optimised =
-                          if (crossSystem == "x86_64-linux-gnu" || crossSystem == "x86_64-linux-musl") then true else false;
-                      };
-                    }
-
-                    # An output for a statically-linked unstripped debug ("dev") binary with `--all-features`
-                    {
-                      name = "${binaryName}-all-features-debug";
-                      value = scopeCrossStatic.main.override {
-                        profile = "dev";
-                        all_features = true;
-                        # debug build users expect full logs
-                        disable_release_max_log_level = true;
-                        disable_features = [
-                          # dont include experimental features
-                          "experimental"
-                          # this is non-functional on nix for some reason
-                          "hardened_malloc"
-                          # conduwuit_mods is a development-only hot reload feature
-                          "conduwuit_mods"
-                        ];
-                      };
-                    }
-
-                    # An output for a statically-linked binary with hardened_malloc
-                    {
-                      name = "${binaryName}-hmalloc";
-                      value = scopeCrossStatic.main.override {
-                        features = [ "hardened_malloc" ];
-                      };
-                    }
-                  ]
-                )
-                [
-                  #"x86_64-apple-darwin"
-                  #"aarch64-apple-darwin"
-                  "x86_64-linux-gnu"
-                  "x86_64-linux-musl"
-                  "aarch64-linux-musl"
-                ]
-            )
-          );
-      }
-    );
+    inputs@{ flake-parts, ... }:
+    flake-parts.lib.mkFlake { inherit inputs; } {
+      imports = [ ./nix ];
+      systems = [
+        # good support
+        "x86_64-linux"
+        # support untested but theoretically there
+        "aarch64-linux"
+      ];
+    };
 }

nix/checks/default.nix

@@ -0,0 +1,108 @@
+{ inputs, ... }:
+{
+  perSystem =
+    {
+      self',
+      lib,
+      pkgs,
+      ...
+    }:
+    let
+      uwulib = inputs.self.uwulib.init pkgs;
+
+      rocksdbAllFeatures = self'.packages.rocksdb.override {
+        enableJemalloc = true;
+        enableLiburing = true;
+      };
+
+      commonAttrs = (uwulib.build.commonAttrs { }) // {
+        buildInputs = [
+          pkgs.liburing
+          pkgs.rust-jemalloc-sys-unprefixed
+          rocksdbAllFeatures
+        ];
+        nativeBuildInputs = [
+          pkgs.pkg-config
+          # bindgen needs the build platform's libclang. Apparently due to "splicing
+          # weirdness", pkgs.rustPlatform.bindgenHook on its own doesn't quite do the
+          # right thing here.
+          pkgs.rustPlatform.bindgenHook
+        ];
+        env = {
+          LIBCLANG_PATH = lib.makeLibraryPath [ pkgs.llvmPackages.libclang.lib ];
+          LD_LIBRARY_PATH = lib.makeLibraryPath [
+            pkgs.liburing
+            pkgs.rust-jemalloc-sys-unprefixed
+            rocksdbAllFeatures
+          ];
+        }
+        // uwulib.environment.buildPackageEnv
+        // {
+          ROCKSDB_INCLUDE_DIR = "${rocksdbAllFeatures}/include";
+          ROCKSDB_LIB_DIR = "${rocksdbAllFeatures}/lib";
+        };
+      };
+      cargoArtifacts = self'.packages.continuwuity-all-features-deps;
+    in
+    {
+      # taken from
+      #
+      # https://crane.dev/examples/quick-start.html
+      checks = {
+        continuwuity-all-features-build = self'.packages.continuwuity-all-features-bin;
+
+        continuwuity-all-features-clippy = uwulib.build.craneLibForChecks.cargoClippy (
+          commonAttrs
+          // {
+            inherit cargoArtifacts;
+            cargoClippyExtraArgs = "-- --deny warnings";
+          }
+        );
+
+        continuwuity-all-features-docs = uwulib.build.craneLibForChecks.cargoDoc (
+          commonAttrs
+          // {
+            inherit cargoArtifacts;
+            # This can be commented out or tweaked as necessary, e.g. set to
+            # `--deny rustdoc::broken-intra-doc-links` to only enforce that lint
+            env.RUSTDOCFLAGS = "--deny warnings";
+          }
+        );
+
+        # Check formatting
+        continuwuity-all-features-fmt = uwulib.build.craneLibForChecks.cargoFmt {
+          src = uwulib.build.src;
+        };
+
+        continuwuity-all-features-toml-fmt = uwulib.build.craneLibForChecks.taploFmt {
+          src = pkgs.lib.sources.sourceFilesBySuffices uwulib.build.src [ ".toml" ];
+          # taplo arguments can be further customized below as needed
+          taploExtraArgs = "--config ${inputs.self}/taplo.toml";
+        };
+
+        # Audit dependencies
+        continuwuity-all-features-audit = uwulib.build.craneLibForChecks.cargoAudit {
+          inherit (inputs) advisory-db;
+          src = uwulib.build.src;
+        };
+
+        # Audit licenses
+        continuwuity-all-features-deny = uwulib.build.craneLibForChecks.cargoDeny {
+          src = uwulib.build.src;
+        };
+
+        # Run tests with cargo-nextest
+        # Consider setting `doCheck = false` on `continuwuity-all-features` if you do not want
+        # the tests to run twice
+        continuwuity-all-features-nextest = uwulib.build.craneLibForChecks.cargoNextest (
+          commonAttrs
+          // {
+            inherit cargoArtifacts;
+            partitions = 1;
+            partitionType = "count";
+            cargoNextestPartitionsExtraArgs = "--no-tests=pass";
+          }
+        );
+      };
+    };
+}

nix/default.nix

@@ -0,0 +1,11 @@
+{
+  imports = [
+    ./checks
+    ./packages
+    ./shells
+    ./tests
+
+    ./hydra.nix
+    ./fmt.nix
+  ];
+}

nix/fmt.nix

@@ -0,0 +1,37 @@
+{ inputs, ... }:
+{
+  # load the flake module from upstream
+  imports = [ inputs.treefmt-nix.flakeModule ];
+
+  perSystem =
+    { self', lib, ... }:
+    {
+      treefmt = {
+        # repo root as project root
+        projectRoot = inputs.self;
+
+        # the formatters
+        programs = {
+          nixfmt.enable = true;
+          typos = {
+            enable = true;
+            configFile = "${inputs.self}/.typos.toml";
+          };
+          taplo = {
+            enable = true;
+            settings = lib.importTOML "${inputs.self}/taplo.toml";
+          };
+        };
+
+        settings.formatter.rustfmt = {
+          command = "${lib.getExe' self'.packages.dev-toolchain "rustfmt"}";
+          includes = [ "**/*.rs" ];
+          options = [
+            "--unstable-features"
+            "--edition=2024"
+            "--config-path=${inputs.self}/rustfmt.toml"
+          ];
+        };
+      };
+    };
+}

nix/hydra.nix

@@ -0,0 +1,9 @@
+{ inputs, ... }:
+let
+  lib = inputs.nixpkgs.lib;
+in
+{
+  flake.hydraJobs.packages = builtins.mapAttrs (
+    _name: lib.hydraJob
+  ) inputs.self.packages.x86_64-linux;
+}

nix/packages/continuwuity/default.nix

@@ -0,0 +1,60 @@
+{ inputs, ... }:
+{
+  perSystem =
+    {
+      self',
+      lib,
+      pkgs,
+      ...
+    }:
+    let
+      uwulib = inputs.self.uwulib.init pkgs;
+    in
+    {
+      packages =
+        lib.pipe
+          [
+            # this is the default variant
+            {
+              variantName = "default";
+              commonAttrsArgs.profile = "release";
+              rocksdb = self'.packages.rocksdb;
+              features = { };
+            }
+            # this is the variant with all features enabled (liburing + jemalloc)
+            {
+              variantName = "all-features";
+              commonAttrsArgs.profile = "release";
+              rocksdb = self'.packages.rocksdb.override {
+                enableJemalloc = true;
+                enableLiburing = true;
+              };
+              features = {
+                enabledFeatures = "all";
+                disabledFeatures = uwulib.features.defaultDisabledFeatures ++ [ "bindgen-static" ];
+              };
+            }
+          ]
+          [
+            (builtins.map (cfg: rec {
+              deps = {
+                name = "continuwuity-${cfg.variantName}-deps";
+                value = uwulib.build.buildDeps {
+                  features = uwulib.features.calcFeatures cfg.features;
+                  inherit (cfg) commonAttrsArgs rocksdb;
+                };
+              };
+              bin = {
+                name = "continuwuity-${cfg.variantName}-bin";
+                value = uwulib.build.buildPackage {
+                  deps = self'.packages.${deps.name};
+                  features = uwulib.features.calcFeatures cfg.features;
+                  inherit (cfg) commonAttrsArgs rocksdb;
+                };
+              };
+            }))
+            (builtins.concatMap builtins.attrValues)
+            builtins.listToAttrs
+          ];
+    };
+}

nix/packages/default.nix

@@ -0,0 +1,14 @@
+{
+  imports = [
+    ./continuwuity
+    ./rocksdb
+    ./rust.nix
+    ./uwulib
+  ];
+
+  perSystem =
+    { self', ... }:
+    {
+      packages.default = self'.packages.continuwuity-default-bin;
+    };
+}

nix/packages/rocksdb/default.nix

@@ -0,0 +1,12 @@
+{
+  perSystem =
+    {
+      pkgs,
+      ...
+    }:
+    {
+      packages = {
+        rocksdb = pkgs.callPackage ./package.nix { };
+      };
+    };
+}

nix/packages/rocksdb/package.nix

@@ -0,0 +1,88 @@
+{
+  lib,
+  stdenv,
+
+  rocksdb,
+  liburing,
+  rust-jemalloc-sys-unprefixed,
+
+  enableJemalloc ? false,
+  enableLiburing ? false,
+
+  fetchFromGitea,
+
+  ...
+}:
+let
+  notDarwin = !stdenv.hostPlatform.isDarwin;
+in
+(rocksdb.override {
+  # Override the liburing input for the build with our own so
+  # we have it built with the library flag
+  inherit liburing;
+  jemalloc = rust-jemalloc-sys-unprefixed;
+
+  # rocksdb fails to build with prefixed jemalloc, which is required on
+  # darwin due to [1]. In this case, fall back to building rocksdb with
+  # libc malloc. This should not cause conflicts, because all of the
+  # jemalloc symbols are prefixed.
+  #
+  # [1]: https://github.com/tikv/jemallocator/blob/ab0676d77e81268cd09b059260c75b38dbef2d51/jemalloc-sys/src/env.rs#L17
+  enableJemalloc = enableJemalloc && notDarwin;
+
+  # for some reason enableLiburing in nixpkgs rocksdb is default true
+  # which breaks Darwin entirely
+  enableLiburing = enableLiburing && notDarwin;
+}).overrideAttrs
+  (old: {
+    src = fetchFromGitea {
+      domain = "forgejo.ellis.link";
+      owner = "continuwuation";
+      repo = "rocksdb";
+      rev = "10.5.fb";
+      sha256 = "sha256-X4ApGLkHF9ceBtBg77dimEpu720I79ffLoyPa8JMHaU=";
+    };
+    version = "10.5.fb";
+    cmakeFlags =
+      lib.subtractLists (builtins.map (flag: lib.cmakeBool flag true) [
+        # No real reason to have snappy or zlib, no one uses this
+        "WITH_SNAPPY"
+        "ZLIB"
+        "WITH_ZLIB"
+        # We don't need to use ldb or sst_dump (core_tools)
+        "WITH_CORE_TOOLS"
+        # We don't need to build rocksdb tests
+        "WITH_TESTS"
+        # We use rust-rocksdb via C interface and don't need C++ RTTI
+        "USE_RTTI"
+        # This doesn't exist in RocksDB, and USE_SSE is deprecated for
+        # PORTABLE=$(march)
+        "FORCE_SSE42"
+      ]) old.cmakeFlags
+      ++ (builtins.map (flag: lib.cmakeBool flag false) [
+        # No real reason to have snappy, no one uses this
+        "WITH_SNAPPY"
+        "ZLIB"
+        "WITH_ZLIB"
+        # We don't need to use ldb or sst_dump (core_tools)
+        "WITH_CORE_TOOLS"
+        # We don't need trace tools
+        "WITH_TRACE_TOOLS"
+        # We don't need to build rocksdb tests
+        "WITH_TESTS"
+        # We use rust-rocksdb via C interface and don't need C++ RTTI
+        "USE_RTTI"
+      ]);
+
+    enableLiburing = enableLiburing && notDarwin;
+
+    # outputs has "tools" which we don't need or use
+    outputs = [ "out" ];
+
+    # preInstall hooks has stuff for messing with ldb/sst_dump which we don't need or use
+    preInstall = "";
+
+    # We have this already at https://forgejo.ellis.link/continuwuation/rocksdb/commit/a935c0273e1ba44eacf88ce3685a9b9831486155
+    # Unsetting `patches` so we don't have to revert it and make this nix exclusive
+    patches = [ ];
+  })

nix/packages/rust.nix

@@ -0,0 +1,32 @@
+{ inputs, ... }:
+{
+  perSystem =
+    {
+      system,
+      lib,
+      ...
+    }:
+    {
+      packages =
+        let
+          fnx = inputs.fenix.packages.${system};
+
+          stable = fnx.fromToolchainFile {
+            file = inputs.self + "/rust-toolchain.toml";
+
+            # See also `rust-toolchain.toml`
+            sha256 = "sha256-SJwZ8g0zF2WrKDVmHrVG3pD2RGoQeo24MEXnNx5FyuI=";
+          };
+        in
+        {
+          # used for building nix stuff (doesn't include rustfmt overhead)
+          build-toolchain = stable;
+          # used for dev shells
+          dev-toolchain = fnx.combine [
+            stable
+            # use the nightly rustfmt because we use nightly features
+            fnx.complete.rustfmt
+          ];
+        };
+    };
+}

nix/packages/uwulib/build.nix

@@ -0,0 +1,108 @@
+args@{ pkgs, inputs, ... }:
+let
+  inherit (pkgs) lib;
+  uwuenv = import ./environment.nix args;
+  selfpkgs = inputs.self.packages.${pkgs.stdenv.system};
+in
+rec {
+  # basic, very minimal instance of the crane library with a minimal rust toolchain
+  craneLib = (inputs.crane.mkLib pkgs).overrideToolchain (_: selfpkgs.build-toolchain);
+  # the checks require more rust toolchain components, hence we have this separate instance of the crane library
+  craneLibForChecks = (inputs.crane.mkLib pkgs).overrideToolchain (_: selfpkgs.dev-toolchain);
+
+  # meta information (name, version, etc) of the rust crate based on the Cargo.toml
+  crateInfo = craneLib.crateNameFromCargoToml { cargoToml = "${inputs.self}/Cargo.toml"; };
+
+  src =
+    let
+      # see https://crane.dev/API.html#cranelibfiltercargosources
+      #
+      # we need to keep the `web` directory which would be filtered out by the regular source filtering function
+      #
+      # https://crane.dev/API.html#cranelibcleancargosource
+      isWebTemplate = path: _type: builtins.match ".*src/web.*" path != null;
+      isRust = craneLib.filterCargoSources;
+      isNix = path: _type: builtins.match ".+/nix.*" path != null;
+      webOrRustNotNix = p: t: !(isNix p t) && (isWebTemplate p t || isRust p t);
+    in
+    lib.cleanSourceWith {
+      src = inputs.self;
+      filter = webOrRustNotNix;
+      name = "source";
+    };
+
+  # common attrs that are shared between building continuwuity's deps and the package itself
+  commonAttrs =
+    {
+      profile ? "dev",
+      ...
+    }:
+    {
+      inherit (crateInfo)
+        pname
+        version
+        ;
+      inherit src;
+
+      # this prevents unnecessary rebuilds
+      strictDeps = true;
+
+      dontStrip = profile == "dev" || profile == "test";
+      dontPatchELF = profile == "dev" || profile == "test";
+
+      doCheck = true;
+
+      nativeBuildInputs = [
+        # bindgen needs the build platform's libclang. Apparently due to "splicing
+        # weirdness", pkgs.rustPlatform.bindgenHook on its own doesn't quite do the
+        # right thing here.
+        pkgs.rustPlatform.bindgenHook
+      ];
+    };
+
+  makeRocksDBEnv =
+    { rocksdb }:
+    {
+      ROCKSDB_INCLUDE_DIR = "${rocksdb}/include";
+      ROCKSDB_LIB_DIR = "${rocksdb}/lib";
+    };
+
+  # function that builds the continuwuity dependencies derivation
+  buildDeps =
+    {
+      rocksdb,
+      features,
+      commonAttrsArgs,
+    }:
+    craneLib.buildDepsOnly (
+      (commonAttrs commonAttrsArgs)
+      // {
+        env = uwuenv.buildDepsOnlyEnv // (makeRocksDBEnv { inherit rocksdb; });
+        inherit (features) cargoExtraArgs;
+      }
+
+    );
+
+  # function that builds the continuwuity package
+  buildPackage =
+    {
+      deps,
+      rocksdb,
+      features,
+      commonAttrsArgs,
+    }:
+    let
+      rocksdbEnv = makeRocksDBEnv { inherit rocksdb; };
+    in
+    craneLib.buildPackage (
+      (commonAttrs commonAttrsArgs)
+      // {
+        cargoArtifacts = deps;
+        doCheck = true;
+        env = uwuenv.buildPackageEnv // rocksdbEnv;
+        passthru.env = uwuenv.buildPackageEnv // rocksdbEnv;
+        meta.mainProgram = crateInfo.pname;
+        inherit (features) cargoExtraArgs;
+      }
+    );
+}

nix/packages/uwulib/default.nix

@@ -0,0 +1,10 @@
+{ inputs, ... }:
+{
+  flake.uwulib = {
+    init = pkgs: {
+      features = import ./features.nix { inherit pkgs inputs; };
+      environment = import ./environment.nix { inherit pkgs inputs; };
+      build = import ./build.nix { inherit pkgs inputs; };
+    };
+  };
+}

nix/packages/uwulib/environment.nix

@@ -0,0 +1,18 @@
+args@{ pkgs, inputs, ... }:
+let
+  uwubuild = import ./build.nix args;
+in
+rec {
+  buildDepsOnlyEnv = {
+    # https://crane.dev/faq/rebuilds-bindgen.html
+    NIX_OUTPATH_USED_AS_RANDOM_SEED = "aaaaaaaaaa";
+    CARGO_PROFILE = "release";
+  }
+  // uwubuild.craneLib.mkCrossToolchainEnv (p: pkgs.clangStdenv);
+
+  buildPackageEnv = {
+    GIT_COMMIT_HASH = inputs.self.rev or inputs.self.dirtyRev or "";
+    GIT_COMMIT_HASH_SHORT = inputs.self.shortRev or inputs.self.dirtyShortRev or "";
+  }
+  // buildDepsOnlyEnv;
+}

nix/packages/uwulib/features.nix

@@ -0,0 +1,77 @@
+{ pkgs, inputs, ... }:
+let
+  inherit (pkgs) lib;
+in
+rec {
+  defaultDisabledFeatures = [
+    # dont include experimental features
+    "experimental"
+    # jemalloc profiling/stats features are expensive and shouldn't
+    # be expected on non-debug builds.
+    "jemalloc_prof"
+    "jemalloc_stats"
+    # this is non-functional on nix for some reason
+    "hardened_malloc"
+    # conduwuit_mods is a development-only hot reload feature
+    "conduwuit_mods"
+    # we don't want to enable this feature set by default but be more specific about it
+    "full"
+  ];
+  # We perform default-feature unification in nix, because some of the dependencies
+  # on the nix side depend on feature values.
+  calcFeatures =
+    {
+      tomlPath ? "${inputs.self}/src/main",
+      # either a list of feature names or a string "all" which enables all non-default features
+      enabledFeatures ? [ ],
+      disabledFeatures ? defaultDisabledFeatures,
+      default_features ? true,
+      disable_release_max_log_level ? false,
+    }:
+    let
+      # simple helper to get the contents of a Cargo.toml file in a nix format
+      getToml = path: lib.importTOML "${path}/Cargo.toml";
+
+      # get all the features except for the default features
+      allFeatures = lib.pipe tomlPath [
+        getToml
+        (manifest: manifest.features)
+        lib.attrNames
+        (lib.remove "default")
+      ];
+
+      # get just the default enabled features
+      allDefaultFeatures = lib.pipe tomlPath [
+        getToml
+        (manifest: manifest.features.default)
+      ];
+
+      # depending on the value of enabledFeatures choose just a set or all non-default features
+      #
+      # - [ list of features ] -> choose exactly the features listed
+      # - "all" -> choose all non-default features
+      additionalFeatures = if enabledFeatures == "all" then allFeatures else enabledFeatures;
+
+      # unification with default features (if enabled)
+      features = lib.unique (additionalFeatures ++ lib.optionals default_features allDefaultFeatures);
+
+      # prepare the features that are subtracted from the set
+      disabledFeatures' =
+        disabledFeatures ++ lib.optionals disable_release_max_log_level [ "release_max_log_level" ];
+
+      # construct the final feature set
+      finalFeatures = lib.subtractLists disabledFeatures' features;
+    in
+    {
+      # final feature set, useful for querying it
+      features = finalFeatures;
+
+      # crane flag with the relevant features
+      cargoExtraArgs = builtins.concatStringsSep " " [
+        "--no-default-features"
+        "--locked"
+        (lib.optionalString (finalFeatures != [ ]) "--features")
+        (builtins.concatStringsSep "," finalFeatures)
+      ];
+    };
+}

nix/shells/default.nix

@@ -0,0 +1,29 @@
+{ inputs, ... }:
+{
+  perSystem =
+    {
+      self',
+      lib,
+      pkgs,
+      ...
+    }:
+    let
+      uwulib = inputs.self.uwulib.init pkgs;
+      rocksdbAllFeatures = self'.packages.rocksdb.override {
+        enableJemalloc = true;
+        enableLiburing = true;
+      };
+    in
+    {
+      # basic nix shell containing all things necessary to build continuwuity in all flavors manually (on x86_64-linux)
+      devShells.default = uwulib.build.craneLib.devShell {
+        packages = [
+          pkgs.pkg-config
+          pkgs.liburing
+          pkgs.rust-jemalloc-sys-unprefixed
+          rocksdbAllFeatures
+        ];
+        env.LIBCLANG_PATH = lib.makeLibraryPath [ pkgs.llvmPackages.libclang.lib ];
+      };
+    };
+}

nix/tests/default.nix

@@ -0,0 +1,124 @@
+{
+  perSystem =
+    {
+      self',
+      lib,
+      pkgs,
+      ...
+    }:
+    {
+      # run some nixos tests as checks
+      checks = lib.pipe self'.packages [
+        # we take all packages (names)
+        builtins.attrNames
+        # we filter out all packages that end with `-bin` (which we are interested in for testing)
+        (builtins.filter (lib.hasSuffix "-bin"))
+        # for each of these binaries we built the basic nixos test
+        #
+        # this test was initially yoinked from
+        #
+        # https://github.com/NixOS/nixpkgs/blob/960ce26339661b1b69c6f12b9063ca51b688615f/nixos/tests/matrix/continuwuity.nix
+        (builtins.map (name: {
+          name = "test-${name}";
+          value = pkgs.testers.runNixOSTest {
+            inherit name;
+
+            nodes = {
+              continuwuity = {
+                services.matrix-continuwuity = {
+                  enable = true;
+                  package = self'.packages.${name};
+                  settings.global = {
+                    server_name = name;
+                    address = [ "0.0.0.0" ];
+                    allow_registration = true;
+                    yes_i_am_very_very_sure_i_want_an_open_registration_server_prone_to_abuse = true;
+                  };
+                  extraEnvironment.RUST_BACKTRACE = "yes";
+                };
+                networking.firewall.allowedTCPPorts = [ 6167 ];
+              };
+              client =
+                { pkgs, ... }:
+                {
+                  environment.systemPackages = [
+                    (pkgs.writers.writePython3Bin "do_test" { libraries = [ pkgs.python3Packages.matrix-nio ]; } ''
+                      import asyncio
+                      import nio
+
+
+                      async def main() -> None:
+                          # Connect to continuwuity
+                          client = nio.AsyncClient("http://continuwuity:6167", "alice")
+
+                          # Register as user alice
+                          response = await client.register("alice", "my-secret-password")
+
+                          # Log in as user alice
+                          response = await client.login("my-secret-password")
+
+                          # Create a new room
+                          response = await client.room_create(federate=False)
+                          print("Matrix room create response:", response)
+                          assert isinstance(response, nio.RoomCreateResponse)
+                          room_id = response.room_id
+
+                          # Join the room
+                          response = await client.join(room_id)
+                          print("Matrix join response:", response)
+                          assert isinstance(response, nio.JoinResponse)
+
+                          # Send a message to the room
+                          response = await client.room_send(
+                              room_id=room_id,
+                              message_type="m.room.message",
+                              content={
+                                  "msgtype": "m.text",
+                                  "body": "Hello continuwuity!"
+                              }
+                          )
+                          print("Matrix room send response:", response)
+                          assert isinstance(response, nio.RoomSendResponse)
+
+                          # Sync responses
+                          response = await client.sync(timeout=30000)
+                          print("Matrix sync response:", response)
+                          assert isinstance(response, nio.SyncResponse)
+
+                          # Check the message was received by continuwuity
+                          last_message = response.rooms.join[room_id].timeline.events[-1].body
+                          assert last_message == "Hello continuwuity!"
+
+                          # Leave the room
+                          response = await client.room_leave(room_id)
+                          print("Matrix room leave response:", response)
+                          assert isinstance(response, nio.RoomLeaveResponse)
+
+                          # Close the client
+                          await client.close()
+
+
+                      if __name__ == "__main__":
+                          asyncio.run(main())
+                    '')
+                  ];
+                };
+            };
+
+            testScript = ''
+              start_all()
+
+              with subtest("start continuwuity"):
+                    continuwuity.wait_for_unit("continuwuity.service")
+                    continuwuity.wait_for_open_port(6167)
+
+              with subtest("ensure messages can be exchanged"):
+                    client.succeed("do_test >&2")
+            '';
+
+          };
+        }))
+        builtins.listToAttrs
+      ];
+    };
+}

pkg/conduwuit.service

@@ -12,13 +12,14 @@ Group=conduwuit
 Type=notify-reload
 ReloadSignal=SIGUSR1
 
-Environment="CONTINUWUITY_CONFIG=/etc/conduwuit/conduwuit.toml"
-
 Environment="CONTINUWUITY_LOG_TO_JOURNALD=true"
 Environment="CONTINUWUITY_JOURNALD_IDENTIFIER=%N"
-Environment="CONTINUWUITY_DATABASE_PATH=/var/lib/conduwuit"
+Environment="CONTINUWUITY_DATABASE_PATH=%S/conduwuit"
+Environment="CONTINUWUITY_CONFIG_RELOAD_SIGNAL=true"
 
-ExecStart=/usr/bin/conduwuit
+LoadCredential=conduwuit.toml:/etc/conduwuit/conduwuit.toml
+
+ExecStart=/usr/bin/conduwuit --config ${CREDENTIALS_DIRECTORY}/conduwuit.toml
 
 AmbientCapabilities=
 CapabilityBoundingSet=
@@ -52,8 +53,9 @@ SystemCallFilter=@system-service @resources
 SystemCallFilter=~@clock @debug @module @mount @reboot @swap @cpu-emulation @obsolete @timer @chown @setuid @privileged @keyring @ipc
 SystemCallErrorNumber=EPERM
 
+# ConfigurationDirectory isn't specified here because it's created by
+# the distro's package manager.
 StateDirectory=conduwuit
-ConfigurationDirectory=conduwuit
 RuntimeDirectory=conduwuit
 RuntimeDirectoryMode=0750
 
@@ -61,7 +63,7 @@ Restart=on-failure
 RestartSec=5
 
 TimeoutStopSec=4m
-TimeoutStartSec=4m
+TimeoutStartSec=10m
 
 StartLimitInterval=1m
 StartLimitBurst=5

pkg/fedora/continuwuity.spec.rpkg

@@ -1,6 +1,5 @@
-# This should be run using rpkg-util: https://docs.pagure.org/rpkg-util
+# This should be run using rpkg: https://docs.pagure.org/rpkg
 # it requires Internet access and is not suitable for Fedora main repos
-# TODO: rpkg-util is no longer maintained, find a replacement
 
 Name:           continuwuity
 Version:        {{{ git_repo_version }}}
@@ -52,7 +51,7 @@ find .cargo/registry/ -executable -name "*.rs" -exec chmod -x {} +
 %install
 install -Dpm0755 target/rpm/conduwuit -t %{buildroot}%{_bindir}
 install -Dpm0644 pkg/conduwuit.service -t %{buildroot}%{_unitdir}
-install -Dpm0644 conduwuit-example.toml %{buildroot}%{_sysconfdir}/conduwuit/conduwuit.toml
+install -Dpm0600 conduwuit-example.toml %{buildroot}%{_sysconfdir}/conduwuit/conduwuit.toml
 
 %files
 %license LICENSE
@@ -61,7 +60,7 @@ install -Dpm0644 conduwuit-example.toml %{buildroot}%{_sysconfdir}/conduwuit/con
 %doc CONTRIBUTING.md
 %doc README.md
 %doc SECURITY.md
-%config %{_sysconfdir}/conduwuit/conduwuit.toml
+%config(noreplace) %{_sysconfdir}/conduwuit/conduwuit.toml
 
 %{_bindir}/conduwuit
 %{_unitdir}/conduwuit.service

pkg/nix/pkgs/main/cross-compilation-env.nix

@@ -1,83 +0,0 @@
-{ lib
-, pkgsBuildHost
-, rust
-, stdenv
-}:
-
-lib.optionalAttrs stdenv.hostPlatform.isStatic
-  {
-    ROCKSDB_STATIC = "";
-  }
-//
-{
-  CARGO_BUILD_RUSTFLAGS =
-    lib.concatStringsSep
-      " "
-      (lib.optionals
-        stdenv.hostPlatform.isStatic
-        [ "-C" "relocation-model=static" ]
-      ++ lib.optionals
-        (stdenv.buildPlatform.config != stdenv.hostPlatform.config)
-        [
-          "-l"
-          "c"
-
-          "-l"
-          "stdc++"
-
-          "-L"
-          "${stdenv.cc.cc.lib}/${stdenv.hostPlatform.config}/lib"
-        ]
-      );
-}
-
-  # What follows is stolen from [here][0]. Its purpose is to properly
-  # configure compilers and linkers for various stages of the build, and
-  # even covers the case of build scripts that need native code compiled and
-  # run on the build platform (I think).
-  #
-  # [0]: https://github.com/NixOS/nixpkgs/blob/nixpkgs-unstable/pkgs/build-support/rust/lib/default.nix#L48-L68
-  //
-(
-  let
-    inherit (rust.lib) envVars;
-  in
-  lib.optionalAttrs
-    (stdenv.targetPlatform.rust.rustcTarget
-    != stdenv.hostPlatform.rust.rustcTarget)
-    (
-      let
-        inherit (stdenv.targetPlatform.rust) cargoEnvVarTarget;
-      in
-      {
-        "CC_${cargoEnvVarTarget}" = envVars.ccForTarget;
-        "CXX_${cargoEnvVarTarget}" = envVars.cxxForTarget;
-        "CARGO_TARGET_${cargoEnvVarTarget}_LINKER" = envVars.ccForTarget;
-      }
-    )
-  //
-  (
-    let
-      inherit (stdenv.hostPlatform.rust) cargoEnvVarTarget rustcTarget;
-    in
-    {
-      "CC_${cargoEnvVarTarget}" = envVars.ccForHost;
-      "CXX_${cargoEnvVarTarget}" = envVars.cxxForHost;
-      "CARGO_TARGET_${cargoEnvVarTarget}_LINKER" = envVars.ccForHost;
-      CARGO_BUILD_TARGET = rustcTarget;
-    }
-  )
-  //
-  (
-    let
-      inherit (stdenv.buildPlatform.rust) cargoEnvVarTarget;
-    in
-    {
-      "CC_${cargoEnvVarTarget}" = envVars.ccForBuild;
-      "CXX_${cargoEnvVarTarget}" = envVars.cxxForBuild;
-      "CARGO_TARGET_${cargoEnvVarTarget}_LINKER" = envVars.ccForBuild;
-      HOST_CC = "${pkgsBuildHost.stdenv.cc}/bin/cc";
-      HOST_CXX = "${pkgsBuildHost.stdenv.cc}/bin/c++";
-    }
-  )
-)

pkg/nix/pkgs/main/default.nix

@@ -1,224 +0,0 @@
-# Dependencies (keep sorted)
-{ craneLib
-, inputs
-, jq
-, lib
-, libiconv
-, liburing
-, pkgsBuildHost
-, rocksdb
-, removeReferencesTo
-, rust
-, rust-jemalloc-sys
-, stdenv
-
-  # Options (keep sorted)
-, all_features ? false
-, default_features ? true
-  # default list of disabled features
-, disable_features ? [
-    # dont include experimental features
-    "experimental"
-    # jemalloc profiling/stats features are expensive and shouldn't
-    # be expected on non-debug builds.
-    "jemalloc_prof"
-    "jemalloc_stats"
-    # this is non-functional on nix for some reason
-    "hardened_malloc"
-    # conduwuit_mods is a development-only hot reload feature
-    "conduwuit_mods"
-  ]
-, disable_release_max_log_level ? false
-, features ? [ ]
-, profile ? "release"
-  # rocksdb compiled with -march=haswell and target-cpu=haswell rustflag
-  # haswell is pretty much any x86 cpu made in the last 12 years, and
-  # supports modern CPU extensions that rocksdb can make use of.
-  # disable if trying to make a portable x86_64 build for very old hardware
-, x86_64_haswell_target_optimised ? false
-}:
-
-let
-  # We perform default-feature unification in nix, because some of the dependencies
-  # on the nix side depend on feature values.
-  crateFeatures = path:
-    let manifest = lib.importTOML "${path}/Cargo.toml"; in
-    lib.remove "default" (lib.attrNames manifest.features);
-  crateDefaultFeatures = path:
-    (lib.importTOML "${path}/Cargo.toml").features.default;
-  allDefaultFeatures = crateDefaultFeatures "${inputs.self}/src/main";
-  allFeatures = crateFeatures "${inputs.self}/src/main";
-  features' = lib.unique
-    (features ++
-      lib.optionals default_features allDefaultFeatures ++
-      lib.optionals all_features allFeatures);
-  disable_features' = disable_features ++ lib.optionals disable_release_max_log_level [ "release_max_log_level" ];
-  features'' = lib.subtractLists disable_features' features';
-
-  featureEnabled = feature: builtins.elem feature features'';
-
-  enableLiburing = featureEnabled "io_uring" && !stdenv.hostPlatform.isDarwin;
-
-  # This derivation will set the JEMALLOC_OVERRIDE variable, causing the
-  # tikv-jemalloc-sys crate to use the nixpkgs jemalloc instead of building it's
-  # own. In order for this to work, we need to set flags on the build that match
-  # whatever flags tikv-jemalloc-sys was going to use. These are dependent on
-  # which features we enable in tikv-jemalloc-sys.
-  rust-jemalloc-sys' = (rust-jemalloc-sys.override {
-    # tikv-jemalloc-sys/unprefixed_malloc_on_supported_platforms feature
-    unprefixed = true;
-  }).overrideAttrs (old: {
-    configureFlags = old.configureFlags ++
-      # we dont need docs
-      [ "--disable-doc" ] ++
-      # we dont need cxx/C++ integration
-      [ "--disable-cxx" ] ++
-      # tikv-jemalloc-sys/profiling feature
-      lib.optional (featureEnabled "jemalloc_prof") "--enable-prof" ++
-      # tikv-jemalloc-sys/stats feature
-      (if (featureEnabled "jemalloc_stats") then [ "--enable-stats" ] else [ "--disable-stats" ]);
-  });
-
-  buildDepsOnlyEnv =
-    let
-      rocksdb' = (rocksdb.override {
-        jemalloc = lib.optional (featureEnabled "jemalloc") rust-jemalloc-sys';
-        # rocksdb fails to build with prefixed jemalloc, which is required on
-        # darwin due to [1]. In this case, fall back to building rocksdb with
-        # libc malloc. This should not cause conflicts, because all of the
-        # jemalloc symbols are prefixed.
-        #
-        # [1]: https://github.com/tikv/jemallocator/blob/ab0676d77e81268cd09b059260c75b38dbef2d51/jemalloc-sys/src/env.rs#L17
-        enableJemalloc = featureEnabled "jemalloc" && !stdenv.hostPlatform.isDarwin;
-
-        # for some reason enableLiburing in nixpkgs rocksdb is default true
-        # which breaks Darwin entirely
-        inherit enableLiburing;
-      }).overrideAttrs (old: {
-        inherit enableLiburing;
-        cmakeFlags = (if x86_64_haswell_target_optimised then
-          (lib.subtractLists [
-            # dont make a portable build if x86_64_haswell_target_optimised is enabled
-            "-DPORTABLE=1"
-          ]
-            old.cmakeFlags
-          ++ [ "-DPORTABLE=haswell" ]) else [ "-DPORTABLE=1" ]
-        )
-        ++ old.cmakeFlags;
-
-        # outputs has "tools" which we dont need or use
-        outputs = [ "out" ];
-
-        # preInstall hooks has stuff for messing with ldb/sst_dump which we dont need or use
-        preInstall = "";
-      });
-    in
-    {
-      # https://crane.dev/faq/rebuilds-bindgen.html
-      NIX_OUTPATH_USED_AS_RANDOM_SEED = "aaaaaaaaaa";
-
-      CARGO_PROFILE = profile;
-      ROCKSDB_INCLUDE_DIR = "${rocksdb'}/include";
-      ROCKSDB_LIB_DIR = "${rocksdb'}/lib";
-    }
-    //
-    (import ./cross-compilation-env.nix {
-      # Keep sorted
-      inherit
-        lib
-        pkgsBuildHost
-        rust
-        stdenv;
-    });
-
-  buildPackageEnv = {
-    GIT_COMMIT_HASH = inputs.self.rev or inputs.self.dirtyRev or "";
-    GIT_COMMIT_HASH_SHORT = inputs.self.shortRev or inputs.self.dirtyShortRev or "";
-  } // buildDepsOnlyEnv // {
-    # Only needed in static stdenv because these are transitive dependencies of rocksdb
-    CARGO_BUILD_RUSTFLAGS = buildDepsOnlyEnv.CARGO_BUILD_RUSTFLAGS
-      + lib.optionalString (enableLiburing && stdenv.hostPlatform.isStatic)
-      " -L${lib.getLib liburing}/lib -luring"
-      + lib.optionalString x86_64_haswell_target_optimised
-      " -Ctarget-cpu=haswell";
-  };
-
-
-
-  commonAttrs = {
-    inherit
-      (craneLib.crateNameFromCargoToml {
-        cargoToml = "${inputs.self}/Cargo.toml";
-      })
-      pname
-      version;
-
-    src = let filter = inputs.nix-filter.lib; in filter {
-      root = inputs.self;
-
-      # Keep sorted
-      include = [
-        ".cargo"
-        "Cargo.lock"
-        "Cargo.toml"
-        "src"
-        "xtask"
-      ];
-    };
-
-    doCheck = true;
-
-    cargoExtraArgs = "--no-default-features --locked "
-      + lib.optionalString
-      (features'' != [ ])
-      "--features " + (builtins.concatStringsSep "," features'');
-
-    dontStrip = profile == "dev" || profile == "test";
-    dontPatchELF = profile == "dev" || profile == "test";
-
-    buildInputs = lib.optional (featureEnabled "jemalloc") rust-jemalloc-sys'
-      # needed to build Rust applications on macOS
-      ++ lib.optionals stdenv.hostPlatform.isDarwin [
-      # https://github.com/NixOS/nixpkgs/issues/206242
-      # ld: library not found for -liconv
-      libiconv
-      # https://stackoverflow.com/questions/69869574/properly-adding-darwin-apple-sdk-to-a-nix-shell
-      # https://discourse.nixos.org/t/compile-a-rust-binary-on-macos-dbcrossbar/8612
-      pkgsBuildHost.darwin.apple_sdk.frameworks.Security
-    ];
-
-    nativeBuildInputs = [
-      # bindgen needs the build platform's libclang. Apparently due to "splicing
-      # weirdness", pkgs.rustPlatform.bindgenHook on its own doesn't quite do the
-      # right thing here.
-      pkgsBuildHost.rustPlatform.bindgenHook
-
-      # We don't actually depend on `jq`, but crane's `buildPackage` does, but
-      # its `buildDepsOnly` doesn't. This causes those two derivations to have
-      # differing values for `NIX_CFLAGS_COMPILE`, which contributes to spurious
-      # rebuilds of bindgen and its depedents.
-      jq
-    ];
-  };
-in
-
-craneLib.buildPackage (commonAttrs // {
-  cargoArtifacts = craneLib.buildDepsOnly (commonAttrs // {
-    env = buildDepsOnlyEnv;
-  });
-
-  doCheck = true;
-
-  cargoExtraArgs = "--no-default-features --locked "
-    + lib.optionalString
-    (features'' != [ ])
-    "--features " + (builtins.concatStringsSep "," features'');
-
-  env = buildPackageEnv;
-
-  passthru = {
-    env = buildPackageEnv;
-  };
-
-  meta.mainProgram = commonAttrs.pname;
-})

renovate.json

@@ -1,22 +1,28 @@
 {
     "$schema": "https://docs.renovatebot.com/renovate-schema.json",
-    "extends": ["config:recommended"],
+    "extends": ["config:recommended", "replacements:all"],
+    "osvVulnerabilityAlerts": true,
     "lockFileMaintenance": {
         "enabled": true,
         "schedule": ["at any time"]
     },
+    "platformAutomerge": true,
     "nix": {
         "enabled": true
     },
+    "pre-commit": {
+        "enabled": true
+    },
     "labels": ["Dependencies", "Dependencies/Renovate"],
     "ignoreDeps": [
         "tikv-jemallocator",
         "tikv-jemalloc-sys",
         "tikv-jemalloc-ctl",
-        "opentelemetry",
-        "opentelemetry_sdk",
-        "opentelemetry-jaeger",
-        "tracing-opentelemetry"
+        "rustyline-async",
+        "event-listener",
+        "async-channel",
+        "core_affinity",
+        "hyper-util"
     ],
     "github-actions": {
         "enabled": true,
@@ -29,10 +35,15 @@
     },
     "packageRules": [
         {
-            "description": "Batch minor and patch GitHub Actions updates",
-            "matchManagers": ["github-actions"],
-            "matchUpdateTypes": ["minor", "patch"],
-            "groupName": "github-actions-non-major"
+            "description": "Batch patch-level Rust dependency updates",
+            "matchManagers": ["cargo"],
+            "matchUpdateTypes": ["patch"],
+            "groupName": "rust-patch-updates"
+        },
+        {
+            "description": "Limit concurrent Cargo PRs",
+            "matchManagers": ["cargo"],
+            "prConcurrentLimit": 5
         },
         {
             "description": "Group Rust toolchain updates into a single PR",
@@ -41,19 +52,37 @@
             "groupName": "rust-toolchain"
         },
         {
-            "description": "Group lockfile updates into a single PR",
-            "matchUpdateTypes": ["lockFileMaintenance"],
-            "groupName": "lockfile-maintenance"
+            "description": "Batch minor and patch GitHub Actions updates",
+            "matchManagers": ["github-actions"],
+            "matchUpdateTypes": ["minor", "patch"],
+            "groupName": "github-actions-non-major"
         },
         {
-            "description": "Batch patch-level Rust dependency updates",
-            "matchManagers": ["cargo"],
-            "matchUpdateTypes": ["patch"],
-            "groupName": "rust-patch-updates"
+            "description": "Pin forgejo artifact actions to prevent breaking changes",
+            "matchManagers": ["github-actions"],
+            "matchPackageNames": ["forgejo/upload-artifact", "forgejo/download-artifact"],
+            "enabled": false
         },
         {
-            "matchManagers": ["cargo"],
-            "prConcurrentLimit": 5
+            "description": "Auto-merge renovatebot docker image updates",
+            "matchDatasources": ["docker"],
+            "matchPackageNames": ["ghcr.io/renovatebot/renovate"],
+            "automerge": true,
+            "automergeStrategy": "fast-forward",
+            "extends": ["schedule:earlyMondays"]
         }
+    ],
+    "customManagers": [
+       {
+         "customType": "regex",
+         "description": "Update _VERSION variables in Dockerfiles",
+         "managerFilePatterns": [
+           "/(^|/)([Dd]ocker|[Cc]ontainer)file[^/]*$/",
+           "/(^|/|\\.)([Dd]ocker|[Cc]ontainer)file$/"
+         ],
+         "matchStrings": [
+           "# renovate: datasource=(?<datasource>[a-zA-Z0-9-._]+?) depName=(?<depName>[^\\s]+?)(?: (lookupName|packageName)=(?<packageName>[^\\s]+?))?(?: versioning=(?<versioning>[^\\s]+?))?(?: extractVersion=(?<extractVersion>[^\\s]+?))?(?: registryUrl=(?<registryUrl>[^\\s]+?))?\\s+(?:ENV\\s+|ARG\\s+)?[A-Za-z0-9_]+?_VERSION[ =][\"']?(?<currentValue>.+?)[\"']?\\s+(?:(?:ENV\\s+|ARG\\s+)?[A-Za-z0-9_]+?_CHECKSUM[ =][\"']?(?<currentDigest>.+?)[\"']?\\s)?"
+         ]
+       }
     ]
 }

rust-toolchain.toml

@@ -10,7 +10,7 @@
 
 [toolchain]
 profile = "minimal"
-channel = "1.89.0"
+channel = "1.90.0"
 components = [
     # For rust-analyzer
     "rust-src",

src/admin/Cargo.toml

@@ -85,7 +85,7 @@ futures.workspace = true
 log.workspace = true
 ruma.workspace = true
 serde_json.workspace = true
-serde_yaml.workspace = true
+serde-saphyr.workspace = true
 tokio.workspace = true
 tracing-subscriber.workspace = true
 tracing.workspace = true

src/admin/appservice/commands.rs

@@ -16,7 +16,7 @@ pub(super) async fn register(&self) -> Result {
 
 	let range = 1..checked!(body_len - 1)?;
 	let appservice_config_body = body[range].join("\n");
-	let parsed_config = serde_yaml::from_str(&appservice_config_body);
+	let parsed_config = serde_saphyr::from_str(&appservice_config_body);
 	match parsed_config {
 		| Err(e) => return Err!("Could not parse appservice config as YAML: {e}"),
 		| Ok(registration) => match self
@@ -57,7 +57,7 @@ pub(super) async fn show_appservice_config(&self, appservice_identifier: String)
 	{
 		| None => return Err!("Appservice does not exist."),
 		| Some(config) => {
-			let config_str = serde_yaml::to_string(&config)?;
+			let config_str = serde_saphyr::to_string(&config)?;
 			write!(self, "Config for {appservice_identifier}:\n\n```yaml\n{config_str}\n```")
 		},
 	}

src/admin/debug/commands.rs

@@ -632,6 +632,7 @@ pub(super) async fn force_set_room_state_from_server(
 			.add_pdu_outlier(&event_id, &value);
 	}
 
+	info!("Resolving new room state");
 	let new_room_state = self
 		.services
 		.rooms
@@ -639,7 +640,7 @@ pub(super) async fn force_set_room_state_from_server(
 		.resolve_state(&room_id, &room_version, state)
 		.await?;
 
-	info!("Forcing new room state");
+	info!("Compressing new room state");
 	let HashSetCompressStateEvent {
 		shortstatehash: short_state_hash,
 		added,
@@ -653,6 +654,7 @@ pub(super) async fn force_set_room_state_from_server(
 
 	let state_lock = self.services.rooms.state.mutex.lock(&*room_id).await;
 
+	info!("Forcing new room state");
 	self.services
 		.rooms
 		.state

src/admin/media/commands.rs

@@ -2,7 +2,8 @@
 
 use conduwuit::{
 	Err, Result, debug, debug_info, debug_warn, error, info, trace,
-	utils::time::parse_timepoint_ago, warn,
+	utils::time::{TimeDirection, parse_timepoint_ago},
+	warn,
 };
 use conduwuit_service::media::Dim;
 use ruma::{Mxc, OwnedEventId, OwnedMxcUri, OwnedServerName};
@@ -235,14 +236,19 @@ pub(super) async fn delete_past_remote_media(
 	}
 	assert!(!(before && after), "--before and --after should not be specified together");
 
-	let duration = parse_timepoint_ago(&duration)?;
+	let direction = if after {
+		TimeDirection::After
+	} else {
+		TimeDirection::Before
+	};
+
+	let time_boundary = parse_timepoint_ago(&duration)?;
 	let deleted_count = self
 		.services
 		.media
-		.delete_all_remote_media_at_after_time(
-			duration,
-			before,
-			after,
+		.delete_all_media_within_timeframe(
+			time_boundary,
+			direction,
 			yes_i_want_to_delete_local_media,
 		)
 		.await?;
@@ -360,7 +366,7 @@ pub(super) async fn get_remote_thumbnail(
 ) -> Result {
 	let mxc: Mxc<'_> = mxc.as_str().try_into()?;
 	let timeout = Duration::from_millis(timeout.into());
-	let dim = Dim::new(width, height, None, None);
+	let dim = Dim::new(width, height, None);
 	let mut result = self
 		.services
 		.media

src/admin/media/mod.rs

@@ -27,12 +27,24 @@ pub enum MediaCommand {
 	///   filesystem. This will always ignore errors.
 	DeleteList,
 
-	/// - Deletes all remote (and optionally local) media created before or
-	///   after [duration] time using filesystem metadata first created at date,
-	///   or fallback to last modified date. This will always ignore errors by
-	///   default.
+	/// Deletes all remote (and optionally local) media created before/after
+	/// [duration] ago, using filesystem metadata first created at date, or
+	/// fallback to last modified date. This will always ignore errors by
+	/// default.
+	///
+	/// * Examples:
+	///   * Delete all remote media older than a year:
+	///
+	///     `!admin media delete-past-remote-media -b 1y`
+	///
+	///   * Delete all remote and local media from 3 days ago, up until now:
+	///
+	///     `!admin media delete-past-remote-media -a 3d
+	/// --yes-i-want-to-delete-local-media`
+	#[command(verbatim_doc_comment)]
 	DeletePastRemoteMedia {
-		/// - The relative time (e.g. 30s, 5m, 7d) within which to search
+		/// - The relative time (e.g. 30s, 5m, 7d) from now within which to
+		///   search
 		duration: String,
 
 		/// - Only delete media created before [duration] ago

src/admin/query/account_data.rs

@@ -41,7 +41,7 @@ async fn changes_since(
 	let results: Vec<_> = self
 		.services
 		.account_data
-		.changes_since(room_id.as_deref(), &user_id, since, None)
+		.changes_since(room_id.as_deref(), &user_id, Some(since), None)
 		.collect()
 		.await;
 	let query_time = timer.elapsed();

src/admin/user/commands.rs

@@ -179,7 +179,11 @@ pub(super) async fn create_user(&self, username: String, password: Option<String
 			.await
 			.is_ok_and(is_equal_to!(1))
 		{
-			self.services.admin.make_user_admin(&user_id).await?;
+			self.services
+				.admin
+				.make_user_admin(&user_id)
+				.boxed()
+				.await?;
 			warn!("Granting {user_id} admin privileges as the first user");
 		}
 	} else {
@@ -217,7 +221,9 @@ pub(super) async fn deactivate(&self, no_leave_rooms: bool, user_id: String) ->
 			.collect()
 			.await;
 
-		full_user_deactivate(self.services, &user_id, &all_joined_rooms).await?;
+		full_user_deactivate(self.services, &user_id, &all_joined_rooms)
+			.boxed()
+			.await?;
 		update_displayname(self.services, &user_id, None, &all_joined_rooms).await;
 		update_avatar_url(self.services, &user_id, None, None, &all_joined_rooms).await;
 		leave_all_rooms(self.services, &user_id).await;
@@ -376,7 +382,9 @@ pub(super) async fn deactivate_all(&self, no_leave_rooms: bool, force: bool) ->
 						.collect()
 						.await;
 
-					full_user_deactivate(self.services, &user_id, &all_joined_rooms).await?;
+					full_user_deactivate(self.services, &user_id, &all_joined_rooms)
+						.boxed()
+						.await?;
 					update_displayname(self.services, &user_id, None, &all_joined_rooms).await;
 					update_avatar_url(self.services, &user_id, None, None, &all_joined_rooms)
 						.await;
@@ -756,7 +764,7 @@ pub(super) async fn force_demote(&self, user_id: String, room_id: OwnedRoomOrAli
 		.build_and_append_pdu(
 			PduBuilder::state(String::new(), &power_levels_content),
 			&user_id,
-			&room_id,
+			Some(&room_id),
 			&state_lock,
 		)
 		.await?;
@@ -776,7 +784,11 @@ pub(super) async fn make_user_admin(&self, user_id: String) -> Result {
 		"Parsed user_id must be a local user"
 	);
 
-	self.services.admin.make_user_admin(&user_id).await?;
+	self.services
+		.admin
+		.make_user_admin(&user_id)
+		.boxed()
+		.await?;
 
 	self.write_str(&format!("{user_id} has been granted admin privileges.",))
 		.await
@@ -901,7 +913,13 @@ pub(super) async fn redact_event(&self, event_id: OwnedEventId) -> Result {
 	);
 
 	let redaction_event_id = {
-		let state_lock = self.services.rooms.state.mutex.lock(event.room_id()).await;
+		let state_lock = self
+			.services
+			.rooms
+			.state
+			.mutex
+			.lock(&event.room_id_or_hash())
+			.await;
 
 		self.services
 			.rooms
@@ -915,7 +933,7 @@ pub(super) async fn redact_event(&self, event_id: OwnedEventId) -> Result {
 					})
 				},
 				event.sender(),
-				event.room_id(),
+				Some(&event.room_id_or_hash()),
 				&state_lock,
 			)
 			.await?

src/api/client/account.rs

@@ -405,41 +405,36 @@ pub(crate) async fn register_route(
 		)
 		.await?;
 
-	if (!is_guest && body.inhibit_login)
+	// Generate new device id if the user didn't specify one
+	let no_device = body.inhibit_login
 		|| body
 			.appservice_info
 			.as_ref()
-			.is_some_and(|appservice| appservice.registration.device_management)
-	{
-		return Ok(register::v3::Response {
-			access_token: None,
-			user_id,
-			device_id: None,
-			refresh_token: None,
-			expires_in: None,
-		});
-	}
+			.is_some_and(|aps| aps.registration.device_management);
+	let (token, device) = if !no_device {
+		// Don't create a device for inhibited logins
+		let device_id = if is_guest { None } else { body.device_id.clone() }
+			.unwrap_or_else(|| utils::random_string(DEVICE_ID_LENGTH).into());
 
-	// Generate new device id if the user didn't specify one
-	let device_id = if is_guest { None } else { body.device_id.clone() }
-		.unwrap_or_else(|| utils::random_string(DEVICE_ID_LENGTH).into());
+		// Generate new token for the device
+		let new_token = utils::random_string(TOKEN_LENGTH);
 
-	// Generate new token for the device
-	let token = utils::random_string(TOKEN_LENGTH);
-
-	// Create device for this account
-	services
-		.users
-		.create_device(
-			&user_id,
-			&device_id,
-			&token,
-			body.initial_device_display_name.clone(),
-			Some(client.to_string()),
-		)
-		.await?;
-
-	debug_info!(%user_id, %device_id, "User account was created");
+		// Create device for this account
+		services
+			.users
+			.create_device(
+				&user_id,
+				&device_id,
+				&new_token,
+				body.initial_device_display_name.clone(),
+				Some(client.to_string()),
+			)
+			.await?;
+		debug_info!(%user_id, %device_id, "User account was created");
+		(Some(new_token), Some(device_id))
+	} else {
+		(None, None)
+	};
 
 	let device_display_name = body.initial_device_display_name.as_deref().unwrap_or("");
 
@@ -505,7 +500,7 @@ pub(crate) async fn register_route(
 				.await
 				.is_ok_and(is_equal_to!(1))
 			{
-				services.admin.make_user_admin(&user_id).await?;
+				services.admin.make_user_admin(&user_id).boxed().await?;
 				warn!("Granting {user_id} admin privileges as the first user");
 			} else if services.config.suspend_on_register {
 				// This is not an admin, suspend them.
@@ -583,9 +578,9 @@ pub(crate) async fn register_route(
 	}
 
 	Ok(register::v3::Response {
-		access_token: Some(token),
+		access_token: token,
 		user_id,
-		device_id: Some(device_id),
+		device_id: device,
 		refresh_token: None,
 		expires_in: None,
 	})
@@ -929,7 +924,7 @@ pub async fn full_user_deactivate(
 				.build_and_append_pdu(
 					PduBuilder::state(String::new(), &power_levels_content),
 					user_id,
-					room_id,
+					Some(room_id),
 					&state_lock,
 				)
 				.await

src/api/client/context.rs

@@ -69,7 +69,7 @@ pub(crate) async fn get_context_route(
 
 	let (base_id, base_pdu, visible) = try_join3(base_id, base_pdu, visible).await?;
 
-	if base_pdu.room_id != *room_id || base_pdu.event_id != *event_id {
+	if base_pdu.room_id_or_hash() != *room_id || base_pdu.event_id != *event_id {
 		return Err!(Request(NotFound("Base event not found.")));
 	}
 
@@ -130,7 +130,7 @@ pub(crate) async fn get_context_route(
 	let state_at = events_after
 		.last()
 		.map(ref_at!(1))
-		.map_or(body.event_id.as_ref(), |pdu| pdu.event_id.as_ref());
+		.map_or_else(|| body.event_id.as_ref(), |pdu| pdu.event_id.as_ref());
 
 	let state_ids = services
 		.rooms

src/api/client/keys.rs

@@ -389,7 +389,7 @@ pub(crate) async fn get_key_changes_route(
 	device_list_updates.extend(
 		services
 			.users
-			.keys_changed(sender_user, from, Some(to))
+			.keys_changed(sender_user, Some(from), Some(to))
 			.map(ToOwned::to_owned)
 			.collect::<Vec<_>>()
 			.await,
@@ -401,7 +401,7 @@ pub(crate) async fn get_key_changes_route(
 		device_list_updates.extend(
 			services
 				.users
-				.room_keys_changed(room_id, from, Some(to))
+				.room_keys_changed(room_id, Some(from), Some(to))
 				.map(|(user_id, _)| user_id)
 				.map(ToOwned::to_owned)
 				.collect::<Vec<_>>()

src/api/client/media.rs

@@ -64,10 +64,14 @@ pub(crate) async fn create_content_route(
 		media_id: &utils::random_string(MXC_LENGTH),
 	};
 
-	services
+	if let Err(e) = services
 		.media
 		.create(mxc, Some(user), Some(&content_disposition), content_type, &body.file)
-		.await?;
+		.await
+	{
+		err!("Failed to save uploaded media: {e}");
+		return Err!(Request(Unknown("Failed to save uploaded media")));
+	}
 
 	let blurhash = body.generate_blurhash.then(|| {
 		services
@@ -99,7 +103,7 @@ pub(crate) async fn get_content_thumbnail_route(
 ) -> Result<get_content_thumbnail::v1::Response> {
 	let user = body.sender_user();
 
-	let dim = Dim::from_ruma(body.width, body.height, body.method.clone(), body.animated)?;
+	let dim = Dim::from_ruma(body.width, body.height, body.method.clone())?;
 	let mxc = Mxc {
 		server_name: &body.server_name,
 		media_id: &body.media_id,

src/api/client/media_legacy.rs

@@ -322,7 +322,7 @@ pub(crate) async fn get_content_thumbnail_legacy_route(
 		media_id: &body.media_id,
 	};
 
-	let dim = Dim::from_ruma(body.width, body.height, body.method.clone(), body.animated)?;
+	let dim = Dim::from_ruma(body.width, body.height, body.method.clone())?;
 	match services.media.get_thumbnail(&mxc, &dim).await? {
 		| Some(FileMeta {
 			content,

src/api/client/membership/ban.rs

@@ -49,7 +49,7 @@ pub(crate) async fn ban_user_route(
 				..current_member_content
 			}),
 			sender_user,
-			&body.room_id,
+			Some(&body.room_id),
 			&state_lock,
 		)
 		.await?;

src/api/client/membership/invite.rs

@@ -4,11 +4,14 @@
 	Err, Result, debug_error, err, info,
 	matrix::{event::gen_event_id_canonical_json, pdu::PduBuilder},
 };
-use futures::{FutureExt, join};
+use futures::FutureExt;
 use ruma::{
 	OwnedServerName, RoomId, UserId,
 	api::{client::membership::invite_user, federation::membership::create_invite},
-	events::room::member::{MembershipState, RoomMemberEventContent},
+	events::{
+		invite_permission_config::FilterLevel,
+		room::member::{MembershipState, RoomMemberEventContent},
+	},
 };
 use service::Services;
 
@@ -47,22 +50,21 @@ pub(crate) async fn invite_user_route(
 	.await?;
 
 	match &body.recipient {
-		| invite_user::v3::InvitationRecipient::UserId { user_id } => {
-			let sender_ignored_recipient = services.users.user_is_ignored(sender_user, user_id);
-			let recipient_ignored_by_sender =
-				services.users.user_is_ignored(user_id, sender_user);
+		| invite_user::v3::InvitationRecipient::UserId { user_id: recipient_user } => {
+			let sender_filter_level = services
+				.users
+				.invite_filter_level(recipient_user, sender_user)
+				.await;
 
-			let (sender_ignored_recipient, recipient_ignored_by_sender) =
-				join!(sender_ignored_recipient, recipient_ignored_by_sender);
-
-			if sender_ignored_recipient {
+			if !matches!(sender_filter_level, FilterLevel::Allow) {
+				// drop invites if the sender has the recipient filtered
 				return Ok(invite_user::v3::Response {});
 			}
 
 			if let Ok(target_user_membership) = services
 				.rooms
 				.state_accessor
-				.get_member(&body.room_id, user_id)
+				.get_member(&body.room_id, recipient_user)
 				.await
 			{
 				if target_user_membership.membership == MembershipState::Ban {
@@ -70,16 +72,27 @@ pub(crate) async fn invite_user_route(
 				}
 			}
 
-			if recipient_ignored_by_sender {
-				// silently drop the invite to the recipient if they've been ignored by the
-				// sender, pretend it worked
-				return Ok(invite_user::v3::Response {});
+			// check for blocked invites if the recipient is a local user.
+			if services.globals.user_is_local(recipient_user) {
+				let recipient_filter_level = services
+					.users
+					.invite_filter_level(sender_user, recipient_user)
+					.await;
+
+				// ignored invites aren't handled here
+				// since the recipient's membership should still be changed to `invite`.
+				// they're filtered out in the individual /sync handlers.
+				if matches!(recipient_filter_level, FilterLevel::Block) {
+					return Err!(Request(InviteBlocked(
+						"{recipient_user} has blocked invites from you."
+					)));
+				}
 			}
 
 			invite_helper(
 				&services,
 				sender_user,
-				user_id,
+				recipient_user,
 				&body.room_id,
 				body.reason.clone(),
 				false,
@@ -98,7 +111,7 @@ pub(crate) async fn invite_user_route(
 pub(crate) async fn invite_helper(
 	services: &Services,
 	sender_user: &UserId,
-	user_id: &UserId,
+	recipient_user: &UserId,
 	room_id: &RoomId,
 	reason: Option<String>,
 	is_direct: bool,
@@ -111,12 +124,12 @@ pub(crate) async fn invite_helper(
 		return Err!(Request(Forbidden("Invites are not allowed on this server.")));
 	}
 
-	if !services.globals.user_is_local(user_id) {
+	if !services.globals.user_is_local(recipient_user) {
 		let (pdu, pdu_json, invite_room_state) = {
 			let state_lock = services.rooms.state.mutex.lock(room_id).await;
 
 			let content = RoomMemberEventContent {
-				avatar_url: services.users.avatar_url(user_id).await.ok(),
+				avatar_url: services.users.avatar_url(recipient_user).await.ok(),
 				is_direct: Some(is_direct),
 				reason,
 				..RoomMemberEventContent::new(MembershipState::Invite)
@@ -126,14 +139,14 @@ pub(crate) async fn invite_helper(
 				.rooms
 				.timeline
 				.create_hash_and_sign_event(
-					PduBuilder::state(user_id.to_string(), &content),
+					PduBuilder::state(recipient_user.to_string(), &content),
 					sender_user,
-					room_id,
+					Some(room_id),
 					&state_lock,
 				)
 				.await?;
 
-			let invite_room_state = services.rooms.state.summary_stripped(&pdu).await;
+			let invite_room_state = services.rooms.state.summary_stripped(&pdu, room_id).await;
 
 			drop(state_lock);
 
@@ -144,7 +157,7 @@ pub(crate) async fn invite_helper(
 
 		let response = services
 			.sending
-			.send_federation_request(user_id.server_name(), create_invite::v2::Request {
+			.send_federation_request(recipient_user.server_name(), create_invite::v2::Request {
 				room_id: room_id.to_owned(),
 				event_id: (*pdu.event_id).to_owned(),
 				room_version: room_version_id.clone(),
@@ -173,7 +186,7 @@ pub(crate) async fn invite_helper(
 			return Err!(Request(BadJson(warn!(
 				%pdu.event_id, %event_id,
 				"Server {} sent event with wrong event ID",
-				user_id.server_name()
+				recipient_user.server_name()
 			))));
 		}
 
@@ -213,9 +226,9 @@ pub(crate) async fn invite_helper(
 	let state_lock = services.rooms.state.mutex.lock(room_id).await;
 
 	let content = RoomMemberEventContent {
-		displayname: services.users.displayname(user_id).await.ok(),
-		avatar_url: services.users.avatar_url(user_id).await.ok(),
-		blurhash: services.users.blurhash(user_id).await.ok(),
+		displayname: services.users.displayname(recipient_user).await.ok(),
+		avatar_url: services.users.avatar_url(recipient_user).await.ok(),
+		blurhash: services.users.blurhash(recipient_user).await.ok(),
 		is_direct: Some(is_direct),
 		reason,
 		..RoomMemberEventContent::new(MembershipState::Invite)
@@ -225,9 +238,9 @@ pub(crate) async fn invite_helper(
 		.rooms
 		.timeline
 		.build_and_append_pdu(
-			PduBuilder::state(user_id.to_string(), &content),
+			PduBuilder::state(recipient_user.to_string(), &content),
 			sender_user,
-			room_id,
+			Some(room_id),
 			&state_lock,
 		)
 		.await?;

src/api/client/membership/join.rs

@@ -18,7 +18,7 @@
 	},
 	warn,
 };
-use futures::{FutureExt, StreamExt};
+use futures::{FutureExt, StreamExt, TryFutureExt};
 use ruma::{
 	CanonicalJsonObject, CanonicalJsonValue, OwnedRoomId, OwnedServerName, OwnedUserId, RoomId,
 	RoomVersionId, UserId,
@@ -313,11 +313,14 @@ pub async fn join_room_by_id_helper(
 		}
 	}
 
-	let local_join = server_in_room
-		|| servers.is_empty()
-		|| (servers.len() == 1 && services.globals.server_is_ours(&servers[0]));
+	if !server_in_room && servers.is_empty() {
+		return Err!(Request(NotFound(
+			"No servers were provided to assist in joining the room remotely, and we are not \
+			 already participating in the room."
+		)));
+	}
 
-	if local_join {
+	if server_in_room {
 		join_room_by_id_helper_local(
 			services,
 			sender_user,
@@ -556,6 +559,10 @@ async fn join_room_by_id_helper_remote(
 			services
 				.server_keys
 				.validate_and_add_event_id_no_fetch(pdu, &room_version_id)
+				.inspect_err(|e| {
+					debug_warn!("Could not validate send_join response room_state event: {e:?}");
+				})
+				.inspect(|_| debug!("Completed validating send_join response room_state event"))
 		})
 		.ready_filter_map(Result::ok)
 		.fold(HashMap::new(), |mut state, (event_id, value)| async move {
@@ -566,7 +573,6 @@ async fn join_room_by_id_helper_remote(
 					return state;
 				},
 			};
-
 			services.rooms.outlier.add_pdu_outlier(&event_id, &value);
 			if let Some(state_key) = &pdu.state_key {
 				let shortstatekey = services
@@ -577,7 +583,6 @@ async fn join_room_by_id_helper_remote(
 
 				state.insert(shortstatekey, pdu.event_id.clone());
 			}
-
 			state
 		})
 		.await;
@@ -598,6 +603,7 @@ async fn join_room_by_id_helper_remote(
 		})
 		.ready_filter_map(Result::ok)
 		.ready_for_each(|(event_id, value)| {
+			trace!(%event_id, "Adding PDU as an outlier from send_join auth_chain");
 			services.rooms.outlier.add_pdu_outlier(&event_id, &value);
 		})
 		.await;
@@ -618,6 +624,9 @@ async fn join_room_by_id_helper_remote(
 		&parsed_join_pdu,
 		None, // TODO: third party invite
 		|k, s| state_fetch(k.clone(), s.into()),
+		&state_fetch(StateEventType::RoomCreate, "".into())
+			.await
+			.expect("create event is missing from send_join auth"),
 	)
 	.await
 	.map_err(|e| err!(Request(Forbidden(warn!("Auth check failed: {e:?}")))))?;
@@ -652,7 +661,7 @@ async fn join_room_by_id_helper_remote(
 		.force_state(room_id, statehash_before_join, added, removed, &state_lock)
 		.await?;
 
-	info!("Updating joined counts for new room");
+	debug!("Updating joined counts for new room");
 	services
 		.rooms
 		.state_cache
@@ -665,7 +674,7 @@ async fn join_room_by_id_helper_remote(
 	let statehash_after_join = services
 		.rooms
 		.state
-		.append_to_state(&parsed_join_pdu)
+		.append_to_state(&parsed_join_pdu, room_id)
 		.await?;
 
 	info!("Appending new room join event");
@@ -677,6 +686,7 @@ async fn join_room_by_id_helper_remote(
 			join_event,
 			once(parsed_join_pdu.event_id.borrow()),
 			&state_lock,
+			room_id,
 		)
 		.await?;
 
@@ -732,6 +742,7 @@ async fn join_room_by_id_helper_local(
 			.iter()
 			.stream()
 			.any(|restriction_room_id| {
+				trace!("Checking if {sender_user} is joined to {restriction_room_id}");
 				services
 					.rooms
 					.state_cache
@@ -744,6 +755,7 @@ async fn join_room_by_id_helper_local(
 				.state_cache
 				.local_users_in_room(room_id)
 				.filter(|user| {
+					trace!("Checking if {user} can invite {sender_user} to {room_id}");
 					services.rooms.state_accessor.user_can_invite(
 						room_id,
 						user,
@@ -756,6 +768,7 @@ async fn join_room_by_id_helper_local(
 				.await
 				.map(ToOwned::to_owned)
 		} else {
+			trace!("No restriction rooms are joined by {sender_user}");
 			None
 		}
 	};
@@ -776,7 +789,7 @@ async fn join_room_by_id_helper_local(
 		.build_and_append_pdu(
 			PduBuilder::state(sender_user.to_string(), &content),
 			sender_user,
-			room_id,
+			Some(room_id),
 			&state_lock,
 		)
 		.await

src/api/client/membership/kick.rs

@@ -54,7 +54,7 @@ pub(crate) async fn kick_user_route(
 				..event
 			}),
 			sender_user,
-			&body.room_id,
+			Some(&body.room_id),
 			&state_lock,
 		)
 		.await?;

src/api/client/membership/knock.rs

@@ -5,7 +5,7 @@
 use conduwuit::{
 	Err, Result, debug, debug_info, debug_warn, err, info,
 	matrix::{
-		event::{Event, gen_event_id},
+		event::gen_event_id,
 		pdu::{PduBuilder, PduEvent},
 	},
 	result::FlatOk,
@@ -373,7 +373,7 @@ async fn knock_room_helper_local(
 		.build_and_append_pdu(
 			PduBuilder::state(sender_user.to_string(), &content),
 			sender_user,
-			room_id,
+			Some(room_id),
 			&state_lock,
 		)
 		.await
@@ -458,7 +458,7 @@ async fn knock_room_helper_local(
 			.await,
 	};
 
-	let send_knock_response = services
+	services
 		.sending
 		.send_federation_request(&remote_server, send_knock_request)
 		.await?;
@@ -477,20 +477,14 @@ async fn knock_room_helper_local(
 		.map_err(|e| err!(BadServerResponse("Invalid knock event PDU: {e:?}")))?;
 
 	info!("Updating membership locally to knock state with provided stripped state events");
+	// TODO: this call does not appear to do anything because `update_membership`
+	// doesn't call `mark_as_knock`. investigate further, ideally with the aim of
+	// removing this call entirely -- Ginger thinks `update_membership` should only
+	// be called from `force_state` and `append_pdu`.
 	services
 		.rooms
 		.state_cache
-		.update_membership(
-			room_id,
-			sender_user,
-			parsed_knock_pdu
-				.get_content::<RoomMemberEventContent>()
-				.expect("we just created this"),
-			sender_user,
-			Some(send_knock_response.knock_room_state),
-			None,
-			false,
-		)
+		.update_membership(room_id, sender_user, &parsed_knock_pdu, false)
 		.await?;
 
 	info!("Appending room knock event locally");
@@ -502,6 +496,7 @@ async fn knock_room_helper_local(
 			knock_event,
 			once(parsed_knock_pdu.event_id.borrow()),
 			&state_lock,
+			room_id,
 		)
 		.await?;
 
@@ -672,24 +667,15 @@ async fn knock_room_helper_remote(
 	let statehash_after_knock = services
 		.rooms
 		.state
-		.append_to_state(&parsed_knock_pdu)
+		.append_to_state(&parsed_knock_pdu, room_id)
 		.await?;
 
 	info!("Updating membership locally to knock state with provided stripped state events");
+	// TODO: see TODO on the other call to `update_membership`
 	services
 		.rooms
 		.state_cache
-		.update_membership(
-			room_id,
-			sender_user,
-			parsed_knock_pdu
-				.get_content::<RoomMemberEventContent>()
-				.expect("we just created this"),
-			sender_user,
-			Some(send_knock_response.knock_room_state),
-			None,
-			false,
-		)
+		.update_membership(room_id, sender_user, &parsed_knock_pdu, false)
 		.await?;
 
 	info!("Appending room knock event locally");
@@ -701,6 +687,7 @@ async fn knock_room_helper_remote(
 			knock_event,
 			once(parsed_knock_pdu.event_id.borrow()),
 			&state_lock,
+			room_id,
 		)
 		.await?;
 

src/api/client/membership/leave.rs

@@ -2,12 +2,12 @@
 
 use axum::extract::State;
 use conduwuit::{
-	Err, Result, debug_info, debug_warn, err,
+	Err, Pdu, Result, debug_info, debug_warn, err,
 	matrix::{event::gen_event_id, pdu::PduBuilder},
 	utils::{self, FutureBoolExt, future::ReadyEqExt},
 	warn,
 };
-use futures::{FutureExt, StreamExt, TryFutureExt, pin_mut};
+use futures::{FutureExt, StreamExt, pin_mut};
 use ruma::{
 	CanonicalJsonObject, CanonicalJsonValue, OwnedServerName, RoomId, RoomVersionId, UserId,
 	api::{
@@ -81,42 +81,9 @@ pub async fn leave_room(
 	room_id: &RoomId,
 	reason: Option<String>,
 ) -> Result {
-	let default_member_content = RoomMemberEventContent {
-		membership: MembershipState::Leave,
-		reason: reason.clone(),
-		join_authorized_via_users_server: None,
-		is_direct: None,
-		avatar_url: None,
-		displayname: None,
-		third_party_invite: None,
-		blurhash: None,
-		redact_events: None,
-	};
-
 	let is_banned = services.rooms.metadata.is_banned(room_id);
 	let is_disabled = services.rooms.metadata.is_disabled(room_id);
 
-	pin_mut!(is_banned, is_disabled);
-	if is_banned.or(is_disabled).await {
-		// the room is banned/disabled, the room must be rejected locally since we
-		// cant/dont want to federate with this server
-		services
-			.rooms
-			.state_cache
-			.update_membership(
-				room_id,
-				user_id,
-				default_member_content,
-				user_id,
-				None,
-				None,
-				true,
-			)
-			.await?;
-
-		return Ok(());
-	}
-
 	let dont_have_room = services
 		.rooms
 		.state_cache
@@ -129,43 +96,41 @@ pub async fn leave_room(
 		.is_knocked(user_id, room_id)
 		.eq(&false);
 
-	// Ask a remote server if we don't have this room and are not knocking on it
-	if dont_have_room.and(not_knocked).await {
-		if let Err(e) = remote_leave_room(services, user_id, room_id, reason.clone())
-			.boxed()
-			.await
-		{
-			warn!(%user_id, "Failed to leave room {room_id} remotely: {e}");
-			// Don't tell the client about this error
-		}
+	pin_mut!(is_banned, is_disabled);
 
-		let last_state = services
-			.rooms
-			.state_cache
-			.invite_state(user_id, room_id)
-			.or_else(|_| services.rooms.state_cache.knock_state(user_id, room_id))
-			.or_else(|_| services.rooms.state_cache.left_state(user_id, room_id))
-			.await
-			.ok();
+	/*
+	there are three possible cases when leaving a room:
+	1. the room is banned or disabled, so we're not federating with it.
+	2. nobody on the homeserver is in the room, which can happen if the user is rejecting an invite
+	   to a room that we don't have any members in.
+	3. someone else on the homeserver is in the room. in this case we can leave like normal by sending a PDU over federation.
 
-		// We always drop the invite, we can't rely on other servers
-		services
-			.rooms
-			.state_cache
-			.update_membership(
-				room_id,
-				user_id,
-				default_member_content,
-				user_id,
-				last_state,
-				None,
-				true,
-			)
-			.await?;
+	in cases 1 and 2, we have to update the state cache using `mark_as_left` directly.
+	otherwise `build_and_append_pdu` will take care of updating the state cache for us.
+	*/
+
+	// `leave_pdu` is the outlier `m.room.member` event which will be synced to the
+	// user. if it's None the sync handler will create a dummy PDU.
+	let leave_pdu = if is_banned.or(is_disabled).await {
+		// case 1: the room is banned/disabled. we don't want to federate with another
+		// server to leave, so we can't create an outlier PDU.
+		None
+	} else if dont_have_room.and(not_knocked).await {
+		// case 2: ask a remote server to assist us with leaving
+		// we always mark the room as left locally, regardless of if the federated leave
+		// failed
+
+		remote_leave_room(services, user_id, room_id, reason.clone())
+			.await
+			.inspect_err(|err| {
+				warn!(%user_id, "Failed to leave room {room_id} remotely: {err}");
+			})
+			.ok()
 	} else {
+		// case 3: we can leave by sending a PDU.
 		let state_lock = services.rooms.state.mutex.lock(room_id).await;
 
-		let Ok(event) = services
+		let user_member_event_content = services
 			.rooms
 			.state_accessor
 			.room_state_get_content::<RoomMemberEventContent>(
@@ -173,44 +138,61 @@ pub async fn leave_room(
 				&StateEventType::RoomMember,
 				user_id.as_str(),
 			)
-			.await
-		else {
-			debug_warn!(
-				"Trying to leave a room you are not a member of, marking room as left locally."
-			);
+			.await;
 
-			return services
-				.rooms
-				.state_cache
-				.update_membership(
-					room_id,
-					user_id,
-					default_member_content,
-					user_id,
-					None,
-					None,
-					true,
-				)
-				.await;
-		};
+		match user_member_event_content {
+			| Ok(content) => {
+				services
+					.rooms
+					.timeline
+					.build_and_append_pdu(
+						PduBuilder::state(user_id.to_string(), &RoomMemberEventContent {
+							membership: MembershipState::Leave,
+							reason,
+							join_authorized_via_users_server: None,
+							is_direct: None,
+							..content
+						}),
+						user_id,
+						Some(room_id),
+						&state_lock,
+					)
+					.await?;
 
-		services
-			.rooms
-			.timeline
-			.build_and_append_pdu(
-				PduBuilder::state(user_id.to_string(), &RoomMemberEventContent {
-					membership: MembershipState::Leave,
-					reason,
-					join_authorized_via_users_server: None,
-					is_direct: None,
-					..event
-				}),
-				user_id,
-				room_id,
-				&state_lock,
-			)
-			.await?;
-	}
+				// `build_and_append_pdu` calls `mark_as_left` internally, so we return early.
+				return Ok(());
+			},
+			| Err(_) => {
+				// an exception to case 3 is if the user isn't even in the room they're trying
+				// to leave. this can happen if the client's caching is wrong.
+				debug_warn!(
+					"Trying to leave a room you are not a member of, marking room as left \
+					 locally."
+				);
+
+				// return the existing leave state, if one exists. `mark_as_left` will then
+				// update the `roomuserid_leftcount` table, making the leave come down sync
+				// again.
+				services
+					.rooms
+					.state_cache
+					.left_state(user_id, room_id)
+					.await?
+			},
+		}
+	};
+
+	services
+		.rooms
+		.state_cache
+		.mark_as_left(user_id, room_id, leave_pdu)
+		.await;
+
+	services
+		.rooms
+		.state_cache
+		.update_joined_count(room_id)
+		.await;
 
 	Ok(())
 }
@@ -220,7 +202,7 @@ pub async fn remote_leave_room(
 	user_id: &UserId,
 	room_id: &RoomId,
 	reason: Option<String>,
-) -> Result<()> {
+) -> Result<Pdu> {
 	let mut make_leave_response_and_server =
 		Err!(BadServerResponse("No remote server available to assist in leaving {room_id}."));
 
@@ -373,7 +355,7 @@ pub async fn remote_leave_room(
 			&remote_server,
 			federation::membership::create_leave_event::v2::Request {
 				room_id: room_id.to_owned(),
-				event_id,
+				event_id: event_id.clone(),
 				pdu: services
 					.sending
 					.convert_to_outgoing_federation_event(leave_event.clone())
@@ -382,5 +364,14 @@ pub async fn remote_leave_room(
 		)
 		.await?;
 
-	Ok(())
+	services
+		.rooms
+		.outlier
+		.add_pdu_outlier(&event_id, &leave_event);
+
+	let leave_pdu = Pdu::from_id_val(&event_id, leave_event).map_err(|e| {
+		err!(BadServerResponse("Invalid leave PDU received during federated leave: {e:?}"))
+	})?;
+
+	Ok(leave_pdu)
 }

src/api/client/membership/mod.rs

@@ -69,11 +69,11 @@ pub(crate) async fn banned_room_check(
 	}
 
 	if let Some(room_id) = room_id {
-		if services.rooms.metadata.is_banned(room_id).await
-			|| services
-				.moderation
-				.is_remote_server_forbidden(room_id.server_name().expect("legacy room mxid"))
-		{
+		let room_banned = services.rooms.metadata.is_banned(room_id).await;
+		let server_banned = room_id.server_name().is_some_and(|server_name| {
+			services.moderation.is_remote_server_forbidden(server_name)
+		});
+		if room_banned || server_banned {
 			warn!(
 				"User {user_id} who is not an admin attempted to send an invite for or \
 				 attempted to join a banned room or banned room server name: {room_id}"
@@ -106,7 +106,6 @@ pub(crate) async fn banned_room_check(
 					.boxed()
 					.await?;
 			}
-
 			return Err!(Request(Forbidden("This room is banned on this homeserver.")));
 		}
 	} else if let Some(server_name) = server_name {

src/api/client/membership/unban.rs

@@ -47,7 +47,7 @@ pub(crate) async fn unban_user_route(
 				..current_member_content
 			}),
 			sender_user,
-			&body.room_id,
+			Some(&body.room_id),
 			&state_lock,
 		)
 		.await?;

src/api/client/message.rs

@@ -16,7 +16,7 @@
 	Services,
 	rooms::{
 		lazy_loading,
-		lazy_loading::{Options, Witness},
+		lazy_loading::{MemberSet, Options},
 		timeline::PdusIterItem,
 	},
 };
@@ -30,6 +30,7 @@
 	events::{
 		AnyStateEvent, StateEventType,
 		TimelineEventType::{self, *},
+		invite_permission_config::FilterLevel,
 	},
 	serde::Raw,
 };
@@ -161,7 +162,7 @@ pub(crate) async fn get_message_events_route(
 
 	let state = witness
 		.map(Option::into_iter)
-		.map(|option| option.flat_map(Witness::into_iter))
+		.map(|option| option.flat_map(MemberSet::into_iter))
 		.map(IterStream::stream)
 		.into_stream()
 		.flatten()
@@ -191,7 +192,7 @@ pub(crate) async fn lazy_loading_witness<'a, I>(
 	services: &Services,
 	lazy_loading_context: &lazy_loading::Context<'_>,
 	events: I,
-) -> Witness
+) -> MemberSet
 where
 	I: Iterator<Item = &'a PdusIterItem> + Clone + Send,
 {
@@ -212,10 +213,10 @@ pub(crate) async fn lazy_loading_witness<'a, I>(
 	let receipts = services
 		.rooms
 		.read_receipt
-		.readreceipts_since(lazy_loading_context.room_id, oldest.into_unsigned());
+		.readreceipts_since(lazy_loading_context.room_id, Some(oldest.into_unsigned()));
 
 	pin_mut!(receipts);
-	let witness: Witness = events
+	let witness: MemberSet = events
 		.stream()
 		.map(ref_at!(1))
 		.map(Event::sender)
@@ -223,7 +224,7 @@ pub(crate) async fn lazy_loading_witness<'a, I>(
 		.chain(
 			receipts
 				.ready_take_while(|(_, c, _)| *c <= newest.into_unsigned())
-				.map(|(user_id, ..)| user_id.to_owned()),
+				.map(|(user_id, ..)| user_id),
 		)
 		.collect()
 		.await;
@@ -231,7 +232,7 @@ pub(crate) async fn lazy_loading_witness<'a, I>(
 	services
 		.rooms
 		.lazy_loading
-		.witness_retain(witness, lazy_loading_context)
+		.retain_lazy_members(witness, lazy_loading_context)
 		.await
 }
 
@@ -267,7 +268,7 @@ pub(crate) async fn ignored_filter(
 pub(crate) async fn is_ignored_pdu<Pdu>(
 	services: &Services,
 	event: &Pdu,
-	user_id: &UserId,
+	recipient_user: &UserId,
 ) -> bool
 where
 	Pdu: Event + Send + Sync,
@@ -278,20 +279,29 @@ pub(crate) async fn is_ignored_pdu<Pdu>(
 		return true;
 	}
 
-	let ignored_type = IGNORED_MESSAGE_TYPES.binary_search(event.kind()).is_ok();
-
-	let ignored_server = services
+	let sender_user = event.sender();
+	let type_ignored = IGNORED_MESSAGE_TYPES.binary_search(event.kind()).is_ok();
+	let server_ignored = services
 		.moderation
-		.is_remote_server_ignored(event.sender().server_name());
+		.is_remote_server_ignored(sender_user.server_name());
+	let user_ignored = services
+		.users
+		.user_is_ignored(sender_user, recipient_user)
+		.await;
 
-	if ignored_type
-		&& (ignored_server
-			|| (!services.config.send_messages_from_ignored_users_to_client
-				&& services
-					.users
-					.user_is_ignored(event.sender(), user_id)
-					.await))
-	{
+	if !type_ignored {
+		// We cannot safely ignore this type
+		return false;
+	}
+
+	if server_ignored {
+		// the sender's server is ignored, so ignore this event
+		return true;
+	}
+
+	if user_ignored && !services.config.send_messages_from_ignored_users_to_client {
+		// the recipient of this PDU has the sender ignored, and we're not
+		// configured to send ignored messages to clients
 		return true;
 	}
 
@@ -309,7 +319,7 @@ pub(crate) async fn visibility_filter(
 	services
 		.rooms
 		.state_accessor
-		.user_can_see_event(user_id, pdu.room_id(), pdu.event_id())
+		.user_can_see_event(user_id, &pdu.room_id_or_hash(), pdu.event_id())
 		.await
 		.then_some(item)
 }
@@ -320,6 +330,29 @@ pub(crate) fn event_filter(item: PdusIterItem, filter: &RoomEventFilter) -> Opti
 	filter.matches(pdu).then_some(item)
 }
 
+#[inline]
+pub(crate) async fn is_ignored_invite(
+	services: &Services,
+	recipient_user: &UserId,
+	room_id: &RoomId,
+) -> bool {
+	let Ok(sender_user) = services
+		.rooms
+		.state_cache
+		.invite_sender(recipient_user, room_id)
+		.await
+	else {
+		// the invite may have been sent before the invite_sender table existed.
+		// assume it's not ignored
+		return false;
+	};
+
+	services
+		.users
+		.invite_filter_level(&sender_user, recipient_user)
+		.await == FilterLevel::Ignore
+}
+
 #[cfg_attr(debug_assertions, ctor::ctor)]
 fn _is_sorted() {
 	debug_assert!(

src/api/client/profile.rs

@@ -1,5 +1,3 @@
-use std::collections::BTreeMap;
-
 use axum::extract::State;
 use conduwuit::{
 	Err, Result,
@@ -226,7 +224,8 @@ pub(crate) async fn get_avatar_url_route(
 
 /// # `GET /_matrix/client/v3/profile/{userId}`
 ///
-/// Returns the displayname, avatar_url, blurhash, and tz of the user.
+/// Returns the displayname, avatar_url, blurhash, and custom profile fields of
+/// the user.
 ///
 /// - If user is on another server and we do not have a local copy already,
 ///   fetch profile over federation.
@@ -260,9 +259,6 @@ pub(crate) async fn get_profile_route(
 			services
 				.users
 				.set_blurhash(&body.user_id, response.blurhash.clone());
-			services
-				.users
-				.set_timezone(&body.user_id, response.tz.clone());
 
 			for (profile_key, profile_key_value) in &response.custom_profile_fields {
 				services.users.set_profile_key(
@@ -276,7 +272,6 @@ pub(crate) async fn get_profile_route(
 				displayname: response.displayname,
 				avatar_url: response.avatar_url,
 				blurhash: response.blurhash,
-				tz: response.tz,
 				custom_profile_fields: response.custom_profile_fields,
 			});
 		}
@@ -288,21 +283,11 @@ pub(crate) async fn get_profile_route(
 		return Err!(Request(NotFound("Profile was not found.")));
 	}
 
-	let mut custom_profile_fields: BTreeMap<String, serde_json::Value> = services
-		.users
-		.all_profile_keys(&body.user_id)
-		.collect()
-		.await;
-
-	// services.users.timezone will collect the MSC4175 timezone key if it exists
-	custom_profile_fields.remove("us.cloke.msc4175.tz");
-	custom_profile_fields.remove("m.tz");
-
-	let (avatar_url, blurhash, displayname, tz) = join4(
+	let (avatar_url, blurhash, displayname, custom_profile_fields) = join4(
 		services.users.avatar_url(&body.user_id).ok(),
 		services.users.blurhash(&body.user_id).ok(),
 		services.users.displayname(&body.user_id).ok(),
-		services.users.timezone(&body.user_id).ok(),
+		services.users.all_profile_keys(&body.user_id).collect(),
 	)
 	.await;
 
@@ -310,7 +295,6 @@ pub(crate) async fn get_profile_route(
 		avatar_url,
 		blurhash,
 		displayname,
-		tz,
 		custom_profile_fields,
 	})
 }
@@ -423,7 +407,7 @@ pub async fn update_all_rooms(
 		if let Err(e) = services
 			.rooms
 			.timeline
-			.build_and_append_pdu(pdu_builder, user_id, room_id, &state_lock)
+			.build_and_append_pdu(pdu_builder, user_id, Some(room_id), &state_lock)
 			.await
 		{
 			warn!(%user_id, %room_id, "Failed to update/send new profile join membership update in room: {e}");

src/api/client/redact.rs

@@ -36,7 +36,7 @@ pub(crate) async fn redact_event_route(
 				})
 			},
 			sender_user,
-			&body.room_id,
+			Some(&body.room_id),
 			&state_lock,
 		)
 		.await?;

src/api/client/relations.rs

@@ -222,7 +222,7 @@ async fn visibility_filter<Pdu: Event + Send + Sync>(
 	services
 		.rooms
 		.state_accessor
-		.user_can_see_event(sender_user, pdu.room_id(), pdu.event_id())
+		.user_can_see_event(sender_user, &pdu.room_id_or_hash(), pdu.event_id())
 		.await
 		.then_some(item)
 }

src/api/client/report.rs

@@ -1,8 +1,8 @@
-use std::{fmt::Write as _, ops::Mul, time::Duration};
+use std::{fmt::Write as _, time::Duration};
 
 use axum::extract::State;
 use axum_client_ip::InsecureClientIp;
-use conduwuit::{Err, Result, debug_info, info, matrix::pdu::PduEvent, utils::ReadyExt};
+use conduwuit::{Err, Event, Result, debug_info, info, matrix::pdu::PduEvent, utils::ReadyExt};
 use conduwuit_service::Services;
 use rand::Rng;
 use ruma::{
@@ -12,7 +12,6 @@
 		room::{report_content, report_room},
 	},
 	events::{Mentions, room::message::RoomMessageEventContent},
-	int,
 };
 use tokio::time::sleep;
 
@@ -25,7 +24,6 @@ struct Report {
 	user_id: Option<OwnedUserId>,
 	report_type: String,
 	reason: Option<String>,
-	score: Option<ruma::Int>,
 }
 
 /// # `POST /_matrix/client/v3/rooms/{roomId}/report`
@@ -50,6 +48,15 @@ pub(crate) async fn report_room_route(
 
 	delay_response().await;
 
+	// We log this early in case the room ID does actually exist, in which case
+	// admins who scan their logs can see the report and choose to investigate at
+	// their discretion.
+	info!(
+		"Received room report by user {sender_user} for room {} with reason: \"{}\"",
+		body.room_id,
+		body.reason.as_deref().unwrap_or("")
+	);
+
 	if !services
 		.rooms
 		.state_cache
@@ -60,11 +67,6 @@ pub(crate) async fn report_room_route(
 			"Room does not exist to us, no local users have joined at all"
 		)));
 	}
-	info!(
-		"Received room report by user {sender_user} for room {} with reason: \"{}\"",
-		body.room_id,
-		body.reason.as_deref().unwrap_or("")
-	);
 
 	let report = Report {
 		sender: sender_user.to_owned(),
@@ -73,7 +75,6 @@ pub(crate) async fn report_room_route(
 		user_id: None,
 		report_type: "room".to_owned(),
 		reason: body.reason.clone(),
-		score: None,
 	};
 
 	services.admin.send_message(build_report(report)).await.ok();
@@ -109,7 +110,6 @@ pub(crate) async fn report_event_route(
 		&body.room_id,
 		sender_user,
 		body.reason.as_ref(),
-		body.score,
 		&pdu,
 	)
 	.await?;
@@ -127,7 +127,6 @@ pub(crate) async fn report_event_route(
 		user_id: None,
 		report_type: "event".to_owned(),
 		reason: body.reason.clone(),
-		score: body.score,
 	};
 	services.admin.send_message(build_report(report)).await.ok();
 
@@ -166,7 +165,6 @@ pub(crate) async fn report_user_route(
 		user_id: Some(body.user_id.clone()),
 		report_type: "user".to_owned(),
 		reason: body.reason.clone(),
-		score: None,
 	};
 
 	info!(
@@ -192,7 +190,6 @@ async fn is_event_report_valid(
 	room_id: &RoomId,
 	sender_user: &UserId,
 	reason: Option<&String>,
-	score: Option<ruma::Int>,
 	pdu: &PduEvent,
 ) -> Result<()> {
 	debug_info!(
@@ -200,14 +197,10 @@ async fn is_event_report_valid(
 		 valid"
 	);
 
-	if room_id != pdu.room_id {
+	if room_id != pdu.room_id_or_hash() {
 		return Err!(Request(NotFound("Event ID does not belong to the reported room",)));
 	}
 
-	if score.is_some_and(|s| s > int!(0) || s < int!(-100)) {
-		return Err!(Request(InvalidParam("Invalid score, must be within 0 to -100",)));
-	}
-
 	if reason.as_ref().is_some_and(|s| s.len() > 750) {
 		return Err!(Request(
 			InvalidParam("Reason too long, should be 750 characters or fewer",)
@@ -240,9 +233,6 @@ fn build_report(report: Report) -> RoomMessageEventContent {
 	if report.event_id.is_some() {
 		let _ = writeln!(text, "- Reported Event ID: `{}`", report.event_id.unwrap());
 	}
-	if let Some(score) = report.score {
-		let _ = writeln!(text, "- User-supplied offensiveness score: {}%", score.mul(int!(-1)));
-	}
 	if let Some(reason) = report.reason {
 		let _ = writeln!(text, "- Report Reason: {reason}");
 	}

src/api/client/room/create.rs

@@ -1,10 +1,10 @@
-use std::collections::BTreeMap;
+use std::collections::{BTreeMap, BTreeSet};
 
 use axum::extract::State;
 use conduwuit::{
-	Err, Result, debug_info, debug_warn, err, info,
+	Err, Result, RoomVersion, debug, debug_info, debug_warn, err, info,
 	matrix::{StateKey, pdu::PduBuilder},
-	warn,
+	trace, warn,
 };
 use conduwuit_service::{Services, appservice::RegistrationInfo};
 use futures::FutureExt;
@@ -13,6 +13,7 @@
 	api::client::room::{self, create_room},
 	events::{
 		TimelineEventType,
+		invite_permission_config::FilterLevel,
 		room::{
 			canonical_alias::RoomCanonicalAliasEventContent,
 			create::RoomCreateEventContent,
@@ -49,6 +50,7 @@
 /// - Send events implied by `name` and `topic`
 /// - Send invite events
 #[allow(clippy::large_stack_frames)]
+#[allow(clippy::cognitive_complexity)]
 pub(crate) async fn create_room_route(
 	State(services): State<crate::State>,
 	body: Ruma<create_room::v3::Request>,
@@ -68,51 +70,6 @@ pub(crate) async fn create_room_route(
 		return Err!(Request(UserSuspended("You cannot perform this action while suspended.")));
 	}
 
-	let room_id: OwnedRoomId = match &body.room_id {
-		| Some(custom_room_id) => custom_room_id_check(&services, custom_room_id)?,
-		| _ => RoomId::new(&services.server.name),
-	};
-
-	// check if room ID doesn't already exist instead of erroring on auth check
-	if services.rooms.short.get_shortroomid(&room_id).await.is_ok() {
-		return Err!(Request(RoomInUse("Room with that custom room ID already exists",)));
-	}
-
-	if body.visibility == room::Visibility::Public
-		&& services.server.config.lockdown_public_room_directory
-		&& !services.users.is_admin(sender_user).await
-		&& body.appservice_info.is_none()
-	{
-		warn!(
-			"Non-admin user {sender_user} tried to publish {room_id} to the room directory \
-			 while \"lockdown_public_room_directory\" is enabled"
-		);
-
-		if services.server.config.admin_room_notices {
-			services
-				.admin
-				.notice(&format!(
-					"Non-admin user {sender_user} tried to publish {room_id} to the room \
-					 directory while \"lockdown_public_room_directory\" is enabled"
-				))
-				.await;
-		}
-
-		return Err!(Request(Forbidden("Publishing rooms to the room directory is not allowed")));
-	}
-	let _short_id = services
-		.rooms
-		.short
-		.get_or_create_shortroomid(&room_id)
-		.await;
-	let state_lock = services.rooms.state.mutex.lock(&room_id).await;
-
-	let alias: Option<OwnedRoomAliasId> = match body.room_alias_name.as_ref() {
-		| Some(alias) =>
-			Some(room_alias_check(&services, alias, body.appservice_info.as_ref()).await?),
-		| _ => None,
-	};
-
 	let room_version = match body.room_version.clone() {
 		| Some(room_version) =>
 			if services.server.supported_room_version(&room_version) {
@@ -124,6 +81,86 @@ pub(crate) async fn create_room_route(
 			},
 		| None => services.server.config.default_room_version.clone(),
 	};
+	let room_features = RoomVersion::new(&room_version)?;
+
+	let room_id: Option<OwnedRoomId> = if !room_features.room_ids_as_hashes {
+		match &body.room_id {
+			| Some(custom_room_id) => Some(custom_room_id_check(&services, custom_room_id)?),
+			| None => Some(RoomId::new(services.globals.server_name())),
+		}
+	} else {
+		None
+	};
+
+	// check if room ID doesn't already exist instead of erroring on auth check
+	if let Some(ref room_id) = room_id {
+		if services.rooms.short.get_shortroomid(room_id).await.is_ok() {
+			return Err!(Request(RoomInUse("Room with that custom room ID already exists",)));
+		}
+	}
+
+	if body.visibility == room::Visibility::Public
+		&& services.server.config.lockdown_public_room_directory
+		&& !services.users.is_admin(sender_user).await
+		&& body.appservice_info.is_none()
+	{
+		warn!(
+			"Non-admin user {sender_user} tried to publish {room_id:?} to the room directory \
+			 while \"lockdown_public_room_directory\" is enabled"
+		);
+
+		if services.server.config.admin_room_notices {
+			services
+				.admin
+				.notice(&format!(
+					"Non-admin user {sender_user} tried to publish {room_id:?} to the room \
+					 directory while \"lockdown_public_room_directory\" is enabled"
+				))
+				.await;
+		}
+
+		return Err!(Request(Forbidden("Publishing rooms to the room directory is not allowed")));
+	}
+
+	let mut invitees = BTreeSet::new();
+
+	for recipient_user in &body.invite {
+		if !matches!(
+			services
+				.users
+				.invite_filter_level(recipient_user, sender_user)
+				.await,
+			FilterLevel::Allow
+		) {
+			// drop invites if the creator has them blocked
+			continue;
+		}
+
+		// if the recipient of the invite is local and has the sender blocked, error
+		// out. if the recipient is remote we can't tell yet, and if they're local and
+		// have the sender _ignored_ their invite will be filtered out in
+		// the handlers for the individual /sync endpoints
+		if services.globals.user_is_local(recipient_user)
+			&& matches!(
+				services
+					.users
+					.invite_filter_level(sender_user, recipient_user)
+					.await,
+				FilterLevel::Block
+			) {
+			return Err!(Request(InviteBlocked(
+				"{recipient_user} has blocked invites from you."
+			)));
+		}
+
+		invitees.insert(recipient_user.clone());
+	}
+
+	let alias: Option<OwnedRoomAliasId> = match body.room_alias_name.as_ref() {
+		| Some(alias) =>
+			Some(room_alias_check(&services, alias, body.appservice_info.as_ref()).await?),
+		| _ => None,
+	};
 
 	let create_content = match &body.creation_content {
 		| Some(content) => {
@@ -164,18 +201,36 @@ pub(crate) async fn create_room_route(
 			let content = match room_version {
 				| V1 | V2 | V3 | V4 | V5 | V6 | V7 | V8 | V9 | V10 =>
 					RoomCreateEventContent::new_v1(sender_user.to_owned()),
-				| _ => RoomCreateEventContent::new_v11(),
+				| V11 => RoomCreateEventContent::new_v11(),
+				| _ => RoomCreateEventContent::new_v12(),
 			};
 			let mut content =
-				serde_json::from_str::<CanonicalJsonObject>(to_raw_value(&content)?.get())
-					.unwrap();
+				serde_json::from_str::<CanonicalJsonObject>(to_raw_value(&content)?.get())?;
 			content.insert("room_version".into(), json!(room_version.as_str()).try_into()?);
 			content
 		},
 	};
 
+	let state_lock = match room_id.clone() {
+		| Some(room_id) => {
+			let _short_id = services
+				.rooms
+				.short
+				.get_or_create_shortroomid(&room_id)
+				.await;
+			services.rooms.state.mutex.lock(&room_id).await
+		},
+		| None => {
+			let temp_room_id = RoomId::new(services.globals.server_name());
+			trace!("Locking temporary room state mutex for {temp_room_id}");
+			services.rooms.state.mutex.lock(&temp_room_id).await
+		},
+	};
+
 	// 1. The room create event
-	services
+	debug!("Creating room create event for {sender_user} in room {room_id:?}");
+	let tmp_id = room_id.as_deref();
+	let create_event_id = services
 		.rooms
 		.timeline
 		.build_and_append_pdu(
@@ -186,13 +241,26 @@ pub(crate) async fn create_room_route(
 				..Default::default()
 			},
 			sender_user,
-			&room_id,
+			tmp_id,
 			&state_lock,
 		)
 		.boxed()
 		.await?;
+	trace!("Created room create event with ID {}", &create_event_id);
+	let room_id = match room_id.clone() {
+		| Some(room_id) => room_id,
+		| None => {
+			let as_room_id = create_event_id.as_str().replace('$', "!");
+			trace!("Creating room with v12 room ID {as_room_id}");
+			RoomId::parse(&as_room_id)?.to_owned()
+		},
+	};
+	drop(state_lock);
+	debug!("Room created with ID {room_id}");
+	let state_lock = services.rooms.state.mutex.lock(&room_id).await;
 
 	// 2. Let the room creator join
+	debug_info!("Joining {sender_user} to room {room_id}");
 	services
 		.rooms
 		.timeline
@@ -205,7 +273,7 @@ pub(crate) async fn create_room_route(
 				..RoomMemberEventContent::new(MembershipState::Join)
 			}),
 			sender_user,
-			&room_id,
+			Some(&room_id),
 			&state_lock,
 		)
 		.boxed()
@@ -219,26 +287,45 @@ pub(crate) async fn create_room_route(
 		| _ => RoomPreset::PrivateChat, // Room visibility should not be custom
 	});
 
-	let mut users = BTreeMap::from_iter([(sender_user.to_owned(), int!(100))]);
+	let mut power_levels_to_grant = BTreeMap::from_iter([(sender_user.to_owned(), int!(100))]);
 
 	if preset == RoomPreset::TrustedPrivateChat {
-		for invite in &body.invite {
-			if services.users.user_is_ignored(sender_user, invite).await {
-				continue;
-			} else if services.users.user_is_ignored(invite, sender_user).await {
-				// silently drop the invite to the recipient if they've been ignored by the
-				// sender, pretend it worked
-				continue;
-			}
-
-			users.insert(invite.clone(), int!(100));
+		for recipient_user in &invitees {
+			power_levels_to_grant.insert(recipient_user.clone(), int!(100));
 		}
 	}
 
+	let mut creators: Vec<OwnedUserId> = vec![sender_user.to_owned()];
+	// Do we care about additional_creators?
+	if room_features.explicitly_privilege_room_creators {
+		// Have they been specified?
+		if let Some(additional_creators) = create_content.get("additional_creators") {
+			// Are they a real array?
+			if let Some(additional_creators) = additional_creators.as_array() {
+				// Iterate through them
+				for creator in additional_creators {
+					// Are they a string?
+					if let Some(creator) = creator.as_str() {
+						// Do they parse into a real user ID?
+						if let Ok(creator) = OwnedUserId::parse(creator) {
+							// Add them to the power levels and creators
+							creators.push(creator.clone());
+						}
+					}
+				}
+			}
+		}
+	} else {
+		power_levels_to_grant.insert(sender_user.to_owned(), int!(100));
+		creators.clear(); // If this vec is not empty, default_power_levels_content will
+		// treat this as a v12 room
+	}
+
 	let power_levels_content = default_power_levels_content(
 		body.power_level_content_override.as_ref(),
 		&body.visibility,
-		users,
+		power_levels_to_grant,
+		creators,
 	)?;
 
 	services
@@ -252,7 +339,7 @@ pub(crate) async fn create_room_route(
 				..Default::default()
 			},
 			sender_user,
-			&room_id,
+			Some(&room_id),
 			&state_lock,
 		)
 		.boxed()
@@ -269,7 +356,7 @@ pub(crate) async fn create_room_route(
 					alt_aliases: vec![],
 				}),
 				sender_user,
-				&room_id,
+				Some(&room_id),
 				&state_lock,
 			)
 			.boxed()
@@ -292,7 +379,7 @@ pub(crate) async fn create_room_route(
 				}),
 			),
 			sender_user,
-			&room_id,
+			Some(&room_id),
 			&state_lock,
 		)
 		.boxed()
@@ -308,7 +395,7 @@ pub(crate) async fn create_room_route(
 				&RoomHistoryVisibilityEventContent::new(HistoryVisibility::Shared),
 			),
 			sender_user,
-			&room_id,
+			Some(&room_id),
 			&state_lock,
 		)
 		.boxed()
@@ -327,7 +414,7 @@ pub(crate) async fn create_room_route(
 				}),
 			),
 			sender_user,
-			&room_id,
+			Some(&room_id),
 			&state_lock,
 		)
 		.boxed()
@@ -363,7 +450,7 @@ pub(crate) async fn create_room_route(
 		services
 			.rooms
 			.timeline
-			.build_and_append_pdu(pdu_builder, sender_user, &room_id, &state_lock)
+			.build_and_append_pdu(pdu_builder, sender_user, Some(&room_id), &state_lock)
 			.boxed()
 			.await?;
 	}
@@ -376,7 +463,7 @@ pub(crate) async fn create_room_route(
 			.build_and_append_pdu(
 				PduBuilder::state(String::new(), &RoomNameEventContent::new(name.clone())),
 				sender_user,
-				&room_id,
+				Some(&room_id),
 				&state_lock,
 			)
 			.boxed()
@@ -390,7 +477,7 @@ pub(crate) async fn create_room_route(
 			.build_and_append_pdu(
 				PduBuilder::state(String::new(), &RoomTopicEventContent { topic: topic.clone() }),
 				sender_user,
-				&room_id,
+				Some(&room_id),
 				&state_lock,
 			)
 			.boxed()
@@ -399,17 +486,9 @@ pub(crate) async fn create_room_route(
 
 	// 8. Events implied by invite (and TODO: invite_3pid)
 	drop(state_lock);
-	for user_id in &body.invite {
-		if services.users.user_is_ignored(sender_user, user_id).await {
-			continue;
-		} else if services.users.user_is_ignored(user_id, sender_user).await {
-			// silently drop the invite to the recipient if they've been ignored by the
-			// sender, pretend it worked
-			continue;
-		}
-
+	for recipient_user in &invitees {
 		if let Err(e) =
-			invite_helper(&services, sender_user, user_id, &room_id, None, body.is_direct)
+			invite_helper(&services, sender_user, recipient_user, &room_id, None, body.is_direct)
 				.boxed()
 				.await
 		{
@@ -450,6 +529,7 @@ fn default_power_levels_content(
 	power_level_content_override: Option<&Raw<RoomPowerLevelsEventContent>>,
 	visibility: &room::Visibility,
 	users: BTreeMap<OwnedUserId, Int>,
+	creators: Vec<OwnedUserId>,
 ) -> Result<serde_json::Value> {
 	let mut power_levels_content =
 		serde_json::to_value(RoomPowerLevelsEventContent { users, ..Default::default() })
@@ -499,6 +579,19 @@ fn default_power_levels_content(
 		}
 	}
 
+	if !creators.is_empty() {
+		// Raise the default power level of tombstone to 150
+		power_levels_content["events"]["m.room.tombstone"] =
+			serde_json::to_value(150).expect("150 is valid Value");
+		for creator in creators {
+			// Omit creators from the power level list altogether
+			power_levels_content["users"]
+				.as_object_mut()
+				.expect("users is an object")
+				.remove(creator.as_str());
+		}
+	}
+
 	Ok(power_levels_content)
 }
 

src/api/client/room/upgrade.rs

@@ -2,7 +2,7 @@
 
 use axum::extract::State;
 use conduwuit::{
-	Err, Error, Event, Result, debug, err, info,
+	Err, Error, Event, Result, RoomVersion, debug, err, info,
 	matrix::{StateKey, pdu::PduBuilder},
 };
 use futures::{FutureExt, StreamExt};
@@ -68,37 +68,77 @@ pub(crate) async fn upgrade_room_route(
 		return Err!(Request(UserSuspended("You cannot perform this action while suspended.")));
 	}
 
+	// First, check if the user has permission to upgrade the room (send tombstone
+	// event)
+	let old_room_state_lock = services.rooms.state.mutex.lock(&body.room_id).await;
+
+	// Check tombstone permission by attempting to create (but not send) the event
+	// Note that this does internally call the policy server with a fake room ID,
+	// which may not be good?
+	let tombstone_test_result = services
+		.rooms
+		.timeline
+		.create_hash_and_sign_event(
+			PduBuilder::state(StateKey::new(), &RoomTombstoneEventContent {
+				body: "This room has been replaced".to_owned(),
+				replacement_room: RoomId::new(services.globals.server_name()),
+			}),
+			sender_user,
+			Some(&body.room_id),
+			&old_room_state_lock,
+		)
+		.await;
+
+	if let Err(_e) = tombstone_test_result {
+		return Err!(Request(Forbidden("User does not have permission to upgrade this room.")));
+	}
+
+	drop(old_room_state_lock);
+
 	// Create a replacement room
-	let replacement_room = RoomId::new(services.globals.server_name());
+	let room_features = RoomVersion::new(&body.new_version)?;
+	let replacement_room_owned = if !room_features.room_ids_as_hashes {
+		Some(RoomId::new(services.globals.server_name()))
+	} else {
+		None
+	};
+	let replacement_room: Option<&RoomId> = replacement_room_owned.as_ref().map(AsRef::as_ref);
+	let replacement_room_tmp = match replacement_room {
+		| Some(v) => v,
+		| None => &RoomId::new(services.globals.server_name()),
+	};
 
 	let _short_id = services
 		.rooms
 		.short
-		.get_or_create_shortroomid(&replacement_room)
+		.get_or_create_shortroomid(replacement_room_tmp)
 		.await;
 
-	let state_lock = services.rooms.state.mutex.lock(&body.room_id).await;
-
-	// Send a m.room.tombstone event to the old room to indicate that it is not
-	// intended to be used any further Fail if the sender does not have the required
-	// permissions
-	let tombstone_event_id = services
-		.rooms
-		.timeline
-		.build_and_append_pdu(
-			PduBuilder::state(StateKey::new(), &RoomTombstoneEventContent {
-				body: "This room has been replaced".to_owned(),
-				replacement_room: replacement_room.clone(),
-			}),
-			sender_user,
-			&body.room_id,
-			&state_lock,
-		)
-		.await?;
-
-	// Change lock to replacement room
-	drop(state_lock);
-	let state_lock = services.rooms.state.mutex.lock(&replacement_room).await;
+	// For pre-v12 rooms, send tombstone before creating replacement room
+	let tombstone_event_id = if !room_features.room_ids_as_hashes {
+		let state_lock = services.rooms.state.mutex.lock(&body.room_id).await;
+		// Send a m.room.tombstone event to the old room to indicate that it is not
+		// intended to be used any further
+		let tombstone_event_id = services
+			.rooms
+			.timeline
+			.build_and_append_pdu(
+				PduBuilder::state(StateKey::new(), &RoomTombstoneEventContent {
+					body: "This room has been replaced".to_owned(),
+					replacement_room: replacement_room.unwrap().to_owned(),
+				}),
+				sender_user,
+				Some(&body.room_id),
+				&state_lock,
+			)
+			.await?;
+		// Change lock to replacement room
+		drop(state_lock);
+		Some(tombstone_event_id)
+	} else {
+		None
+	};
+	let state_lock = services.rooms.state.mutex.lock(replacement_room_tmp).await;
 
 	// Get the old room creation event
 	let mut create_event_content: CanonicalJsonObject = services
@@ -111,7 +151,7 @@ pub(crate) async fn upgrade_room_route(
 	// Use the m.room.tombstone event as the predecessor
 	let predecessor = Some(ruma::events::room::create::PreviousRoom::new(
 		body.room_id.clone(),
-		Some(tombstone_event_id),
+		tombstone_event_id,
 	));
 
 	// Send a m.room.create event containing a predecessor field and the applicable
@@ -132,6 +172,7 @@ pub(crate) async fn upgrade_room_route(
 				// "creator" key no longer exists in V11 rooms
 				create_event_content.remove("creator");
 			},
+			// TODO(hydra): additional_creators
 		}
 	}
 
@@ -159,7 +200,7 @@ pub(crate) async fn upgrade_room_route(
 		return Err(Error::BadRequest(ErrorKind::BadJson, "Error forming creation event"));
 	}
 
-	services
+	let create_event_id = services
 		.rooms
 		.timeline
 		.build_and_append_pdu(
@@ -173,11 +214,18 @@ pub(crate) async fn upgrade_room_route(
 				timestamp: None,
 			},
 			sender_user,
-			&replacement_room,
+			replacement_room,
 			&state_lock,
 		)
 		.boxed()
 		.await?;
+	let create_id = create_event_id.as_str().replace('$', "!");
+	let (replacement_room, state_lock) = if room_features.room_ids_as_hashes {
+		let parsed_room_id = RoomId::parse(&create_id)?;
+		(Some(parsed_room_id), services.rooms.state.mutex.lock(parsed_room_id).await)
+	} else {
+		(replacement_room, state_lock)
+	};
 
 	// Join the new room
 	services
@@ -204,7 +252,7 @@ pub(crate) async fn upgrade_room_route(
 				timestamp: None,
 			},
 			sender_user,
-			&replacement_room,
+			replacement_room,
 			&state_lock,
 		)
 		.boxed()
@@ -243,7 +291,7 @@ pub(crate) async fn upgrade_room_route(
 						..Default::default()
 					},
 					sender_user,
-					&replacement_room,
+					replacement_room,
 					&state_lock,
 				)
 				.boxed()
@@ -268,7 +316,7 @@ pub(crate) async fn upgrade_room_route(
 		services
 			.rooms
 			.alias
-			.set_alias(alias, &replacement_room, sender_user)?;
+			.set_alias(alias, replacement_room.unwrap(), sender_user)?;
 	}
 
 	// Get the old room power levels
@@ -302,7 +350,7 @@ pub(crate) async fn upgrade_room_route(
 				..power_levels_event_content
 			}),
 			sender_user,
-			&body.room_id,
+			Some(&body.room_id),
 			&state_lock,
 		)
 		.boxed()
@@ -310,6 +358,27 @@ pub(crate) async fn upgrade_room_route(
 
 	drop(state_lock);
 
+	// For v12 rooms, send tombstone AFTER creating replacement room
+	if room_features.room_ids_as_hashes {
+		let old_room_state_lock = services.rooms.state.mutex.lock(&body.room_id).await;
+		// For v12 rooms, no event reference in predecessor due to cyclic dependency -
+		// could best effort one maybe?
+		services
+			.rooms
+			.timeline
+			.build_and_append_pdu(
+				PduBuilder::state(StateKey::new(), &RoomTombstoneEventContent {
+					body: "This room has been replaced".to_owned(),
+					replacement_room: replacement_room.unwrap().to_owned(),
+				}),
+				sender_user,
+				Some(&body.room_id),
+				&old_room_state_lock,
+			)
+			.await?;
+		drop(old_room_state_lock);
+	}
+
 	// Check if the old room has a space parent, and if so, whether we should update
 	// it (m.space.parent, room_id)
 	let parents = services
@@ -334,8 +403,9 @@ pub(crate) async fn upgrade_room_route(
 			continue;
 		};
 		debug!(
-			"Updating space {space_id} child event for room {} to {replacement_room}",
-			&body.room_id
+			"Updating space {space_id} child event for room {} to {}",
+			&body.room_id,
+			replacement_room.unwrap()
 		);
 		// First, drop the space's child event
 		let state_lock = services.rooms.state.mutex.lock(space_id).await;
@@ -352,14 +422,17 @@ pub(crate) async fn upgrade_room_route(
 					..Default::default()
 				},
 				sender_user,
-				space_id,
+				Some(space_id),
 				&state_lock,
 			)
 			.boxed()
 			.await
 			.ok();
 		// Now, add a new child event for the replacement room
-		debug!("Adding space child event for room {replacement_room} in space {space_id}");
+		debug!(
+			"Adding space child event for room {} in space {space_id}",
+			replacement_room.unwrap()
+		);
 		services
 			.rooms
 			.timeline
@@ -372,23 +445,26 @@ pub(crate) async fn upgrade_room_route(
 						suggested: child.suggested,
 					})
 					.expect("event is valid, we just created it"),
-					state_key: Some(replacement_room.as_str().into()),
+					state_key: Some(replacement_room.unwrap().as_str().into()),
 					..Default::default()
 				},
 				sender_user,
-				space_id,
+				Some(space_id),
 				&state_lock,
 			)
 			.boxed()
 			.await
 			.ok();
 		debug!(
-			"Finished updating space {space_id} child event for room {} to {replacement_room}",
-			&body.room_id
+			"Finished updating space {space_id} child event for room {} to {}",
+			&body.room_id,
+			replacement_room.unwrap()
 		);
 		drop(state_lock);
 	}
 
 	// Return the replacement room id
-	Ok(upgrade_room::v3::Response { replacement_room })
+	Ok(upgrade_room::v3::Response {
+		replacement_room: replacement_room.unwrap().to_owned(),
+	})
 }

src/api/client/send.rs

@@ -80,7 +80,7 @@ pub(crate) async fn send_message_event_route(
 				..Default::default()
 			},
 			sender_user,
-			&body.room_id,
+			Some(&body.room_id),
 			&state_lock,
 		)
 		.await?;

src/api/client/session.rs

@@ -145,9 +145,9 @@ pub(super) async fn ldap_login(
 	let is_conduwuit_admin = services.admin.user_is_admin(lowercased_user_id).await;
 
 	if is_ldap_admin && !is_conduwuit_admin {
-		services.admin.make_user_admin(lowercased_user_id).await?;
+		Box::pin(services.admin.make_user_admin(lowercased_user_id)).await?;
 	} else if !is_ldap_admin && is_conduwuit_admin {
-		services.admin.revoke_admin(lowercased_user_id).await?;
+		Box::pin(services.admin.revoke_admin(lowercased_user_id)).await?;
 	}
 
 	Ok(user_id)

src/api/client/space.rs

@@ -44,7 +44,7 @@ pub(crate) async fn get_hierarchy_route(
 		.as_ref()
 		.and_then(|s| PaginationToken::from_str(s).ok());
 
-	// Should prevent unexpeded behaviour in (bad) clients
+	// Should prevent unexpected behaviour in (bad) clients
 	if let Some(ref token) = key {
 		if token.suggested_only != body.suggested_only || token.max_depth != max_depth {
 			return Err!(Request(InvalidParam(

src/api/client/state.rs

@@ -201,7 +201,7 @@ async fn send_state_event_for_key_helper(
 				..Default::default()
 			},
 			sender,
-			room_id,
+			Some(room_id),
 			&state_lock,
 		)
 		.await?;

src/api/client/sync/mod.rs

@@ -1,65 +1,123 @@
 mod v3;
-mod v4;
 mod v5;
 
+use std::collections::VecDeque;
+
 use conduwuit::{
-	Error, PduCount, Result,
+	Event, PduCount, Result, err,
 	matrix::pdu::PduEvent,
+	ref_at, trace,
 	utils::stream::{BroadbandExt, ReadyExt, TryIgnore},
 };
 use conduwuit_service::Services;
-use futures::{StreamExt, pin_mut};
+use futures::StreamExt;
 use ruma::{
-	RoomId, UserId,
+	OwnedUserId, RoomId, UserId,
 	events::TimelineEventType::{
 		self, Beacon, CallInvite, PollStart, RoomEncrypted, RoomMessage, Sticker,
 	},
 };
 
-pub(crate) use self::{
-	v3::sync_events_route, v4::sync_events_v4_route, v5::sync_events_v5_route,
-};
+pub(crate) use self::{v3::sync_events_route, v5::sync_events_v5_route};
 
 pub(crate) const DEFAULT_BUMP_TYPES: &[TimelineEventType; 6] =
 	&[CallInvite, PollStart, Beacon, RoomEncrypted, RoomMessage, Sticker];
 
+#[derive(Default)]
+pub(crate) struct TimelinePdus {
+	pub pdus: VecDeque<(PduCount, PduEvent)>,
+	pub limited: bool,
+}
+
+impl TimelinePdus {
+	fn senders(&self) -> impl Iterator<Item = OwnedUserId> {
+		self.pdus
+			.iter()
+			.map(ref_at!(1))
+			.map(Event::sender)
+			.map(Into::into)
+	}
+}
+
+/// Load up to `limit` PDUs in the range (starting_count, ending_count].
 async fn load_timeline(
 	services: &Services,
 	sender_user: &UserId,
 	room_id: &RoomId,
-	roomsincecount: PduCount,
-	next_batch: Option<PduCount>,
+	starting_count: Option<PduCount>,
+	ending_count: Option<PduCount>,
 	limit: usize,
-) -> Result<(Vec<(PduCount, PduEvent)>, bool), Error> {
-	let last_timeline_count = services
-		.rooms
-		.timeline
-		.last_timeline_count(Some(sender_user), room_id)
-		.await?;
+) -> Result<TimelinePdus> {
+	let mut pdu_stream = match starting_count {
+		| Some(starting_count) => {
+			let last_timeline_count = services
+				.rooms
+				.timeline
+				.last_timeline_count(Some(sender_user), room_id)
+				.await
+				.map_err(|err| {
+					err!(Database(warn!("Failed to fetch end of room timeline: {}", err)))
+				})?;
 
-	if last_timeline_count <= roomsincecount {
-		return Ok((Vec::new(), false));
-	}
+			if last_timeline_count <= starting_count {
+				// no messages have been sent in this room since `starting_count`
+				return Ok(TimelinePdus::default());
+			}
 
-	let non_timeline_pdus = services
-		.rooms
-		.timeline
-		.pdus_rev(Some(sender_user), room_id, None)
-		.ignore_err()
-		.ready_skip_while(|&(pducount, _)| pducount > next_batch.unwrap_or_else(PduCount::max))
-		.ready_take_while(|&(pducount, _)| pducount > roomsincecount);
+			// for incremental sync, stream from the DB all PDUs which were sent after
+			// `starting_count` but before `ending_count`, including `ending_count` but
+			// not `starting_count`. this code is pretty similar to the initial sync
+			// branch, they're separate to allow for future optimization
+			services
+				.rooms
+				.timeline
+				.pdus_rev(
+					Some(sender_user),
+					room_id,
+					ending_count.map(|count| count.saturating_add(1)),
+				)
+				.ignore_err()
+				.ready_take_while(move |&(pducount, _)| pducount > starting_count)
+				.boxed()
+		},
+		| None => {
+			// For initial sync, stream from the DB all PDUs before and including
+			// `ending_count` in reverse order
+			services
+				.rooms
+				.timeline
+				.pdus_rev(
+					Some(sender_user),
+					room_id,
+					ending_count.map(|count| count.saturating_add(1)),
+				)
+				.ignore_err()
+				.boxed()
+		},
+	};
 
-	// Take the last events for the timeline
-	pin_mut!(non_timeline_pdus);
-	let timeline_pdus: Vec<_> = non_timeline_pdus.by_ref().take(limit).collect().await;
+	// Return at most `limit` PDUs from the stream
+	let pdus = pdu_stream
+		.by_ref()
+		.take(limit)
+		.ready_fold(VecDeque::with_capacity(limit), |mut pdus, item| {
+			pdus.push_front(item);
+			pdus
+		})
+		.await;
 
-	let timeline_pdus: Vec<_> = timeline_pdus.into_iter().rev().collect();
+	// The timeline is limited if there are still more PDUs in the stream
+	let limited = pdu_stream.next().await.is_some();
 
-	// They /sync response doesn't always return all messages, so we say the output
-	// is limited unless there are events in non_timeline_pdus
-	let limited = non_timeline_pdus.next().await.is_some();
+	trace!(
+		"syncing {:?} timeline pdus from {:?} to {:?} (limited = {:?})",
+		pdus.len(),
+		starting_count,
+		ending_count,
+		limited,
+	);
 
-	Ok((timeline_pdus, limited))
+	Ok(TimelinePdus { pdus, limited })
 }
 
 async fn share_encrypted_room(

src/api/client/sync/v3/joined.rs

@@ -0,0 +1,852 @@
+use std::collections::{BTreeMap, HashSet};
+
+use conduwuit::{
+	Result, at, debug_warn, err, extract_variant,
+	matrix::{
+		Event,
+		pdu::{PduCount, PduEvent},
+	},
+	trace,
+	utils::{
+		BoolExt, IterStream, ReadyExt, TryFutureExtExt,
+		math::ruma_from_u64,
+		stream::{TryIgnore, WidebandExt},
+	},
+	warn,
+};
+use conduwuit_service::Services;
+use futures::{
+	FutureExt, StreamExt, TryFutureExt,
+	future::{OptionFuture, join, join3, join4, try_join, try_join3},
+};
+use ruma::{
+	OwnedRoomId, OwnedUserId, RoomId, UserId,
+	api::client::sync::sync_events::{
+		UnreadNotificationsCount,
+		v3::{Ephemeral, JoinedRoom, RoomAccountData, RoomSummary, State as RoomState, Timeline},
+	},
+	events::{
+		AnyRawAccountDataEvent, StateEventType,
+		TimelineEventType::*,
+		room::member::{MembershipState, RoomMemberEventContent},
+	},
+	serde::Raw,
+	uint,
+};
+use service::rooms::short::ShortStateHash;
+
+use super::{load_timeline, share_encrypted_room};
+use crate::client::{
+	TimelinePdus, ignored_filter,
+	sync::v3::{
+		DEFAULT_TIMELINE_LIMIT, DeviceListUpdates, SyncContext, prepare_lazily_loaded_members,
+		state::{build_state_incremental, build_state_initial},
+	},
+};
+
+/// Generate the sync response for a room the user is joined to.
+#[tracing::instrument(
+	name = "joined",
+	level = "debug",
+	skip_all,
+	fields(
+		room_id = ?room_id,
+		syncing_user = ?sync_context.syncing_user,
+	),
+)]
+pub(super) async fn load_joined_room(
+	services: &Services,
+	sync_context: SyncContext<'_>,
+	ref room_id: OwnedRoomId,
+) -> Result<(JoinedRoom, DeviceListUpdates)> {
+	/*
+	Building a sync response involves many steps which all depend on each other.
+	To parallelize the process as much as possible, each step is divided into its own function,
+	and `join*` functions are used to perform steps in parallel which do not depend on each other.
+	*/
+
+	let (
+		account_data,
+		ephemeral,
+		StateAndTimeline {
+			state_events,
+			timeline,
+			summary,
+			notification_counts,
+			device_list_updates,
+		},
+	) = try_join3(
+		build_account_data(services, sync_context, room_id),
+		build_ephemeral(services, sync_context, room_id),
+		build_state_and_timeline(services, sync_context, room_id),
+	)
+	.boxed()
+	.await?;
+
+	if !timeline.is_empty() || !state_events.is_empty() {
+		trace!(
+			"syncing {} timeline events (limited = {}) and {} state events",
+			timeline.events.len(),
+			timeline.limited,
+			state_events.len()
+		);
+	}
+
+	let joined_room = JoinedRoom {
+		account_data,
+		summary: summary.unwrap_or_default(),
+		unread_notifications: notification_counts.unwrap_or_default(),
+		timeline,
+		state: RoomState {
+			events: state_events.into_iter().map(Event::into_format).collect(),
+		},
+		ephemeral,
+		unread_thread_notifications: BTreeMap::new(),
+	};
+
+	Ok((joined_room, device_list_updates))
+}
+
+/// Collect changes to the syncing user's account data events.
+#[tracing::instrument(level = "debug", skip_all)]
+async fn build_account_data(
+	services: &Services,
+	SyncContext {
+		syncing_user,
+		last_sync_end_count,
+		current_count,
+		..
+	}: SyncContext<'_>,
+	room_id: &RoomId,
+) -> Result<RoomAccountData> {
+	let account_data_changes = services
+		.account_data
+		.changes_since(Some(room_id), syncing_user, last_sync_end_count, Some(current_count))
+		.ready_filter_map(|e| extract_variant!(e, AnyRawAccountDataEvent::Room))
+		.collect()
+		.await;
+
+	Ok(RoomAccountData { events: account_data_changes })
+}
+
+/// Collect new ephemeral events.
+#[tracing::instrument(level = "debug", skip_all)]
+async fn build_ephemeral(
+	services: &Services,
+	SyncContext { syncing_user, last_sync_end_count, .. }: SyncContext<'_>,
+	room_id: &RoomId,
+) -> Result<Ephemeral> {
+	// note: some of the futures below are boxed. this is because, without the box,
+	// rustc produces over thirty inscrutable errors in `mod.rs` at the call-site
+	// of `load_joined_room`. I don't know why boxing them fixes this -- it seems
+	// to be related to the async closures and borrowing from the sync context.
+
+	// collect updates to read receipts
+	let receipt_events = services
+		.rooms
+		.read_receipt
+		.readreceipts_since(room_id, last_sync_end_count)
+		.filter_map(async |(read_user, _, edu)| {
+			let is_ignored = services
+				.users
+				.user_is_ignored(&read_user, syncing_user)
+				.await;
+
+			// filter out read receipts for ignored users
+			is_ignored.or_some(edu)
+		})
+		.collect::<Vec<_>>()
+		.boxed();
+
+	// collect the updated list of typing users, if it's changed
+	let typing_event = async {
+		let should_send_typing_event = match last_sync_end_count {
+			| Some(last_sync_end_count) => {
+				match services.rooms.typing.last_typing_update(room_id).await {
+					| Ok(last_typing_update) => {
+						// update the typing list if the users typing have changed since the last
+						// sync
+						last_typing_update > last_sync_end_count
+					},
+					| Err(err) => {
+						warn!("Error checking last typing update: {}", err);
+						return None;
+					},
+				}
+			},
+			// always update the typing list on an initial sync
+			| None => true,
+		};
+
+		if should_send_typing_event {
+			let event = services
+				.rooms
+				.typing
+				.typings_event_for_user(room_id, syncing_user)
+				.await;
+
+			if let Ok(event) = event {
+				return Some(
+					Raw::new(&event)
+						.expect("typing event should be valid")
+						.cast(),
+				);
+			}
+		}
+
+		None
+	};
+
+	// collect the syncing user's private-read marker, if it's changed
+	let private_read_event = async {
+		let should_send_private_read = match last_sync_end_count {
+			| Some(last_sync_end_count) => {
+				let last_privateread_update = services
+					.rooms
+					.read_receipt
+					.last_privateread_update(syncing_user, room_id)
+					.await;
+
+				// update the marker if it's changed since the last sync
+				last_privateread_update > last_sync_end_count
+			},
+			// always update the marker on an initial sync
+			| None => true,
+		};
+
+		if should_send_private_read {
+			services
+				.rooms
+				.read_receipt
+				.private_read_get(room_id, syncing_user)
+				.await
+				.ok()
+		} else {
+			None
+		}
+	};
+
+	let (receipt_events, typing_event, private_read_event) =
+		join3(receipt_events, typing_event, private_read_event).await;
+
+	let mut edus = receipt_events;
+	edus.extend(typing_event);
+	edus.extend(private_read_event);
+
+	Ok(Ephemeral { events: edus })
+}
+
+/// A struct to hold the state events, timeline, and other data which is
+/// computed from them.
+struct StateAndTimeline {
+	state_events: Vec<PduEvent>,
+	timeline: Timeline,
+	summary: Option<RoomSummary>,
+	notification_counts: Option<UnreadNotificationsCount>,
+	device_list_updates: DeviceListUpdates,
+}
+
+/// Compute changes to the room's state and timeline.
+#[tracing::instrument(level = "debug", skip_all)]
+async fn build_state_and_timeline(
+	services: &Services,
+	sync_context: SyncContext<'_>,
+	room_id: &RoomId,
+) -> Result<StateAndTimeline> {
+	let (shortstatehashes, timeline) = try_join(
+		fetch_shortstatehashes(services, sync_context, room_id),
+		build_timeline(services, sync_context, room_id),
+	)
+	.await?;
+
+	let (state_events, notification_counts, joined_since_last_sync) = try_join3(
+		build_state_events(services, sync_context, room_id, shortstatehashes, &timeline),
+		build_notification_counts(services, sync_context, room_id, &timeline),
+		check_joined_since_last_sync(services, shortstatehashes, sync_context),
+	)
+	.await?;
+
+	// the timeline should always include at least one PDU if the syncing user
+	// joined since the last sync, that being the syncing user's join event. if
+	// it's empty something is wrong.
+	if joined_since_last_sync && timeline.pdus.is_empty() {
+		warn!("timeline for newly joined room is empty");
+	}
+
+	let (summary, device_list_updates) = try_join(
+		build_room_summary(
+			services,
+			sync_context,
+			room_id,
+			shortstatehashes,
+			&timeline,
+			&state_events,
+			joined_since_last_sync,
+		),
+		build_device_list_updates(
+			services,
+			sync_context,
+			room_id,
+			shortstatehashes,
+			&state_events,
+			joined_since_last_sync,
+		),
+	)
+	.await?;
+
+	// the token which may be passed to the messages endpoint to backfill room
+	// history
+	let prev_batch = timeline.pdus.front().map(at!(0));
+
+	// note: we always indicate a limited timeline if the syncing user just joined
+	// the room, to indicate to the client that it should request backfill (and to
+	// copy Synapse's behavior). for federated room joins, the `timeline` will
+	// usually only include the syncing user's join event.
+	let limited = timeline.limited || joined_since_last_sync;
+
+	// filter out ignored events from the timeline and convert the PDUs into Ruma's
+	// AnySyncTimelineEvent type
+	let filtered_timeline = timeline
+		.pdus
+		.into_iter()
+		.stream()
+		.wide_filter_map(|item| ignored_filter(services, item, sync_context.syncing_user))
+		.map(at!(1))
+		.map(Event::into_format)
+		.collect::<Vec<_>>()
+		.await;
+
+	Ok(StateAndTimeline {
+		state_events,
+		timeline: Timeline {
+			limited,
+			prev_batch: prev_batch.as_ref().map(ToString::to_string),
+			events: filtered_timeline,
+		},
+		summary,
+		notification_counts,
+		device_list_updates,
+	})
+}
+
+/// Shortstatehashes necessary to compute what state events to sync.
+#[derive(Clone, Copy)]
+struct ShortStateHashes {
+	/// The current state of the syncing room.
+	current_shortstatehash: ShortStateHash,
+	/// The state of the syncing room at the end of the last sync.
+	last_sync_end_shortstatehash: Option<ShortStateHash>,
+}
+
+/// Fetch the current_shortstatehash and last_sync_end_shortstatehash.
+#[tracing::instrument(level = "debug", skip_all)]
+async fn fetch_shortstatehashes(
+	services: &Services,
+	SyncContext { last_sync_end_count, current_count, .. }: SyncContext<'_>,
+	room_id: &RoomId,
+) -> Result<ShortStateHashes> {
+	// the room state currently.
+	// TODO: this should be the room state as of `current_count`, but there's no way
+	// to get that right now.
+	let current_shortstatehash = services
+		.rooms
+		.state
+		.get_room_shortstatehash(room_id)
+		.map_err(|_| err!(Database(error!("Room {room_id} has no state"))));
+
+	// the room state as of the end of the last sync.
+	// this will be None if we are doing an initial sync or if we just joined this
+	// room.
+	let last_sync_end_shortstatehash =
+		OptionFuture::from(last_sync_end_count.map(|last_sync_end_count| {
+			// look up the shortstatehash saved by the last sync's call to
+			// `associate_token_shortstatehash`
+			services
+				.rooms
+				.user
+				.get_token_shortstatehash(room_id, last_sync_end_count)
+				.inspect_err(move |_| {
+					debug_warn!(
+						token = last_sync_end_count,
+						"Room has no shortstatehash for this token"
+					);
+				})
+				.ok()
+		}))
+		.map(Option::flatten)
+		.map(Ok);
+
+	let (current_shortstatehash, last_sync_end_shortstatehash) =
+		try_join(current_shortstatehash, last_sync_end_shortstatehash).await?;
+
+	/*
+	associate the `current_count` with the `current_shortstatehash`, so we can
+	use it on the next sync as the `last_sync_end_shortstatehash`.
+
+	TODO: the table written to by this call grows extremely fast, gaining one new entry for each
+	joined room on _every single sync request_. we need to find a better way to remember the shortstatehash
+	between syncs.
+	*/
+	services
+		.rooms
+		.user
+		.associate_token_shortstatehash(room_id, current_count, current_shortstatehash)
+		.await;
+
+	Ok(ShortStateHashes {
+		current_shortstatehash,
+		last_sync_end_shortstatehash,
+	})
+}
+
+/// Fetch recent timeline events.
+#[tracing::instrument(level = "debug", skip_all)]
+async fn build_timeline(
+	services: &Services,
+	sync_context: SyncContext<'_>,
+	room_id: &RoomId,
+) -> Result<TimelinePdus> {
+	let SyncContext {
+		syncing_user,
+		last_sync_end_count,
+		current_count,
+		filter,
+		..
+	} = sync_context;
+
+	/*
+	determine the maximum number of events to return in this sync.
+	if the sync filter specifies a limit, that will be used, otherwise
+	`DEFAULT_TIMELINE_LIMIT` will be used. `DEFAULT_TIMELINE_LIMIT` will also be
+	used if the limit is somehow greater than usize::MAX.
+	*/
+	let timeline_limit = filter
+		.room
+		.timeline
+		.limit
+		.and_then(|limit| limit.try_into().ok())
+		.unwrap_or(DEFAULT_TIMELINE_LIMIT);
+
+	load_timeline(
+		services,
+		syncing_user,
+		room_id,
+		last_sync_end_count.map(PduCount::Normal),
+		Some(PduCount::Normal(current_count)),
+		timeline_limit,
+	)
+	.await
+}
+
+/// Calculate the state events to sync.
+async fn build_state_events(
+	services: &Services,
+	sync_context: SyncContext<'_>,
+	room_id: &RoomId,
+	shortstatehashes: ShortStateHashes,
+	timeline: &TimelinePdus,
+) -> Result<Vec<PduEvent>> {
+	let SyncContext {
+		syncing_user,
+		last_sync_end_count,
+		full_state,
+		..
+	} = sync_context;
+
+	let ShortStateHashes {
+		current_shortstatehash,
+		last_sync_end_shortstatehash,
+	} = shortstatehashes;
+
+	// the spec states that the `state` property only includes state events up to
+	// the beginning of the timeline, so we determine the state of the syncing room
+	// as of the first timeline event. NOTE: this explanation is not entirely
+	// accurate; see the implementation of `build_state_incremental`.
+	let timeline_start_shortstatehash = async {
+		if let Some((_, pdu)) = timeline.pdus.front() {
+			if let Ok(shortstatehash) = services
+				.rooms
+				.state_accessor
+				.pdu_shortstatehash(&pdu.event_id)
+				.await
+			{
+				return shortstatehash;
+			}
+		}
+
+		current_shortstatehash
+	};
+
+	// the user IDs of members whose membership needs to be sent to the client, if
+	// lazy-loading is enabled.
+	let lazily_loaded_members =
+		prepare_lazily_loaded_members(services, sync_context, room_id, timeline.senders());
+
+	let (timeline_start_shortstatehash, lazily_loaded_members) =
+		join(timeline_start_shortstatehash, lazily_loaded_members).await;
+
+	// compute the state delta between the previous sync and this sync.
+	match (last_sync_end_count, last_sync_end_shortstatehash) {
+		/*
+		if `last_sync_end_count` is Some (meaning this is an incremental sync), and `last_sync_end_shortstatehash`
+		is Some (meaning the syncing user didn't just join this room for the first time ever), and `full_state` is false,
+		then use `build_state_incremental`.
+		*/
+		| (Some(last_sync_end_count), Some(last_sync_end_shortstatehash)) if !full_state =>
+			build_state_incremental(
+				services,
+				syncing_user,
+				room_id,
+				PduCount::Normal(last_sync_end_count),
+				last_sync_end_shortstatehash,
+				timeline_start_shortstatehash,
+				current_shortstatehash,
+				timeline,
+				lazily_loaded_members.as_ref(),
+			)
+			.boxed()
+			.await,
+		/*
+		otherwise use `build_state_initial`. note that this branch will be taken if the user joined this room since the last sync
+		for the first time ever, because in that case we have no `last_sync_end_shortstatehash` and can't correctly calculate
+		the state using the incremental sync algorithm.
+		*/
+		| _ =>
+			build_state_initial(
+				services,
+				syncing_user,
+				timeline_start_shortstatehash,
+				lazily_loaded_members.as_ref(),
+			)
+			.boxed()
+			.await,
+	}
+}
+
+/// Compute the number of unread notifications in this room.
+#[tracing::instrument(level = "debug", skip_all)]
+async fn build_notification_counts(
+	services: &Services,
+	SyncContext { syncing_user, last_sync_end_count, .. }: SyncContext<'_>,
+	room_id: &RoomId,
+	timeline: &TimelinePdus,
+) -> Result<Option<UnreadNotificationsCount>> {
+	// determine whether to actually update the notification counts
+	let should_send_notification_counts = async {
+		// if we're going to sync some timeline events, the notification count has
+		// definitely changed to include them
+		if !timeline.pdus.is_empty() {
+			return true;
+		}
+
+		// if this is an initial sync, we need to send notification counts because the
+		// client doesn't know what they are yet
+		let Some(last_sync_end_count) = last_sync_end_count else {
+			return true;
+		};
+
+		let last_notification_read = services
+			.rooms
+			.user
+			.last_notification_read(syncing_user, room_id)
+			.await;
+
+		// if the syncing user has read the events we sent during the last sync, we need
+		// to send a new notification count on this sync.
+		if last_notification_read > last_sync_end_count {
+			return true;
+		}
+
+		// otherwise, nothing's changed.
+		false
+	};
+
+	if should_send_notification_counts.await {
+		let (notification_count, highlight_count) = join(
+			services
+				.rooms
+				.user
+				.notification_count(syncing_user, room_id)
+				.map(TryInto::try_into)
+				.unwrap_or(uint!(0)),
+			services
+				.rooms
+				.user
+				.highlight_count(syncing_user, room_id)
+				.map(TryInto::try_into)
+				.unwrap_or(uint!(0)),
+		)
+		.await;
+
+		trace!(?notification_count, ?highlight_count, "syncing new notification counts");
+
+		Ok(Some(UnreadNotificationsCount {
+			notification_count: Some(notification_count),
+			highlight_count: Some(highlight_count),
+		}))
+	} else {
+		Ok(None)
+	}
+}
+
+/// Check if the syncing user joined the room since their last incremental sync.
+#[tracing::instrument(level = "debug", skip_all)]
+async fn check_joined_since_last_sync(
+	services: &Services,
+	ShortStateHashes { last_sync_end_shortstatehash, .. }: ShortStateHashes,
+	SyncContext { syncing_user, .. }: SyncContext<'_>,
+) -> Result<bool> {
+	// fetch the syncing user's membership event during the last sync.
+	// this will be None if `previous_sync_end_shortstatehash` is None.
+	let membership_during_previous_sync = match last_sync_end_shortstatehash {
+		| Some(last_sync_end_shortstatehash) => services
+			.rooms
+			.state_accessor
+			.state_get_content(
+				last_sync_end_shortstatehash,
+				&StateEventType::RoomMember,
+				syncing_user.as_str(),
+			)
+			.await
+			.inspect_err(|_| debug_warn!("User has no previous membership"))
+			.ok(),
+		| None => None,
+	};
+
+	// TODO: If the requesting user got state-reset out of the room, this
+	// will be `true` when it shouldn't be. this function should never be called
+	// in that situation, but it may be if the membership cache didn't get updated.
+	// the root cause of this needs to be addressed
+	let joined_since_last_sync =
+		membership_during_previous_sync.is_none_or(|content: RoomMemberEventContent| {
+			content.membership != MembershipState::Join
+		});
+
+	if joined_since_last_sync {
+		trace!("user joined since last sync");
+	}
+
+	Ok(joined_since_last_sync)
+}
+
+/// Build the `summary` field of the room object, which includes
+/// the number of joined and invited users and the room's heroes.
+#[tracing::instrument(level = "debug", skip_all)]
+async fn build_room_summary(
+	services: &Services,
+	SyncContext { syncing_user, .. }: SyncContext<'_>,
+	room_id: &RoomId,
+	ShortStateHashes { current_shortstatehash, .. }: ShortStateHashes,
+	timeline: &TimelinePdus,
+	state_events: &[PduEvent],
+	joined_since_last_sync: bool,
+) -> Result<Option<RoomSummary>> {
+	// determine whether any events in the state or timeline are membership events.
+	let are_syncing_membership_events = timeline
+		.pdus
+		.iter()
+		.map(|(_, pdu)| pdu)
+		.chain(state_events.iter())
+		.any(|event| event.kind == RoomMember);
+
+	/*
+	we only need to send an updated room summary if:
+	1. there are membership events in the state or timeline, because they might have changed the
+	   membership counts or heroes, or
+	2. the syncing user just joined this room, which usually implies #1 because their join event should be in the timeline.
+	*/
+	if !(are_syncing_membership_events || joined_since_last_sync) {
+		return Ok(None);
+	}
+
+	let joined_member_count = services
+		.rooms
+		.state_cache
+		.room_joined_count(room_id)
+		.unwrap_or(0);
+
+	let invited_member_count = services
+		.rooms
+		.state_cache
+		.room_invited_count(room_id)
+		.unwrap_or(0);
+
+	let has_name = services
+		.rooms
+		.state_accessor
+		.state_contains_type(current_shortstatehash, &StateEventType::RoomName);
+
+	let has_canonical_alias = services
+		.rooms
+		.state_accessor
+		.state_contains_type(current_shortstatehash, &StateEventType::RoomCanonicalAlias);
+
+	let (joined_member_count, invited_member_count, has_name, has_canonical_alias) =
+		join4(joined_member_count, invited_member_count, has_name, has_canonical_alias).await;
+
+	// only send heroes if the room has neither a name nor a canonical alias
+	let heroes = if !(has_name || has_canonical_alias) {
+		Some(build_heroes(services, room_id, syncing_user, current_shortstatehash).await)
+	} else {
+		None
+	};
+
+	trace!(
+		?joined_member_count,
+		?invited_member_count,
+		heroes_length = heroes.as_ref().map(HashSet::len),
+		"syncing updated summary"
+	);
+
+	Ok(Some(RoomSummary {
+		heroes: heroes
+			.map(|heroes| heroes.into_iter().collect())
+			.unwrap_or_default(),
+		joined_member_count: Some(ruma_from_u64(joined_member_count)),
+		invited_member_count: Some(ruma_from_u64(invited_member_count)),
+	}))
+}
+
+/// Fetch the user IDs to include in the `m.heroes` property of the room
+/// summary.
+async fn build_heroes(
+	services: &Services,
+	room_id: &RoomId,
+	syncing_user: &UserId,
+	current_shortstatehash: ShortStateHash,
+) -> HashSet<OwnedUserId> {
+	const MAX_HERO_COUNT: usize = 5;
+
+	// fetch joined members from the state cache first
+	let joined_members_stream = services
+		.rooms
+		.state_cache
+		.room_members(room_id)
+		.map(ToOwned::to_owned);
+
+	// then fetch invited members
+	let invited_members_stream = services
+		.rooms
+		.state_cache
+		.room_members_invited(room_id)
+		.map(ToOwned::to_owned);
+
+	// then as a last resort fetch every membership event
+	let all_members_stream = services
+		.rooms
+		.short
+		.multi_get_statekey_from_short(
+			services
+				.rooms
+				.state_accessor
+				.state_full_shortids(current_shortstatehash)
+				.ignore_err()
+				.ready_filter_map(|(key, _)| Some(key)),
+		)
+		.ignore_err()
+		.ready_filter_map(|(event_type, state_key)| {
+			if event_type == StateEventType::RoomMember {
+				state_key.to_string().try_into().ok()
+			} else {
+				None
+			}
+		});
+
+	joined_members_stream
+		.chain(invited_members_stream)
+		.chain(all_members_stream)
+		// the hero list should never include the syncing user
+		.ready_filter(|user_id| user_id != syncing_user)
+		.take(MAX_HERO_COUNT)
+		.collect()
+		.await
+}
+
+/// Collect updates to users' device lists for E2EE.
+#[tracing::instrument(level = "debug", skip_all)]
+async fn build_device_list_updates(
+	services: &Services,
+	SyncContext {
+		syncing_user,
+		last_sync_end_count,
+		current_count,
+		..
+	}: SyncContext<'_>,
+	room_id: &RoomId,
+	ShortStateHashes { current_shortstatehash, .. }: ShortStateHashes,
+	state_events: &Vec<PduEvent>,
+	joined_since_last_sync: bool,
+) -> Result<DeviceListUpdates> {
+	let is_encrypted_room = services
+		.rooms
+		.state_accessor
+		.state_get(current_shortstatehash, &StateEventType::RoomEncryption, "")
+		.is_ok();
+
+	// initial syncs don't include device updates, and rooms which aren't encrypted
+	// don't affect them, so return early in either of those cases
+	if last_sync_end_count.is_none() || !(is_encrypted_room.await) {
+		return Ok(DeviceListUpdates::new());
+	}
+
+	let mut device_list_updates = DeviceListUpdates::new();
+
+	// add users with changed keys to the `changed` list
+	services
+		.users
+		.room_keys_changed(room_id, last_sync_end_count, Some(current_count))
+		.map(at!(0))
+		.map(ToOwned::to_owned)
+		.ready_for_each(|user_id| {
+			device_list_updates.changed.insert(user_id);
+		})
+		.await;
+
+	// add users who now share encrypted rooms to `changed` and
+	// users who no longer share encrypted rooms to `left`
+	for state_event in state_events {
+		if state_event.kind == RoomMember {
+			let Some(content): Option<RoomMemberEventContent> = state_event.get_content().ok()
+			else {
+				continue;
+			};
+
+			let Some(user_id): Option<OwnedUserId> = state_event
+				.state_key
+				.as_ref()
+				.and_then(|key| key.parse().ok())
+			else {
+				continue;
+			};
+
+			{
+				use MembershipState::*;
+
+				if matches!(content.membership, Leave | Join) {
+					let shares_encrypted_room =
+						share_encrypted_room(services, syncing_user, &user_id, Some(room_id))
+							.await;
+					match content.membership {
+						| Leave if !shares_encrypted_room => {
+							device_list_updates.left.insert(user_id);
+						},
+						| Join if joined_since_last_sync || shares_encrypted_room => {
+							device_list_updates.changed.insert(user_id);
+						},
+						| _ => (),
+					}
+				}
+			}
+		}
+	}
+
+	if !device_list_updates.is_empty() {
+		trace!(
+			changed = device_list_updates.changed.len(),
+			left = device_list_updates.left.len(),
+			"syncing device list updates"
+		);
+	}
+
+	Ok(device_list_updates)
+}

src/api/client/sync/v3/left.rs

@@ -0,0 +1,349 @@
+use conduwuit::{
+	Event, PduCount, PduEvent, Result, at, debug_warn,
+	pdu::EventHash,
+	trace,
+	utils::{self, IterStream, future::ReadyEqExt, stream::WidebandExt as _},
+};
+use futures::{StreamExt, future::join};
+use ruma::{
+	EventId, OwnedRoomId, RoomId,
+	api::client::sync::sync_events::v3::{LeftRoom, RoomAccountData, State, Timeline},
+	events::{StateEventType, TimelineEventType},
+	uint,
+};
+use serde_json::value::RawValue;
+use service::{Services, rooms::short::ShortStateHash};
+
+use crate::client::{
+	TimelinePdus, ignored_filter,
+	sync::{
+		load_timeline,
+		v3::{
+			DEFAULT_TIMELINE_LIMIT, SyncContext, prepare_lazily_loaded_members,
+			state::build_state_initial,
+		},
+	},
+};
+
+#[tracing::instrument(
+	name = "left",
+	level = "debug",
+	skip_all,
+	fields(
+		room_id = %room_id,
+	),
+)]
+#[allow(clippy::too_many_arguments)]
+pub(super) async fn load_left_room(
+	services: &Services,
+	sync_context: SyncContext<'_>,
+	ref room_id: OwnedRoomId,
+	leave_membership_event: Option<PduEvent>,
+) -> Result<Option<LeftRoom>> {
+	let SyncContext {
+		syncing_user,
+		last_sync_end_count,
+		current_count,
+		filter,
+		..
+	} = sync_context;
+
+	// the global count as of the moment the user left the room
+	let Some(left_count) = services
+		.rooms
+		.state_cache
+		.get_left_count(room_id, syncing_user)
+		.await
+		.ok()
+	else {
+		// if we get here, the membership cache is incorrect, likely due to a state
+		// reset
+		debug_warn!("attempting to sync left room but no left count exists");
+		return Ok(None);
+	};
+
+	// return early if we haven't gotten to this leave yet.
+	// this can happen if the user leaves while a sync response is being generated
+	if current_count < left_count {
+		return Ok(None);
+	}
+
+	// return early if this is an incremental sync, and we've already synced this
+	// leave to the user, and `include_leave` isn't set on the filter.
+	if !filter.room.include_leave && last_sync_end_count >= Some(left_count) {
+		return Ok(None);
+	}
+
+	if let Some(ref leave_membership_event) = leave_membership_event {
+		debug_assert_eq!(
+			leave_membership_event.kind,
+			TimelineEventType::RoomMember,
+			"leave PDU should be m.room.member"
+		);
+	}
+
+	let does_not_exist = services.rooms.metadata.exists(room_id).eq(&false).await;
+
+	let (timeline, state_events) = match leave_membership_event {
+		| Some(leave_membership_event) if does_not_exist => {
+			/*
+			we have none PDUs with left beef for this room, likely because it was a rejected invite to a room
+			which nobody on this homeserver is in. `leave_pdu` is the remote-assisted outlier leave event for the room,
+			which is all we can send to the client.
+
+			if this is an initial sync, don't include this room at all to keep the client from asking for
+			state that we don't have.
+			*/
+
+			if last_sync_end_count.is_none() {
+				return Ok(None);
+			}
+
+			trace!("syncing remote-assisted leave PDU");
+			(TimelinePdus::default(), vec![leave_membership_event])
+		},
+		| Some(leave_membership_event) => {
+			// we have this room in our DB, and can fetch the state and timeline from when
+			// the user left.
+
+			let leave_state_key = syncing_user;
+			debug_assert_eq!(
+				Some(leave_state_key.as_str()),
+				leave_membership_event.state_key(),
+				"leave PDU should be for the user requesting the sync"
+			);
+
+			// the shortstatehash of the state _immediately before_ the syncing user left
+			// this room. the state represented here _does not_ include
+			// `leave_membership_event`.
+			let leave_shortstatehash = services
+				.rooms
+				.state_accessor
+				.pdu_shortstatehash(&leave_membership_event.event_id)
+				.await?;
+
+			let prev_membership_event = services
+				.rooms
+				.state_accessor
+				.state_get(
+					leave_shortstatehash,
+					&StateEventType::RoomMember,
+					leave_state_key.as_str(),
+				)
+				.await?;
+
+			build_left_state_and_timeline(
+				services,
+				sync_context,
+				room_id,
+				leave_membership_event,
+				leave_shortstatehash,
+				prev_membership_event,
+			)
+			.await?
+		},
+		| None => {
+			/*
+			no leave event was actually sent in this room, but we still need to pretend
+			like the user left it. this is usually because the room was banned by a server admin.
+
+			if this is an incremental sync, generate a fake leave event to make the room vanish from clients.
+			otherwise we don't tell the client about this room at all.
+			*/
+			if last_sync_end_count.is_none() {
+				return Ok(None);
+			}
+
+			trace!("syncing dummy leave event");
+			(TimelinePdus::default(), vec![create_dummy_leave_event(
+				services,
+				sync_context,
+				room_id,
+			)])
+		},
+	};
+
+	let raw_timeline_pdus = timeline
+		.pdus
+		.into_iter()
+		.stream()
+		// filter out ignored events from the timeline
+		.wide_filter_map(|item| ignored_filter(services, item, syncing_user))
+		.map(at!(1))
+		.map(Event::into_format)
+		.collect::<Vec<_>>()
+		.await;
+
+	Ok(Some(LeftRoom {
+		account_data: RoomAccountData { events: Vec::new() },
+		timeline: Timeline {
+			limited: timeline.limited,
+			prev_batch: Some(current_count.to_string()),
+			events: raw_timeline_pdus,
+		},
+		state: State {
+			events: state_events.into_iter().map(Event::into_format).collect(),
+		},
+	}))
+}
+
+async fn build_left_state_and_timeline(
+	services: &Services,
+	sync_context: SyncContext<'_>,
+	room_id: &RoomId,
+	leave_membership_event: PduEvent,
+	leave_shortstatehash: ShortStateHash,
+	prev_membership_event: PduEvent,
+) -> Result<(TimelinePdus, Vec<PduEvent>)> {
+	let SyncContext {
+		syncing_user,
+		last_sync_end_count,
+		filter,
+		..
+	} = sync_context;
+
+	let timeline_start_count = if let Some(last_sync_end_count) = last_sync_end_count {
+		// for incremental syncs, start the timeline after `since`
+		PduCount::Normal(last_sync_end_count)
+	} else {
+		// for initial syncs, start the timeline after the previous membership
+		// event. we don't want to include the membership event itself
+		// because clients get confused when they see a `join`
+		// membership event in a `leave` room.
+		services
+			.rooms
+			.timeline
+			.get_pdu_count(&prev_membership_event.event_id)
+			.await?
+	};
+
+	// end the timeline at the user's leave event
+	let timeline_end_count = services
+		.rooms
+		.timeline
+		.get_pdu_count(leave_membership_event.event_id())
+		.await?;
+
+	// limit the timeline using the same logic as for joined rooms
+	let timeline_limit = filter
+		.room
+		.timeline
+		.limit
+		.and_then(|limit| limit.try_into().ok())
+		.unwrap_or(DEFAULT_TIMELINE_LIMIT);
+
+	let timeline = load_timeline(
+		services,
+		syncing_user,
+		room_id,
+		Some(timeline_start_count),
+		Some(timeline_end_count),
+		timeline_limit,
+	)
+	.await?;
+
+	let timeline_start_shortstatehash = async {
+		if let Some((_, pdu)) = timeline.pdus.front() {
+			if let Ok(shortstatehash) = services
+				.rooms
+				.state_accessor
+				.pdu_shortstatehash(&pdu.event_id)
+				.await
+			{
+				return shortstatehash;
+			}
+		}
+
+		// the timeline generally should not be empty (see the TODO further down),
+		// but in case it is we use `leave_shortstatehash` as the state to
+		// send
+		leave_shortstatehash
+	};
+
+	let lazily_loaded_members =
+		prepare_lazily_loaded_members(services, sync_context, room_id, timeline.senders());
+
+	let (timeline_start_shortstatehash, lazily_loaded_members) =
+		join(timeline_start_shortstatehash, lazily_loaded_members).await;
+
+	// TODO: calculate incremental state for incremental syncs.
+	// always calculating initial state _works_ but returns more data and does
+	// more processing than strictly necessary.
+	let mut state = build_state_initial(
+		services,
+		syncing_user,
+		timeline_start_shortstatehash,
+		lazily_loaded_members.as_ref(),
+	)
+	.await?;
+
+	/*
+	remove membership events for the syncing user from state.
+	usually, `state` should include a `join` membership event and `timeline` should include a `leave` one.
+	however, the matrix-js-sdk gets confused when this happens (see [1]) and doesn't process the room leave,
+	so we have to filter out the membership from `state`.
+
+	NOTE: we are sending more information than synapse does in this scenario, because we always
+	calculate `state` for initial syncs, even when the sync being performed is incremental.
+	however, the specification does not forbid sending extraneous events in `state`.
+
+	TODO: there is an additional bug at play here. sometimes `load_joined_room` syncs the `leave` event
+	before `load_left_room` does, which means the `timeline` we sync immediately after a leave is empty.
+	this shouldn't happen -- `timeline` should always include the `leave` event. this is probably
+	a race condition with the membership state cache.
+
+	[1]: https://github.com/matrix-org/matrix-js-sdk/issues/5071
+	*/
+
+	// `state` should only ever include one membership event for the syncing user
+	let membership_event_index = state.iter().position(|pdu| {
+		*pdu.event_type() == TimelineEventType::RoomMember
+			&& pdu.state_key() == Some(syncing_user.as_str())
+	});
+
+	if let Some(index) = membership_event_index {
+		// the ordering of events in `state` does not matter
+		state.swap_remove(index);
+	}
+
+	trace!(
+		?timeline_start_count,
+		?timeline_end_count,
+		"syncing {} timeline events (limited = {}) and {} state events",
+		timeline.pdus.len(),
+		timeline.limited,
+		state.len()
+	);
+
+	Ok((timeline, state))
+}
+
+fn create_dummy_leave_event(
+	services: &Services,
+	SyncContext { syncing_user, .. }: SyncContext<'_>,
+	room_id: &RoomId,
+) -> PduEvent {
+	// TODO: because this event ID is random, it could cause caching issues with
+	// clients. perhaps a database table could be created to hold these dummy
+	// events, or they could be stored as outliers?
+	PduEvent {
+		event_id: EventId::new(services.globals.server_name()),
+		sender: syncing_user.to_owned(),
+		origin: None,
+		origin_server_ts: utils::millis_since_unix_epoch()
+			.try_into()
+			.expect("Timestamp is valid js_int value"),
+		kind: TimelineEventType::RoomMember,
+		content: RawValue::from_string(r#"{"membership": "leave"}"#.to_owned()).unwrap(),
+		state_key: Some(syncing_user.as_str().into()),
+		unsigned: None,
+		// The following keys are dropped on conversion
+		room_id: Some(room_id.to_owned()),
+		prev_events: vec![],
+		depth: uint!(1),
+		auth_events: vec![],
+		redacts: None,
+		hashes: EventHash { sha256: String::new() },
+		signatures: None,
+	}
+}

src/api/client/sync/v3/mod.rs

@@ -0,0 +1,494 @@
+mod joined;
+mod left;
+mod state;
+
+use std::{
+	cmp::{self},
+	collections::{BTreeMap, HashMap, HashSet},
+	time::Duration,
+};
+
+use axum::extract::State;
+use conduwuit::{
+	Result, extract_variant,
+	utils::{
+		ReadyExt, TryFutureExtExt,
+		stream::{BroadbandExt, Tools, WidebandExt},
+	},
+	warn,
+};
+use conduwuit_service::Services;
+use futures::{
+	FutureExt, StreamExt, TryFutureExt,
+	future::{OptionFuture, join3, join4, join5},
+};
+use ruma::{
+	DeviceId, OwnedUserId, RoomId, UserId,
+	api::client::{
+		filter::FilterDefinition,
+		sync::sync_events::{
+			self, DeviceLists,
+			v3::{
+				Filter, GlobalAccountData, InviteState, InvitedRoom, KnockState, KnockedRoom,
+				Presence, Rooms, ToDevice,
+			},
+		},
+		uiaa::UiaaResponse,
+	},
+	events::{
+		AnyRawAccountDataEvent,
+		presence::{PresenceEvent, PresenceEventContent},
+	},
+	serde::Raw,
+};
+use service::rooms::lazy_loading::{self, MemberSet, Options as _};
+
+use super::{load_timeline, share_encrypted_room};
+use crate::{
+	Ruma, RumaResponse,
+	client::{
+		is_ignored_invite,
+		sync::v3::{joined::load_joined_room, left::load_left_room},
+	},
+};
+
+/// The default maximum number of events to return in the `timeline` key of
+/// joined and left rooms. If the number of events sent since the last sync
+/// exceeds this number, the `timeline` will be `limited`.
+const DEFAULT_TIMELINE_LIMIT: usize = 30;
+
+/// A collection of updates to users' device lists, used for E2EE.
+struct DeviceListUpdates {
+	changed: HashSet<OwnedUserId>,
+	left: HashSet<OwnedUserId>,
+}
+
+impl DeviceListUpdates {
+	fn new() -> Self {
+		Self {
+			changed: HashSet::new(),
+			left: HashSet::new(),
+		}
+	}
+
+	fn merge(&mut self, other: Self) {
+		self.changed.extend(other.changed);
+		self.left.extend(other.left);
+	}
+
+	fn is_empty(&self) -> bool { self.changed.is_empty() && self.left.is_empty() }
+}
+
+impl From<DeviceListUpdates> for DeviceLists {
+	fn from(val: DeviceListUpdates) -> Self {
+		Self {
+			changed: val.changed.into_iter().collect(),
+			left: val.left.into_iter().collect(),
+		}
+	}
+}
+
+/// References to common data needed to calculate the sync response.
+#[derive(Clone, Copy)]
+struct SyncContext<'a> {
+	/// The ID of the user requesting this sync.
+	syncing_user: &'a UserId,
+	/// The ID of the device requesting this sync, which will belong to
+	/// `syncing_user`.
+	syncing_device: &'a DeviceId,
+	/// The global count at the end of the previous sync response.
+	/// The previous sync's `current_count` will become the next sync's
+	/// `last_sync_end_count`. This will be None if no `since` query parameter
+	/// was specified, indicating an initial sync.
+	last_sync_end_count: Option<u64>,
+	/// The global count as of when we started building the sync response.
+	/// This is used as an upper bound when querying the database to ensure the
+	/// response represents a snapshot in time and doesn't include data which
+	/// appeared while the response was being built.
+	current_count: u64,
+	/// The `full_state` query parameter, used when syncing state for joined and
+	/// left rooms.
+	full_state: bool,
+	/// The sync filter, which the client uses to specify what data should be
+	/// included in the sync response.
+	filter: &'a FilterDefinition,
+}
+
+impl<'a> SyncContext<'a> {
+	fn lazy_loading_context(&self, room_id: &'a RoomId) -> lazy_loading::Context<'a> {
+		lazy_loading::Context {
+			user_id: self.syncing_user,
+			device_id: Some(self.syncing_device),
+			room_id,
+			token: self.last_sync_end_count,
+			options: Some(&self.filter.room.state.lazy_load_options),
+		}
+	}
+
+	#[inline]
+	fn lazy_loading_enabled(&self) -> bool {
+		(self.filter.room.state.lazy_load_options.is_enabled()
+			|| self.filter.room.timeline.lazy_load_options.is_enabled())
+			&& !self.full_state
+	}
+}
+
+type PresenceUpdates = HashMap<OwnedUserId, PresenceEventContent>;
+
+/// # `GET /_matrix/client/r0/sync`
+///
+/// Synchronize the client's state with the latest state on the server.
+///
+/// - This endpoint takes a `since` parameter which should be the `next_batch`
+///   value from a previous request for incremental syncs.
+///
+/// Calling this endpoint without a `since` parameter returns:
+/// - Some of the most recent events of each timeline
+/// - Notification counts for each room
+/// - Joined and invited member counts, heroes
+/// - All state events
+///
+/// Calling this endpoint with a `since` parameter from a previous `next_batch`
+/// returns: For joined rooms:
+/// - Some of the most recent events of each timeline that happened after since
+/// - If user joined the room after since: All state events (unless lazy loading
+///   is activated) and all device list updates in that room
+/// - If the user was already in the room: A list of all events that are in the
+///   state now, but were not in the state at `since`
+/// - If the state we send contains a member event: Joined and invited member
+///   counts, heroes
+/// - Device list updates that happened after `since`
+/// - If there are events in the timeline we send or the user send updated his
+///   read mark: Notification counts
+/// - EDUs that are active now (read receipts, typing updates, presence)
+/// - TODO: Allow multiple sync streams to support Pantalaimon
+///
+/// For invited rooms:
+/// - If the user was invited after `since`: A subset of the state of the room
+///   at the point of the invite
+///
+/// For left rooms:
+/// - If the user left after `since`: `prev_batch` token, empty state (TODO:
+///   subset of the state at the point of the leave)
+#[tracing::instrument(
+	name = "sync",
+	level = "debug",
+	skip_all,
+	fields(
+		since = %body.body.since.as_deref().unwrap_or_default(),
+    )
+)]
+pub(crate) async fn sync_events_route(
+	State(services): State<crate::State>,
+	body: Ruma<sync_events::v3::Request>,
+) -> Result<sync_events::v3::Response, RumaResponse<UiaaResponse>> {
+	let (sender_user, sender_device) = body.sender();
+
+	// Presence update
+	if services.config.allow_local_presence {
+		services
+			.presence
+			.ping_presence(sender_user, &body.body.set_presence)
+			.await?;
+	}
+
+	// Setup watchers, so if there's no response, we can wait for them
+	let watcher = services.sync.watch(sender_user, sender_device);
+
+	let response = build_sync_events(&services, &body).await?;
+	if body.body.full_state
+		|| !(response.rooms.is_empty()
+			&& response.presence.is_empty()
+			&& response.account_data.is_empty()
+			&& response.device_lists.is_empty()
+			&& response.to_device.is_empty())
+	{
+		return Ok(response);
+	}
+
+	// Hang a few seconds so requests are not spammed
+	// Stop hanging if new info arrives
+	let default = Duration::from_secs(30);
+	let duration = cmp::min(body.body.timeout.unwrap_or(default), default);
+	_ = tokio::time::timeout(duration, watcher).await;
+
+	// Retry returning data
+	build_sync_events(&services, &body).await
+}
+
+pub(crate) async fn build_sync_events(
+	services: &Services,
+	body: &Ruma<sync_events::v3::Request>,
+) -> Result<sync_events::v3::Response, RumaResponse<UiaaResponse>> {
+	let (syncing_user, syncing_device) = body.sender();
+
+	let current_count = services.globals.current_count()?;
+
+	// the `since` token is the last sync end count stringified
+	let last_sync_end_count = body
+		.body
+		.since
+		.as_ref()
+		.and_then(|string| string.parse().ok());
+
+	let full_state = body.body.full_state;
+
+	// FilterDefinition is very large (0x1000 bytes), let's put it on the heap
+	let filter = Box::new(match body.body.filter.as_ref() {
+		// use the default filter if none was specified
+		| None => FilterDefinition::default(),
+		// use inline filters directly
+		| Some(Filter::FilterDefinition(filter)) => filter.clone(),
+		// look up filter IDs from the database
+		| Some(Filter::FilterId(filter_id)) => services
+			.users
+			.get_filter(syncing_user, filter_id)
+			.await
+			.unwrap_or_default(),
+	});
+
+	let context = SyncContext {
+		syncing_user,
+		syncing_device,
+		last_sync_end_count,
+		current_count,
+		full_state,
+		filter: &filter,
+	};
+
+	let joined_rooms = services
+		.rooms
+		.state_cache
+		.rooms_joined(syncing_user)
+		.map(ToOwned::to_owned)
+		.broad_filter_map(|room_id| async {
+			let joined_room = load_joined_room(services, context, room_id.clone()).await;
+
+			match joined_room {
+				| Ok((room, updates)) => Some((room_id, room, updates)),
+				| Err(err) => {
+					warn!(?err, ?room_id, "error loading joined room {}", room_id);
+					None
+				},
+			}
+		})
+		.ready_fold(
+			(BTreeMap::new(), DeviceListUpdates::new()),
+			|(mut joined_rooms, mut all_updates), (room_id, joined_room, updates)| {
+				all_updates.merge(updates);
+
+				if !joined_room.is_empty() {
+					joined_rooms.insert(room_id, joined_room);
+				}
+
+				(joined_rooms, all_updates)
+			},
+		);
+
+	let left_rooms = services
+		.rooms
+		.state_cache
+		.rooms_left(syncing_user)
+		.broad_filter_map(|(room_id, leave_pdu)| {
+			load_left_room(services, context, room_id.clone(), leave_pdu)
+				.map_ok(move |left_room| (room_id, left_room))
+				.ok()
+		})
+		.ready_filter_map(|(room_id, left_room)| left_room.map(|left_room| (room_id, left_room)))
+		.collect();
+
+	let invited_rooms = services
+		.rooms
+		.state_cache
+		.rooms_invited(syncing_user)
+		.wide_filter_map(async |(room_id, invite_state)| {
+			if is_ignored_invite(services, syncing_user, &room_id).await {
+				None
+			} else {
+				Some((room_id, invite_state))
+			}
+		})
+		.fold_default(|mut invited_rooms: BTreeMap<_, _>, (room_id, invite_state)| async move {
+			let invite_count = services
+				.rooms
+				.state_cache
+				.get_invite_count(&room_id, syncing_user)
+				.await
+				.ok();
+
+			// only sync this invite if it was sent after the last /sync call
+			if last_sync_end_count < invite_count {
+				let invited_room = InvitedRoom {
+					invite_state: InviteState { events: invite_state },
+				};
+
+				invited_rooms.insert(room_id, invited_room);
+			}
+			invited_rooms
+		});
+
+	let knocked_rooms = services
+		.rooms
+		.state_cache
+		.rooms_knocked(syncing_user)
+		.fold_default(|mut knocked_rooms: BTreeMap<_, _>, (room_id, knock_state)| async move {
+			let knock_count = services
+				.rooms
+				.state_cache
+				.get_knock_count(&room_id, syncing_user)
+				.await
+				.ok();
+
+			// only sync this knock if it was sent after the last /sync call
+			if last_sync_end_count < knock_count {
+				let knocked_room = KnockedRoom {
+					knock_state: KnockState { events: knock_state },
+				};
+
+				knocked_rooms.insert(room_id, knocked_room);
+			}
+			knocked_rooms
+		});
+
+	let presence_updates: OptionFuture<_> = services
+		.config
+		.allow_local_presence
+		.then(|| process_presence_updates(services, last_sync_end_count, syncing_user))
+		.into();
+
+	let account_data = services
+		.account_data
+		.changes_since(None, syncing_user, last_sync_end_count, Some(current_count))
+		.ready_filter_map(|e| extract_variant!(e, AnyRawAccountDataEvent::Global))
+		.collect();
+
+	// Look for device list updates of this account
+	let keys_changed = services
+		.users
+		.keys_changed(syncing_user, last_sync_end_count, Some(current_count))
+		.map(ToOwned::to_owned)
+		.collect::<HashSet<_>>();
+
+	let to_device_events = services
+		.users
+		.get_to_device_events(
+			syncing_user,
+			syncing_device,
+			last_sync_end_count,
+			Some(current_count),
+		)
+		.collect::<Vec<_>>();
+
+	let device_one_time_keys_count = services
+		.users
+		.count_one_time_keys(syncing_user, syncing_device);
+
+	// Remove all to-device events the device received *last time*
+	let remove_to_device_events =
+		services
+			.users
+			.remove_to_device_events(syncing_user, syncing_device, last_sync_end_count);
+
+	let rooms = join4(joined_rooms, left_rooms, invited_rooms, knocked_rooms);
+	let ephemeral = join3(remove_to_device_events, to_device_events, presence_updates);
+	let top = join5(account_data, ephemeral, device_one_time_keys_count, keys_changed, rooms)
+		.boxed()
+		.await;
+
+	let (account_data, ephemeral, device_one_time_keys_count, keys_changed, rooms) = top;
+	let ((), to_device_events, presence_updates) = ephemeral;
+	let (joined_rooms, left_rooms, invited_rooms, knocked_rooms) = rooms;
+	let (joined_rooms, mut device_list_updates) = joined_rooms;
+	device_list_updates.changed.extend(keys_changed);
+
+	let response = sync_events::v3::Response {
+		account_data: GlobalAccountData { events: account_data },
+		device_lists: device_list_updates.into(),
+		device_one_time_keys_count,
+		// Fallback keys are not yet supported
+		device_unused_fallback_key_types: None,
+		next_batch: current_count.to_string(),
+		presence: Presence {
+			events: presence_updates
+				.into_iter()
+				.flat_map(IntoIterator::into_iter)
+				.map(|(sender, content)| PresenceEvent { content, sender })
+				.map(|ref event| Raw::new(event))
+				.filter_map(Result::ok)
+				.collect(),
+		},
+		rooms: Rooms {
+			leave: left_rooms,
+			join: joined_rooms,
+			invite: invited_rooms,
+			knock: knocked_rooms,
+		},
+		to_device: ToDevice { events: to_device_events },
+	};
+
+	Ok(response)
+}
+
+#[tracing::instrument(name = "presence", level = "debug", skip_all)]
+async fn process_presence_updates(
+	services: &Services,
+	last_sync_end_count: Option<u64>,
+	syncing_user: &UserId,
+) -> PresenceUpdates {
+	services
+		.presence
+		.presence_since(last_sync_end_count.unwrap_or(0)) // send all presences on initial sync
+		.filter(|(user_id, ..)| {
+			services
+				.rooms
+				.state_cache
+				.user_sees_user(syncing_user, user_id)
+		})
+		.filter_map(|(user_id, _, presence_bytes)| {
+			services
+				.presence
+				.from_json_bytes_to_event(presence_bytes, user_id)
+				.map_ok(move |event| (user_id, event))
+				.ok()
+		})
+		.map(|(user_id, event)| (user_id.to_owned(), event.content))
+		.collect()
+		.await
+}
+
+/// Using the provided sync context and an iterator of user IDs in the
+/// `timeline`, return a HashSet of user IDs whose membership events should be
+/// sent to the client if lazy-loading is enabled.
+#[allow(clippy::let_and_return)]
+async fn prepare_lazily_loaded_members(
+	services: &Services,
+	sync_context: SyncContext<'_>,
+	room_id: &RoomId,
+	timeline_members: impl Iterator<Item = OwnedUserId>,
+) -> Option<MemberSet> {
+	let lazy_loading_context = &sync_context.lazy_loading_context(room_id);
+
+	// reset lazy loading state on initial sync.
+	// do this even if lazy loading is disabled so future lazy loads
+	// will have the correct members.
+	if sync_context.last_sync_end_count.is_none() {
+		services
+			.rooms
+			.lazy_loading
+			.reset(lazy_loading_context)
+			.await;
+	}
+
+	// filter the input members through `retain_lazy_members`, which
+	// contains the actual lazy loading logic.
+	let lazily_loaded_members =
+		OptionFuture::from(sync_context.lazy_loading_enabled().then(|| {
+			services
+				.rooms
+				.lazy_loading
+				.retain_lazy_members(timeline_members.collect(), lazy_loading_context)
+		}))
+		.await;
+
+	lazily_loaded_members
+}

src/api/client/sync/v3/state.rs

@@ -0,0 +1,280 @@
+use std::{collections::BTreeSet, ops::ControlFlow};
+
+use conduwuit::{
+	Result, at, is_equal_to,
+	matrix::{
+		Event,
+		pdu::{PduCount, PduEvent},
+	},
+	utils::{
+		BoolExt, IterStream, ReadyExt, TryFutureExtExt,
+		stream::{BroadbandExt, TryIgnore},
+	},
+};
+use conduwuit_service::{
+	Services,
+	rooms::{lazy_loading::MemberSet, short::ShortStateHash},
+};
+use futures::{FutureExt, StreamExt};
+use itertools::Itertools;
+use ruma::{OwnedEventId, RoomId, UserId, events::StateEventType};
+use service::rooms::short::ShortEventId;
+use tracing::trace;
+
+use crate::client::TimelinePdus;
+
+/// Calculate the state events to include in an initial sync response.
+///
+/// If lazy-loading is enabled (`lazily_loaded_members` is Some), the returned
+/// Vec will include the membership events of exclusively the members in
+/// `lazily_loaded_members`.
+#[tracing::instrument(
+	name = "initial",
+	level = "trace",
+	skip_all,
+	fields(current_shortstatehash)
+)]
+#[allow(clippy::too_many_arguments)]
+pub(super) async fn build_state_initial(
+	services: &Services,
+	sender_user: &UserId,
+	timeline_start_shortstatehash: ShortStateHash,
+	lazily_loaded_members: Option<&MemberSet>,
+) -> Result<Vec<PduEvent>> {
+	// load the keys and event IDs of the state events at the start of the timeline
+	let (shortstatekeys, event_ids): (Vec<_>, Vec<_>) = services
+		.rooms
+		.state_accessor
+		.state_full_ids(timeline_start_shortstatehash)
+		.unzip()
+		.await;
+
+	trace!("performing initial sync of {} state events", event_ids.len());
+
+	services
+		.rooms
+		.short
+		// look up the full state keys
+		.multi_get_statekey_from_short(shortstatekeys.into_iter().stream())
+		.zip(event_ids.into_iter().stream())
+		.ready_filter_map(|item| Some((item.0.ok()?, item.1)))
+		.ready_filter_map(|((event_type, state_key), event_id)| {
+			if let Some(lazily_loaded_members) = lazily_loaded_members {
+				/*
+				if lazy loading is enabled, filter out membership events which aren't for a user
+				included in `lazily_loaded_members` or for the user requesting the sync.
+				*/
+				let event_is_redundant = event_type == StateEventType::RoomMember
+					&& state_key.as_str().try_into().is_ok_and(|user_id: &UserId| {
+						sender_user != user_id && !lazily_loaded_members.contains(user_id)
+					});
+
+				event_is_redundant.or_some(event_id)
+			} else {
+				Some(event_id)
+			}
+		})
+		.broad_filter_map(|event_id: OwnedEventId| async move {
+			services.rooms.timeline.get_pdu(&event_id).await.ok()
+		})
+		.collect()
+		.map(Ok)
+		.await
+}
+
+/// Calculate the state events to include in an incremental sync response.
+///
+/// If lazy-loading is enabled (`lazily_loaded_members` is Some), the returned
+/// Vec will include the membership events of all the members in
+/// `lazily_loaded_members`.
+#[tracing::instrument(name = "incremental", level = "trace", skip_all)]
+#[allow(clippy::too_many_arguments)]
+pub(super) async fn build_state_incremental<'a>(
+	services: &Services,
+	sender_user: &'a UserId,
+	room_id: &RoomId,
+	last_sync_end_count: PduCount,
+	last_sync_end_shortstatehash: ShortStateHash,
+	timeline_start_shortstatehash: ShortStateHash,
+	timeline_end_shortstatehash: ShortStateHash,
+	timeline: &TimelinePdus,
+	lazily_loaded_members: Option<&'a MemberSet>,
+) -> Result<Vec<PduEvent>> {
+	/*
+	NB: a limited sync is one where `timeline.limited == true`. Synapse calls this a "gappy" sync internally.
+
+	The algorithm implemented in this function is, currently, quite different from the algorithm vaguely described
+	by the Matrix specification. This is because the specification's description of the `state` property does not accurately
+	reflect how Synapse behaves, and therefore how client SDKs behave. Notable differences include:
+	1. We do not compute the delta using the naive approach of "every state event from the end of the last sync
+	   up to the start of this sync's timeline". see below for details.
+	2. If lazy-loading is enabled, we include lazily-loaded membership events. The specific users to include are determined
+	   elsewhere and supplied to this function in the `lazily_loaded_members` parameter.
+	*/
+
+	/*
+	the `state` property of an incremental sync which isn't limited are _usually_ empty.
+	(note: the specification says that the `state` property is _always_ empty for limited syncs, which is incorrect.)
+	however, if an event in the timeline (`timeline.pdus`) merges a split in the room's DAG (i.e. has multiple `prev_events`),
+	the state at the _end_ of the timeline may include state events which were merged in and don't exist in the state
+	at the _start_ of the timeline. because this is uncommon, we check here to see if any events in the timeline
+	merged a split in the DAG.
+
+	see: https://github.com/element-hq/synapse/issues/16941
+	*/
+
+	let timeline_is_linear = timeline.pdus.is_empty() || {
+		let last_pdu_of_last_sync = services
+			.rooms
+			.timeline
+			.pdus_rev(Some(sender_user), room_id, Some(last_sync_end_count.saturating_add(1)))
+			.boxed()
+			.next()
+			.await
+			.transpose()
+			.expect("last sync should have had some PDUs")
+			.map(at!(1));
+
+		// make sure the prev_events of each pdu in the timeline refer only to the
+		// previous pdu
+		timeline
+			.pdus
+			.iter()
+			.try_fold(last_pdu_of_last_sync.map(|pdu| pdu.event_id), |prev_event_id, (_, pdu)| {
+				if let Ok(pdu_prev_event_id) = pdu.prev_events.iter().exactly_one() {
+					if prev_event_id
+						.as_ref()
+						.is_none_or(is_equal_to!(pdu_prev_event_id))
+					{
+						return ControlFlow::Continue(Some(pdu_prev_event_id.to_owned()));
+					}
+				}
+
+				trace!(
+					"pdu {:?} has split prev_events (expected {:?}): {:?}",
+					pdu.event_id, prev_event_id, pdu.prev_events
+				);
+				ControlFlow::Break(())
+			})
+			.is_continue()
+	};
+
+	if timeline_is_linear && !timeline.limited {
+		// if there are no splits in the DAG and the timeline isn't limited, then
+		// `state` will always be empty unless lazy loading is enabled.
+
+		if let Some(lazily_loaded_members) = lazily_loaded_members {
+			if !timeline.pdus.is_empty() {
+				// lazy loading is enabled, so we return the membership events which were
+				// requested by the caller.
+				let lazy_membership_events: Vec<_> = lazily_loaded_members
+					.iter()
+					.stream()
+					.broad_filter_map(|user_id| async move {
+						if user_id == sender_user {
+							return None;
+						}
+
+						services
+							.rooms
+							.state_accessor
+							.state_get(
+								timeline_start_shortstatehash,
+								&StateEventType::RoomMember,
+								user_id.as_str(),
+							)
+							.ok()
+							.await
+					})
+					.collect()
+					.await;
+
+				if !lazy_membership_events.is_empty() {
+					trace!(
+						"syncing lazy membership events for members: {:?}",
+						lazy_membership_events
+							.iter()
+							.map(|pdu| pdu.state_key().unwrap())
+							.collect::<Vec<_>>()
+					);
+				}
+				return Ok(lazy_membership_events);
+			}
+		}
+
+		// lazy loading is disabled, `state` is empty.
+		return Ok(vec![]);
+	}
+
+	/*
+	at this point, either the timeline is `limited` or the DAG has a split in it. this necessitates
+	computing the incremental state (which may be empty).
+
+	NOTE: this code path does not use the `lazy_membership_events` parameter. any changes to membership will be included
+	in the incremental state. therefore, the incremental state may include "redundant" membership events,
+	which we do not filter out because A. the spec forbids lazy-load filtering if the timeline is `limited`,
+	and B. DAG splits which require sending extra membership state events are (probably) uncommon enough that
+	the performance penalty is acceptable.
+	*/
+
+	trace!(?timeline_is_linear, ?timeline.limited, "computing state for incremental sync");
+
+	// fetch the shorteventids of state events in the timeline
+	let state_events_in_timeline: BTreeSet<ShortEventId> = services
+		.rooms
+		.short
+		.multi_get_or_create_shorteventid(timeline.pdus.iter().filter_map(|(_, pdu)| {
+			if pdu.state_key().is_some() {
+				Some(pdu.event_id.as_ref())
+			} else {
+				None
+			}
+		}))
+		.collect()
+		.await;
+
+	trace!("{} state events in timeline", state_events_in_timeline.len());
+
+	/*
+	fetch the state events which were added since the last sync.
+
+	specifically we fetch the difference between the state at the last sync and the state at the _end_
+	of the timeline, and then we filter out state events in the timeline itself using the shorteventids we fetched.
+	this is necessary to account for splits in the DAG, as explained above.
+	*/
+	let state_diff = services
+		.rooms
+		.short
+		.multi_get_eventid_from_short::<'_, OwnedEventId, _>(
+			services
+				.rooms
+				.state_accessor
+				.state_added((last_sync_end_shortstatehash, timeline_end_shortstatehash))
+				.await?
+				.stream()
+				.ready_filter_map(|(_, shorteventid)| {
+					if state_events_in_timeline.contains(&shorteventid) {
+						None
+					} else {
+						Some(shorteventid)
+					}
+				}),
+		)
+		.ignore_err();
+
+	// finally, fetch the PDU contents and collect them into a vec
+	let state_diff_pdus = state_diff
+		.broad_filter_map(|event_id| async move {
+			services
+				.rooms
+				.timeline
+				.get_non_outlier_pdu(&event_id)
+				.await
+				.ok()
+		})
+		.collect::<Vec<_>>()
+		.await;
+
+	trace!(?state_diff_pdus, "collected state PDUs for incremental sync");
+	Ok(state_diff_pdus)
+}

src/api/client/sync/v4.rs

@@ -1,840 +0,0 @@
-use std::{
-	cmp::{self, Ordering},
-	collections::{BTreeMap, BTreeSet, HashMap, HashSet},
-	time::Duration,
-};
-
-use axum::extract::State;
-use conduwuit::{
-	Err, Error, Event, PduCount, Result, at, debug, error, extract_variant,
-	matrix::TypeStateKey,
-	utils::{
-		BoolExt, IterStream, ReadyExt, TryFutureExtExt,
-		math::{ruma_from_usize, usize_from_ruma, usize_from_u64_truncated},
-	},
-	warn,
-};
-use conduwuit_service::{
-	Services,
-	rooms::read_receipt::pack_receipts,
-	sync::{into_db_key, into_snake_key},
-};
-use futures::{FutureExt, StreamExt, TryFutureExt};
-use ruma::{
-	MilliSecondsSinceUnixEpoch, OwnedEventId, OwnedRoomId, RoomId, UInt, UserId,
-	api::client::sync::sync_events::{
-		self, DeviceLists, UnreadNotificationsCount,
-		v4::{SlidingOp, SlidingSyncRoomHero},
-	},
-	directory::RoomTypeFilter,
-	events::{
-		AnyRawAccountDataEvent, AnySyncEphemeralRoomEvent, StateEventType,
-		TimelineEventType::*,
-		room::member::{MembershipState, RoomMemberEventContent},
-	},
-	serde::Raw,
-	uint,
-};
-
-use super::{load_timeline, share_encrypted_room};
-use crate::{
-	Ruma,
-	client::{DEFAULT_BUMP_TYPES, ignored_filter},
-};
-
-type TodoRooms = BTreeMap<OwnedRoomId, (BTreeSet<TypeStateKey>, usize, u64)>;
-const SINGLE_CONNECTION_SYNC: &str = "single_connection_sync";
-
-#[allow(clippy::cognitive_complexity)]
-/// POST `/_matrix/client/unstable/org.matrix.msc3575/sync`
-///
-/// Sliding Sync endpoint (future endpoint: `/_matrix/client/v4/sync`)
-pub(crate) async fn sync_events_v4_route(
-	State(services): State<crate::State>,
-	body: Ruma<sync_events::v4::Request>,
-) -> Result<sync_events::v4::Response> {
-	debug_assert!(DEFAULT_BUMP_TYPES.is_sorted(), "DEFAULT_BUMP_TYPES is not sorted");
-	let sender_user = body.sender_user.as_ref().expect("user is authenticated");
-	let sender_device = body.sender_device.as_ref().expect("user is authenticated");
-	let mut body = body.body;
-
-	// Setup watchers, so if there's no response, we can wait for them
-	let watcher = services.sync.watch(sender_user, sender_device);
-
-	let next_batch = services.globals.next_count()?;
-
-	let conn_id = body
-		.conn_id
-		.clone()
-		.unwrap_or_else(|| SINGLE_CONNECTION_SYNC.to_owned());
-
-	let globalsince = body
-		.pos
-		.as_ref()
-		.and_then(|string| string.parse().ok())
-		.unwrap_or(0);
-
-	let db_key = into_db_key(sender_user, sender_device, conn_id.clone());
-	if globalsince != 0 && !services.sync.remembered(&db_key) {
-		debug!("Restarting sync stream because it was gone from the database");
-		return Err!(Request(UnknownPos("Connection data lost since last time")));
-	}
-
-	if globalsince == 0 {
-		services.sync.forget_sync_request_connection(&db_key);
-	}
-
-	// Get sticky parameters from cache
-	let snake_key = into_snake_key(sender_user, sender_device, conn_id.clone());
-	let known_rooms = services
-		.sync
-		.update_sync_request_with_cache(&snake_key, &mut body);
-
-	let all_joined_rooms: Vec<_> = services
-		.rooms
-		.state_cache
-		.rooms_joined(sender_user)
-		.map(ToOwned::to_owned)
-		.collect()
-		.await;
-
-	let all_invited_rooms: Vec<_> = services
-		.rooms
-		.state_cache
-		.rooms_invited(sender_user)
-		.map(|r| r.0)
-		.collect()
-		.await;
-
-	let all_knocked_rooms: Vec<_> = services
-		.rooms
-		.state_cache
-		.rooms_knocked(sender_user)
-		.map(|r| r.0)
-		.collect()
-		.await;
-
-	let all_invited_rooms: Vec<&RoomId> = all_invited_rooms.iter().map(AsRef::as_ref).collect();
-	let all_knocked_rooms: Vec<&RoomId> = all_knocked_rooms.iter().map(AsRef::as_ref).collect();
-
-	let all_rooms: Vec<&RoomId> = all_joined_rooms
-		.iter()
-		.map(AsRef::as_ref)
-		.chain(all_invited_rooms.iter().map(AsRef::as_ref))
-		.chain(all_knocked_rooms.iter().map(AsRef::as_ref))
-		.collect();
-
-	let all_joined_rooms = all_joined_rooms.iter().map(AsRef::as_ref).collect();
-	let all_invited_rooms = all_invited_rooms.iter().map(AsRef::as_ref).collect();
-
-	if body.extensions.to_device.enabled.unwrap_or(false) {
-		services
-			.users
-			.remove_to_device_events(sender_user, sender_device, globalsince)
-			.await;
-	}
-
-	let mut left_encrypted_users = HashSet::new(); // Users that have left any encrypted rooms the sender was in
-	let mut device_list_changes = HashSet::new();
-	let mut device_list_left = HashSet::new();
-
-	let mut receipts = sync_events::v4::Receipts { rooms: BTreeMap::new() };
-
-	let mut account_data = sync_events::v4::AccountData {
-		global: Vec::new(),
-		rooms: BTreeMap::new(),
-	};
-	if body.extensions.account_data.enabled.unwrap_or(false) {
-		account_data.global = services
-			.account_data
-			.changes_since(None, sender_user, globalsince, Some(next_batch))
-			.ready_filter_map(|e| extract_variant!(e, AnyRawAccountDataEvent::Global))
-			.collect()
-			.await;
-
-		if let Some(rooms) = body.extensions.account_data.rooms {
-			for room in rooms {
-				account_data.rooms.insert(
-					room.clone(),
-					services
-						.account_data
-						.changes_since(Some(&room), sender_user, globalsince, Some(next_batch))
-						.ready_filter_map(|e| extract_variant!(e, AnyRawAccountDataEvent::Room))
-						.collect()
-						.await,
-				);
-			}
-		}
-	}
-
-	if body.extensions.e2ee.enabled.unwrap_or(false) {
-		// Look for device list updates of this account
-		device_list_changes.extend(
-			services
-				.users
-				.keys_changed(sender_user, globalsince, None)
-				.map(ToOwned::to_owned)
-				.collect::<Vec<_>>()
-				.await,
-		);
-
-		for room_id in &all_joined_rooms {
-			let room_id: &&RoomId = room_id;
-			let Ok(current_shortstatehash) =
-				services.rooms.state.get_room_shortstatehash(room_id).await
-			else {
-				error!("Room {room_id} has no state");
-				continue;
-			};
-
-			let since_shortstatehash = services
-				.rooms
-				.user
-				.get_token_shortstatehash(room_id, globalsince)
-				.await
-				.ok();
-
-			let encrypted_room = services
-				.rooms
-				.state_accessor
-				.state_get(current_shortstatehash, &StateEventType::RoomEncryption, "")
-				.await
-				.is_ok();
-
-			if let Some(since_shortstatehash) = since_shortstatehash {
-				// Skip if there are only timeline changes
-				if since_shortstatehash == current_shortstatehash {
-					continue;
-				}
-
-				let since_encryption = services
-					.rooms
-					.state_accessor
-					.state_get(since_shortstatehash, &StateEventType::RoomEncryption, "")
-					.await;
-
-				let since_sender_member: Option<RoomMemberEventContent> = services
-					.rooms
-					.state_accessor
-					.state_get_content(
-						since_shortstatehash,
-						&StateEventType::RoomMember,
-						sender_user.as_str(),
-					)
-					.ok()
-					.await;
-
-				let joined_since_last_sync = since_sender_member
-					.as_ref()
-					.is_none_or(|member| member.membership != MembershipState::Join);
-
-				let new_encrypted_room = encrypted_room && since_encryption.is_err();
-
-				if encrypted_room {
-					let current_state_ids: HashMap<_, OwnedEventId> = services
-						.rooms
-						.state_accessor
-						.state_full_ids(current_shortstatehash)
-						.collect()
-						.await;
-
-					let since_state_ids: HashMap<_, _> = services
-						.rooms
-						.state_accessor
-						.state_full_ids(since_shortstatehash)
-						.collect()
-						.await;
-
-					for (key, id) in current_state_ids {
-						if since_state_ids.get(&key) != Some(&id) {
-							let Ok(pdu) = services.rooms.timeline.get_pdu(&id).await else {
-								error!("Pdu in state not found: {id}");
-								continue;
-							};
-							if pdu.kind == RoomMember {
-								if let Some(Ok(user_id)) =
-									pdu.state_key.as_deref().map(UserId::parse)
-								{
-									if user_id == sender_user {
-										continue;
-									}
-
-									let content: RoomMemberEventContent = pdu.get_content()?;
-									match content.membership {
-										| MembershipState::Join => {
-											// A new user joined an encrypted room
-											if !share_encrypted_room(
-												&services,
-												sender_user,
-												user_id,
-												Some(room_id),
-											)
-											.await
-											{
-												device_list_changes.insert(user_id.to_owned());
-											}
-										},
-										| MembershipState::Leave => {
-											// Write down users that have left encrypted rooms we
-											// are in
-											left_encrypted_users.insert(user_id.to_owned());
-										},
-										| _ => {},
-									}
-								}
-							}
-						}
-					}
-					if joined_since_last_sync || new_encrypted_room {
-						// If the user is in a new encrypted room, give them all joined users
-						device_list_changes.extend(
-							services
-								.rooms
-								.state_cache
-								.room_members(room_id)
-								// Don't send key updates from the sender to the sender
-								.ready_filter(|&user_id| sender_user != user_id)
-								// Only send keys if the sender doesn't share an encrypted room with the target
-								// already
-								.filter_map(|user_id| {
-									share_encrypted_room(&services, sender_user, user_id, Some(room_id))
-										.map(|res| res.or_some(user_id.to_owned()))
-								})
-								.collect::<Vec<_>>()
-								.await,
-						);
-					}
-				}
-			}
-			// Look for device list updates in this room
-			device_list_changes.extend(
-				services
-					.users
-					.room_keys_changed(room_id, globalsince, None)
-					.map(|(user_id, _)| user_id)
-					.map(ToOwned::to_owned)
-					.collect::<Vec<_>>()
-					.await,
-			);
-		}
-
-		for user_id in left_encrypted_users {
-			let dont_share_encrypted_room =
-				!share_encrypted_room(&services, sender_user, &user_id, None).await;
-
-			// If the user doesn't share an encrypted room with the target anymore, we need
-			// to tell them
-			if dont_share_encrypted_room {
-				device_list_left.insert(user_id);
-			}
-		}
-	}
-
-	let mut lists = BTreeMap::new();
-	let mut todo_rooms: TodoRooms = BTreeMap::new(); // and required state
-
-	for (list_id, list) in &body.lists {
-		let active_rooms = match list.filters.clone().and_then(|f| f.is_invite) {
-			| Some(true) => &all_invited_rooms,
-			| Some(false) => &all_joined_rooms,
-			| None => &all_rooms,
-		};
-
-		let active_rooms = match list.filters.clone().map(|f| f.not_room_types) {
-			| Some(filter) if filter.is_empty() => active_rooms.clone(),
-			| Some(value) => filter_rooms(&services, active_rooms, &value, true).await,
-			| None => active_rooms.clone(),
-		};
-
-		let active_rooms = match list.filters.clone().map(|f| f.room_types) {
-			| Some(filter) if filter.is_empty() => active_rooms.clone(),
-			| Some(value) => filter_rooms(&services, &active_rooms, &value, false).await,
-			| None => active_rooms,
-		};
-
-		let mut new_known_rooms: BTreeSet<OwnedRoomId> = BTreeSet::new();
-
-		let ranges = list.ranges.clone();
-		lists.insert(list_id.clone(), sync_events::v4::SyncList {
-			ops: ranges
-				.into_iter()
-				.map(|mut r| {
-					r.0 = r.0.clamp(
-						uint!(0),
-						UInt::try_from(active_rooms.len().saturating_sub(1)).unwrap_or(UInt::MAX),
-					);
-					r.1 = r.1.clamp(
-						r.0,
-						UInt::try_from(active_rooms.len().saturating_sub(1)).unwrap_or(UInt::MAX),
-					);
-
-					let room_ids = if !active_rooms.is_empty() {
-						active_rooms[usize_from_ruma(r.0)..=usize_from_ruma(r.1)].to_vec()
-					} else {
-						Vec::new()
-					};
-
-					new_known_rooms.extend(room_ids.clone().into_iter().map(ToOwned::to_owned));
-					for room_id in &room_ids {
-						let todo_room = todo_rooms.entry((*room_id).to_owned()).or_insert((
-							BTreeSet::new(),
-							0_usize,
-							u64::MAX,
-						));
-
-						let limit: usize = list
-							.room_details
-							.timeline_limit
-							.map(u64::from)
-							.map_or(10, usize_from_u64_truncated)
-							.min(100);
-
-						todo_room.0.extend(
-							list.room_details
-								.required_state
-								.iter()
-								.map(|(ty, sk)| (ty.clone(), sk.as_str().into())),
-						);
-
-						todo_room.1 = todo_room.1.max(limit);
-						// 0 means unknown because it got out of date
-						todo_room.2 = todo_room.2.min(
-							known_rooms
-								.get(list_id.as_str())
-								.and_then(|k| k.get(*room_id))
-								.copied()
-								.unwrap_or(0),
-						);
-					}
-					sync_events::v4::SyncOp {
-						op: SlidingOp::Sync,
-						range: Some(r),
-						index: None,
-						room_ids: room_ids.into_iter().map(ToOwned::to_owned).collect(),
-						room_id: None,
-					}
-				})
-				.collect(),
-			count: ruma_from_usize(active_rooms.len()),
-		});
-
-		if let Some(conn_id) = &body.conn_id {
-			let db_key = into_db_key(sender_user, sender_device, conn_id);
-			services.sync.update_sync_known_rooms(
-				&db_key,
-				list_id.clone(),
-				new_known_rooms,
-				globalsince,
-			);
-		}
-	}
-
-	let mut known_subscription_rooms = BTreeSet::new();
-	for (room_id, room) in &body.room_subscriptions {
-		if !services.rooms.metadata.exists(room_id).await
-			|| services.rooms.metadata.is_disabled(room_id).await
-			|| services.rooms.metadata.is_banned(room_id).await
-		{
-			continue;
-		}
-		let todo_room =
-			todo_rooms
-				.entry(room_id.clone())
-				.or_insert((BTreeSet::new(), 0_usize, u64::MAX));
-
-		let limit: usize = room
-			.timeline_limit
-			.map(u64::from)
-			.map_or(10, usize_from_u64_truncated)
-			.min(100);
-
-		todo_room.0.extend(
-			room.required_state
-				.iter()
-				.map(|(ty, sk)| (ty.clone(), sk.as_str().into())),
-		);
-		todo_room.1 = todo_room.1.max(limit);
-		// 0 means unknown because it got out of date
-		todo_room.2 = todo_room.2.min(
-			known_rooms
-				.get("subscriptions")
-				.and_then(|k| k.get(room_id))
-				.copied()
-				.unwrap_or(0),
-		);
-		known_subscription_rooms.insert(room_id.clone());
-	}
-
-	for r in body.unsubscribe_rooms {
-		known_subscription_rooms.remove(&r);
-		body.room_subscriptions.remove(&r);
-	}
-
-	if let Some(conn_id) = &body.conn_id {
-		let db_key = into_db_key(sender_user, sender_device, conn_id);
-		services.sync.update_sync_known_rooms(
-			&db_key,
-			"subscriptions".to_owned(),
-			known_subscription_rooms,
-			globalsince,
-		);
-	}
-
-	if let Some(conn_id) = body.conn_id.clone() {
-		let db_key = into_db_key(sender_user, sender_device, conn_id);
-		services
-			.sync
-			.update_sync_subscriptions(&db_key, body.room_subscriptions);
-	}
-
-	let mut rooms = BTreeMap::new();
-	for (room_id, (required_state_request, timeline_limit, roomsince)) in &todo_rooms {
-		let roomsincecount = PduCount::Normal(*roomsince);
-
-		let mut timestamp: Option<_> = None;
-		let mut invite_state = None;
-		let (timeline_pdus, limited);
-		let new_room_id: &RoomId = (*room_id).as_ref();
-		if all_invited_rooms.contains(&new_room_id) {
-			// TODO: figure out a timestamp we can use for remote invites
-			invite_state = services
-				.rooms
-				.state_cache
-				.invite_state(sender_user, room_id)
-				.await
-				.ok();
-
-			(timeline_pdus, limited) = (Vec::new(), true);
-		} else {
-			(timeline_pdus, limited) = match load_timeline(
-				&services,
-				sender_user,
-				room_id,
-				roomsincecount,
-				None,
-				*timeline_limit,
-			)
-			.await
-			{
-				| Ok(value) => value,
-				| Err(err) => {
-					warn!("Encountered missing timeline in {}, error {}", room_id, err);
-					continue;
-				},
-			};
-		}
-
-		account_data.rooms.insert(
-			room_id.to_owned(),
-			services
-				.account_data
-				.changes_since(Some(room_id), sender_user, *roomsince, Some(next_batch))
-				.ready_filter_map(|e| extract_variant!(e, AnyRawAccountDataEvent::Room))
-				.collect()
-				.await,
-		);
-
-		let last_privateread_update = services
-			.rooms
-			.read_receipt
-			.last_privateread_update(sender_user, room_id)
-			.await > *roomsince;
-
-		let private_read_event = if last_privateread_update {
-			services
-				.rooms
-				.read_receipt
-				.private_read_get(room_id, sender_user)
-				.await
-				.ok()
-		} else {
-			None
-		};
-
-		let mut vector: Vec<Raw<AnySyncEphemeralRoomEvent>> = services
-			.rooms
-			.read_receipt
-			.readreceipts_since(room_id, *roomsince)
-			.filter_map(|(read_user, _ts, v)| async move {
-				services
-					.users
-					.user_is_ignored(read_user, sender_user)
-					.await
-					.or_some(v)
-			})
-			.collect()
-			.await;
-
-		if let Some(private_read_event) = private_read_event {
-			vector.push(private_read_event);
-		}
-
-		let receipt_size = vector.len();
-		receipts
-			.rooms
-			.insert(room_id.clone(), pack_receipts(Box::new(vector.into_iter())));
-
-		if roomsince != &0
-			&& timeline_pdus.is_empty()
-			&& account_data.rooms.get(room_id).is_some_and(Vec::is_empty)
-			&& receipt_size == 0
-		{
-			continue;
-		}
-
-		let prev_batch = timeline_pdus
-			.first()
-			.map_or(Ok::<_, Error>(None), |(pdu_count, _)| {
-				Ok(Some(match pdu_count {
-					| PduCount::Backfilled(_) => {
-						error!("timeline in backfill state?!");
-						"0".to_owned()
-					},
-					| PduCount::Normal(c) => c.to_string(),
-				}))
-			})?
-			.or_else(|| {
-				if roomsince != &0 {
-					Some(roomsince.to_string())
-				} else {
-					None
-				}
-			});
-
-		let room_events: Vec<_> = timeline_pdus
-			.iter()
-			.stream()
-			.filter_map(|item| ignored_filter(&services, item.clone(), sender_user))
-			.map(at!(1))
-			.map(Event::into_format)
-			.collect()
-			.await;
-
-		for (_, pdu) in timeline_pdus {
-			let ts = MilliSecondsSinceUnixEpoch(pdu.origin_server_ts);
-			if DEFAULT_BUMP_TYPES.binary_search(&pdu.kind).is_ok()
-				&& timestamp.is_none_or(|time| time <= ts)
-			{
-				timestamp = Some(ts);
-			}
-		}
-
-		let required_state = required_state_request
-			.iter()
-			.stream()
-			.filter_map(|state| async move {
-				services
-					.rooms
-					.state_accessor
-					.room_state_get(room_id, &state.0, &state.1)
-					.await
-					.map(Event::into_format)
-					.ok()
-			})
-			.collect()
-			.await;
-
-		// Heroes
-		let heroes: Vec<_> = services
-			.rooms
-			.state_cache
-			.room_members(room_id)
-			.ready_filter(|&member| member != sender_user)
-			.filter_map(|user_id| {
-				services
-					.rooms
-					.state_accessor
-					.get_member(room_id, user_id)
-					.map_ok(|memberevent| SlidingSyncRoomHero {
-						user_id: user_id.into(),
-						name: memberevent.displayname,
-						avatar: memberevent.avatar_url,
-					})
-					.ok()
-			})
-			.take(5)
-			.collect()
-			.await;
-
-		let name = match heroes.len().cmp(&(1_usize)) {
-			| Ordering::Greater => {
-				let firsts = heroes[1..]
-					.iter()
-					.map(|h| h.name.clone().unwrap_or_else(|| h.user_id.to_string()))
-					.collect::<Vec<_>>()
-					.join(", ");
-
-				let last = heroes[0]
-					.name
-					.clone()
-					.unwrap_or_else(|| heroes[0].user_id.to_string());
-
-				Some(format!("{firsts} and {last}"))
-			},
-			| Ordering::Equal => Some(
-				heroes[0]
-					.name
-					.clone()
-					.unwrap_or_else(|| heroes[0].user_id.to_string()),
-			),
-			| Ordering::Less => None,
-		};
-
-		let heroes_avatar = if heroes.len() == 1 {
-			heroes[0].avatar.clone()
-		} else {
-			None
-		};
-
-		rooms.insert(room_id.clone(), sync_events::v4::SlidingSyncRoom {
-			name: services
-				.rooms
-				.state_accessor
-				.get_name(room_id)
-				.await
-				.ok()
-				.or(name),
-			avatar: match heroes_avatar {
-				| Some(heroes_avatar) => ruma::JsOption::Some(heroes_avatar),
-				| _ => match services.rooms.state_accessor.get_avatar(room_id).await {
-					| ruma::JsOption::Some(avatar) => ruma::JsOption::from_option(avatar.url),
-					| ruma::JsOption::Null => ruma::JsOption::Null,
-					| ruma::JsOption::Undefined => ruma::JsOption::Undefined,
-				},
-			},
-			initial: Some(roomsince == &0),
-			is_dm: None,
-			invite_state,
-			unread_notifications: UnreadNotificationsCount {
-				highlight_count: Some(
-					services
-						.rooms
-						.user
-						.highlight_count(sender_user, room_id)
-						.await
-						.try_into()
-						.expect("notification count can't go that high"),
-				),
-				notification_count: Some(
-					services
-						.rooms
-						.user
-						.notification_count(sender_user, room_id)
-						.await
-						.try_into()
-						.expect("notification count can't go that high"),
-				),
-			},
-			timeline: room_events,
-			required_state,
-			prev_batch,
-			limited,
-			joined_count: Some(
-				services
-					.rooms
-					.state_cache
-					.room_joined_count(room_id)
-					.await
-					.unwrap_or(0)
-					.try_into()
-					.unwrap_or_else(|_| uint!(0)),
-			),
-			invited_count: Some(
-				services
-					.rooms
-					.state_cache
-					.room_invited_count(room_id)
-					.await
-					.unwrap_or(0)
-					.try_into()
-					.unwrap_or_else(|_| uint!(0)),
-			),
-			num_live: None, // Count events in timeline greater than global sync counter
-			timestamp,
-			heroes: Some(heroes),
-		});
-	}
-
-	if rooms.iter().all(|(id, r)| {
-		r.timeline.is_empty() && r.required_state.is_empty() && !receipts.rooms.contains_key(id)
-	}) {
-		// Hang a few seconds so requests are not spammed
-		// Stop hanging if new info arrives
-		let default = Duration::from_secs(30);
-		let duration = cmp::min(body.timeout.unwrap_or(default), default);
-		_ = tokio::time::timeout(duration, watcher).await;
-	}
-
-	Ok(sync_events::v4::Response {
-		initial: globalsince == 0,
-		txn_id: body.txn_id.clone(),
-		pos: next_batch.to_string(),
-		lists,
-		rooms,
-		extensions: sync_events::v4::Extensions {
-			to_device: if body.extensions.to_device.enabled.unwrap_or(false) {
-				Some(sync_events::v4::ToDevice {
-					events: services
-						.users
-						.get_to_device_events(
-							sender_user,
-							sender_device,
-							Some(globalsince),
-							Some(next_batch),
-						)
-						.collect()
-						.await,
-					next_batch: next_batch.to_string(),
-				})
-			} else {
-				None
-			},
-			e2ee: sync_events::v4::E2EE {
-				device_lists: DeviceLists {
-					changed: device_list_changes.into_iter().collect(),
-					left: device_list_left.into_iter().collect(),
-				},
-				device_one_time_keys_count: services
-					.users
-					.count_one_time_keys(sender_user, sender_device)
-					.await,
-				// Fallback keys are not yet supported
-				device_unused_fallback_key_types: None,
-			},
-			account_data,
-			receipts,
-			typing: sync_events::v4::Typing { rooms: BTreeMap::new() },
-		},
-		delta_token: None,
-	})
-}
-
-async fn filter_rooms<'a>(
-	services: &Services,
-	rooms: &[&'a RoomId],
-	filter: &[RoomTypeFilter],
-	negate: bool,
-) -> Vec<&'a RoomId> {
-	rooms
-		.iter()
-		.stream()
-		.filter_map(|r| async move {
-			let room_type = services.rooms.state_accessor.get_room_type(r).await;
-
-			if room_type.as_ref().is_err_and(|e| !e.is_not_found()) {
-				return None;
-			}
-
-			let room_type_filter = RoomTypeFilter::from(room_type.ok());
-
-			let include = if negate {
-				!filter.contains(&room_type_filter)
-			} else {
-				filter.is_empty() || filter.contains(&room_type_filter)
-			};
-
-			include.then_some(r)
-		})
-		.collect()
-		.await
-}

src/api/client/sync/v5.rs

@@ -1,6 +1,6 @@
 use std::{
 	cmp::{self, Ordering},
-	collections::{BTreeMap, BTreeSet, HashMap, HashSet},
+	collections::{BTreeMap, BTreeSet, HashMap, HashSet, VecDeque},
 	ops::Deref,
 	time::Duration,
 };
@@ -14,6 +14,7 @@
 		BoolExt, FutureBoolExt, IterStream, ReadyExt, TryFutureExtExt,
 		future::ReadyEqExt,
 		math::{ruma_from_usize, usize_from_ruma},
+		stream::WidebandExt,
 	},
 	warn,
 };
@@ -30,6 +31,7 @@
 	events::{
 		AnyRawAccountDataEvent, AnySyncEphemeralRoomEvent, StateEventType, TimelineEventType,
 		room::member::{MembershipState, RoomMemberEventContent},
+		typing::TypingEventContent,
 	},
 	serde::Raw,
 	uint,
@@ -38,7 +40,9 @@
 use super::share_encrypted_room;
 use crate::{
 	Ruma,
-	client::{DEFAULT_BUMP_TYPES, ignored_filter, sync::load_timeline},
+	client::{
+		DEFAULT_BUMP_TYPES, TimelinePdus, ignored_filter, is_ignored_invite, sync::load_timeline,
+	},
 };
 
 type SyncInfo<'a> = (&'a UserId, &'a DeviceId, u64, &'a sync_events::v5::Request);
@@ -106,6 +110,13 @@ pub(crate) async fn sync_events_v5_route(
 		.rooms
 		.state_cache
 		.rooms_invited(sender_user)
+		.wide_filter_map(async |(room_id, invite_state)| {
+			if is_ignored_invite(services, sender_user, &room_id).await {
+				None
+			} else {
+				Some((room_id, invite_state))
+			}
+		})
 		.map(|r| r.0)
 		.collect::<Vec<OwnedRoomId>>();
 
@@ -202,6 +213,9 @@ pub(crate) async fn sync_events_v5_route(
 		_ = tokio::time::timeout(duration, watcher).await;
 	}
 
+	let typing = collect_typing_events(services, sender_user, &body, &todo_rooms).await?;
+	response.extensions.typing = typing;
+
 	trace!(
 		rooms = ?response.rooms.len(),
 		account_data = ?response.extensions.account_data.rooms.len(),
@@ -285,6 +299,8 @@ async fn handle_lists<'a, Rooms, AllRooms>(
 	Rooms: Iterator<Item = &'a RoomId> + Clone + Send + 'a,
 	AllRooms: Iterator<Item = &'a RoomId> + Clone + Send + 'a,
 {
+	// TODO MSC4186: Implement remaining list filters: is_dm, is_encrypted,
+	// room_types.
 	for (list_id, list) in &body.lists {
 		let active_rooms: Vec<_> = match list.filters.as_ref().and_then(|f| f.is_invite) {
 			| None => all_rooms.clone().collect(),
@@ -312,6 +328,7 @@ async fn handle_lists<'a, Rooms, AllRooms>(
 
 		for mut range in ranges {
 			range.0 = uint!(0);
+			range.1 = range.1.checked_add(uint!(1)).unwrap_or(range.1);
 			range.1 = range
 				.1
 				.clamp(range.0, UInt::try_from(active_rooms.len()).unwrap_or(UInt::MAX));
@@ -400,13 +417,13 @@ async fn process_rooms<'a, Rooms>(
 				.await
 				.ok();
 
-			(timeline_pdus, limited) = (Vec::new(), true);
+			(timeline_pdus, limited) = (VecDeque::new(), true);
 		} else {
-			(timeline_pdus, limited) = match load_timeline(
+			TimelinePdus { pdus: timeline_pdus, limited } = match load_timeline(
 				services,
 				sender_user,
 				room_id,
-				roomsincecount,
+				Some(roomsincecount),
 				Some(PduCount::from(next_batch)),
 				*timeline_limit,
 			)
@@ -425,7 +442,7 @@ async fn process_rooms<'a, Rooms>(
 				room_id.to_owned(),
 				services
 					.account_data
-					.changes_since(Some(room_id), sender_user, *roomsince, Some(next_batch))
+					.changes_since(Some(room_id), sender_user, Some(*roomsince), Some(next_batch))
 					.ready_filter_map(|e| extract_variant!(e, AnyRawAccountDataEvent::Room))
 					.collect()
 					.await,
@@ -451,11 +468,11 @@ async fn process_rooms<'a, Rooms>(
 		let mut receipts: Vec<Raw<AnySyncEphemeralRoomEvent>> = services
 			.rooms
 			.read_receipt
-			.readreceipts_since(room_id, *roomsince)
+			.readreceipts_since(room_id, Some(*roomsince))
 			.filter_map(|(read_user, _ts, v)| async move {
 				services
 					.users
-					.user_is_ignored(read_user, sender_user)
+					.user_is_ignored(&read_user, sender_user)
 					.await
 					.or_some(v)
 			})
@@ -490,7 +507,7 @@ async fn process_rooms<'a, Rooms>(
 		}
 
 		let prev_batch = timeline_pdus
-			.first()
+			.front()
 			.map_or(Ok::<_, Error>(None), |(pdu_count, _)| {
 				Ok(Some(match pdu_count {
 					| PduCount::Backfilled(_) => {
@@ -663,6 +680,62 @@ async fn process_rooms<'a, Rooms>(
 	}
 	Ok(rooms)
 }
+
+async fn collect_typing_events(
+	services: &Services,
+	sender_user: &UserId,
+	body: &sync_events::v5::Request,
+	todo_rooms: &TodoRooms,
+) -> Result<sync_events::v5::response::Typing> {
+	if !body.extensions.typing.enabled.unwrap_or(false) {
+		return Ok(sync_events::v5::response::Typing::default());
+	}
+	let rooms: Vec<_> = body.extensions.typing.rooms.clone().unwrap_or_else(|| {
+		body.room_subscriptions
+			.keys()
+			.map(ToOwned::to_owned)
+			.collect()
+	});
+	let lists: Vec<_> = body
+		.extensions
+		.typing
+		.lists
+		.clone()
+		.unwrap_or_else(|| body.lists.keys().map(ToOwned::to_owned).collect::<Vec<_>>());
+
+	if rooms.is_empty() && lists.is_empty() {
+		return Ok(sync_events::v5::response::Typing::default());
+	}
+
+	let mut typing_response = sync_events::v5::response::Typing::default();
+	for (room_id, (_, _, roomsince)) in todo_rooms {
+		if services.rooms.typing.last_typing_update(room_id).await? <= *roomsince {
+			continue;
+		}
+
+		match services
+			.rooms
+			.typing
+			.typing_users_for_user(room_id, sender_user)
+			.await
+		{
+			| Ok(typing_users) => {
+				typing_response.rooms.insert(
+					room_id.to_owned(), // Already OwnedRoomId
+					Raw::new(&sync_events::v5::response::SyncTypingEvent {
+						content: TypingEventContent::new(typing_users),
+					})?,
+				);
+			},
+			| Err(e) => {
+				warn!(%room_id, "Failed to get typing events for room: {}", e);
+			},
+		}
+	}
+
+	Ok(typing_response)
+}
+
 async fn collect_account_data(
 	services: &Services,
 	(sender_user, _, globalsince, body): (&UserId, &DeviceId, u64, &sync_events::v5::Request),
@@ -678,7 +751,7 @@ async fn collect_account_data(
 
 	account_data.global = services
 		.account_data
-		.changes_since(None, sender_user, globalsince, None)
+		.changes_since(None, sender_user, Some(globalsince), None)
 		.ready_filter_map(|e| extract_variant!(e, AnyRawAccountDataEvent::Global))
 		.collect()
 		.await;
@@ -689,7 +762,7 @@ async fn collect_account_data(
 				room.clone(),
 				services
 					.account_data
-					.changes_since(Some(room), sender_user, globalsince, None)
+					.changes_since(Some(room), sender_user, Some(globalsince), None)
 					.ready_filter_map(|e| extract_variant!(e, AnyRawAccountDataEvent::Room))
 					.collect()
 					.await,
@@ -723,7 +796,7 @@ async fn collect_e2ee<'a, Rooms>(
 	device_list_changes.extend(
 		services
 			.users
-			.keys_changed(sender_user, globalsince, None)
+			.keys_changed(sender_user, Some(globalsince), None)
 			.map(ToOwned::to_owned)
 			.collect::<Vec<_>>()
 			.await,
@@ -859,7 +932,7 @@ async fn collect_e2ee<'a, Rooms>(
 		device_list_changes.extend(
 			services
 				.users
-				.room_keys_changed(room_id, globalsince, None)
+				.room_keys_changed(room_id, Some(globalsince), None)
 				.map(|(user_id, _)| user_id)
 				.map(ToOwned::to_owned)
 				.collect::<Vec<_>>()

src/api/client/unstable.rs

@@ -2,18 +2,14 @@
 
 use axum::extract::State;
 use axum_client_ip::InsecureClientIp;
-use conduwuit::{Err, Error, Result};
+use conduwuit::{Err, Result};
 use futures::StreamExt;
 use ruma::{
 	OwnedRoomId,
 	api::{
 		client::{
-			error::ErrorKind,
 			membership::mutual_rooms,
-			profile::{
-				delete_profile_key, delete_timezone_key, get_profile_key, get_timezone_key,
-				set_profile_key, set_timezone_key,
-			},
+			profile::{delete_profile_key, get_profile_key, set_profile_key},
 		},
 		federation,
 	},
@@ -60,62 +56,6 @@ pub(crate) async fn get_mutual_rooms_route(
 	})
 }
 
-/// # `DELETE /_matrix/client/unstable/uk.tcpip.msc4133/profile/:user_id/us.cloke.msc4175.tz`
-///
-/// Deletes the `tz` (timezone) of a user, as per MSC4133 and MSC4175.
-///
-/// - Also makes sure other users receive the update using presence EDUs
-pub(crate) async fn delete_timezone_key_route(
-	State(services): State<crate::State>,
-	body: Ruma<delete_timezone_key::unstable::Request>,
-) -> Result<delete_timezone_key::unstable::Response> {
-	let sender_user = body.sender_user();
-
-	if *sender_user != body.user_id && body.appservice_info.is_none() {
-		return Err!(Request(Forbidden("You cannot update the profile of another user")));
-	}
-
-	services.users.set_timezone(&body.user_id, None);
-
-	if services.config.allow_local_presence {
-		// Presence update
-		services
-			.presence
-			.ping_presence(&body.user_id, &PresenceState::Online)
-			.await?;
-	}
-
-	Ok(delete_timezone_key::unstable::Response {})
-}
-
-/// # `PUT /_matrix/client/unstable/uk.tcpip.msc4133/profile/:user_id/us.cloke.msc4175.tz`
-///
-/// Updates the `tz` (timezone) of a user, as per MSC4133 and MSC4175.
-///
-/// - Also makes sure other users receive the update using presence EDUs
-pub(crate) async fn set_timezone_key_route(
-	State(services): State<crate::State>,
-	body: Ruma<set_timezone_key::unstable::Request>,
-) -> Result<set_timezone_key::unstable::Response> {
-	let sender_user = body.sender_user();
-
-	if *sender_user != body.user_id && body.appservice_info.is_none() {
-		return Err!(Request(Forbidden("You cannot update the profile of another user")));
-	}
-
-	services.users.set_timezone(&body.user_id, body.tz.clone());
-
-	if services.config.allow_local_presence {
-		// Presence update
-		services
-			.presence
-			.ping_presence(&body.user_id, &PresenceState::Online)
-			.await?;
-	}
-
-	Ok(set_timezone_key::unstable::Response {})
-}
-
 /// # `PUT /_matrix/client/unstable/uk.tcpip.msc4133/profile/{user_id}/{field}`
 ///
 /// Updates the profile key-value field of a user, as per MSC4133.
@@ -150,19 +90,14 @@ pub(crate) async fn set_profile_key_route(
 		)));
 	};
 
-	if body
-		.kv_pair
-		.keys()
-		.any(|key| key.starts_with("u.") && !profile_key_value.is_string())
-	{
-		return Err!(Request(BadJson("u.* profile key fields must be strings")));
-	}
-
 	if body.kv_pair.keys().any(|key| key.len() > 128) {
 		return Err!(Request(BadJson("Key names cannot be longer than 128 bytes")));
 	}
 
 	if body.key_name == "displayname" {
+		let Some(display_name) = profile_key_value.as_str() else {
+			return Err!(Request(BadJson("displayname must be a string")));
+		};
 		let all_joined_rooms: Vec<OwnedRoomId> = services
 			.rooms
 			.state_cache
@@ -174,12 +109,15 @@ pub(crate) async fn set_profile_key_route(
 		update_displayname(
 			&services,
 			&body.user_id,
-			Some(profile_key_value.to_string()),
+			Some(display_name.to_owned()),
 			&all_joined_rooms,
 		)
 		.await;
 	} else if body.key_name == "avatar_url" {
-		let mxc = ruma::OwnedMxcUri::from(profile_key_value.to_string());
+		let Some(avatar_url) = profile_key_value.as_str() else {
+			return Err!(Request(BadJson("avatar_url must be a string")));
+		};
+		let mxc = ruma::OwnedMxcUri::from(avatar_url);
 
 		let all_joined_rooms: Vec<OwnedRoomId> = services
 			.rooms
@@ -268,70 +206,12 @@ pub(crate) async fn delete_profile_key_route(
 	Ok(delete_profile_key::unstable::Response {})
 }
 
-/// # `GET /_matrix/client/unstable/uk.tcpip.msc4133/profile/:user_id/us.cloke.msc4175.tz`
-///
-/// Returns the `timezone` of the user as per MSC4133 and MSC4175.
-///
-/// - If user is on another server and we do not have a local copy already fetch
-///   `timezone` over federation
-pub(crate) async fn get_timezone_key_route(
-	State(services): State<crate::State>,
-	body: Ruma<get_timezone_key::unstable::Request>,
-) -> Result<get_timezone_key::unstable::Response> {
-	if !services.globals.user_is_local(&body.user_id) {
-		// Create and update our local copy of the user
-		if let Ok(response) = services
-			.sending
-			.send_federation_request(
-				body.user_id.server_name(),
-				federation::query::get_profile_information::v1::Request {
-					user_id: body.user_id.clone(),
-					field: None, // we want the full user's profile to update locally as well
-				},
-			)
-			.await
-		{
-			if !services.users.exists(&body.user_id).await {
-				services.users.create(&body.user_id, None, None).await?;
-			}
-
-			services
-				.users
-				.set_displayname(&body.user_id, response.displayname.clone());
-
-			services
-				.users
-				.set_avatar_url(&body.user_id, response.avatar_url.clone());
-
-			services
-				.users
-				.set_blurhash(&body.user_id, response.blurhash.clone());
-
-			services
-				.users
-				.set_timezone(&body.user_id, response.tz.clone());
-
-			return Ok(get_timezone_key::unstable::Response { tz: response.tz });
-		}
-	}
-
-	if !services.users.exists(&body.user_id).await {
-		// Return 404 if this user doesn't exist and we couldn't fetch it over
-		// federation
-		return Err(Error::BadRequest(ErrorKind::NotFound, "Profile was not found."));
-	}
-
-	Ok(get_timezone_key::unstable::Response {
-		tz: services.users.timezone(&body.user_id).await.ok(),
-	})
-}
-
 /// # `GET /_matrix/client/unstable/uk.tcpip.msc4133/profile/{userId}/{field}}`
 ///
 /// Gets the profile key-value field of a user, as per MSC4133.
 ///
 /// - If user is on another server and we do not have a local copy already fetch
-///   `timezone` over federation
+///   the value over federation
 pub(crate) async fn get_profile_key_route(
 	State(services): State<crate::State>,
 	body: Ruma<get_profile_key::unstable::Request>,
@@ -367,10 +247,6 @@ pub(crate) async fn get_profile_key_route(
 				.users
 				.set_blurhash(&body.user_id, response.blurhash.clone());
 
-			services
-				.users
-				.set_timezone(&body.user_id, response.tz.clone());
-
 			match response.custom_profile_fields.get(&body.key_name) {
 				| Some(value) => {
 					profile_key_value.insert(body.key_name.clone(), value.clone());

src/api/client/unversioned.rs

@@ -52,13 +52,13 @@ pub(crate) async fn get_supported_versions_route(
 			("org.matrix.msc3026.busy_presence".to_owned(), true), /* busy presence status (https://github.com/matrix-org/matrix-spec-proposals/pull/3026) */
 			("org.matrix.msc3827".to_owned(), true), /* filtering of /publicRooms by room type (https://github.com/matrix-org/matrix-spec-proposals/pull/3827) */
 			("org.matrix.msc3952_intentional_mentions".to_owned(), true), /* intentional mentions (https://github.com/matrix-org/matrix-spec-proposals/pull/3952) */
-			("org.matrix.msc3575".to_owned(), true), /* sliding sync (https://github.com/matrix-org/matrix-spec-proposals/pull/3575/files#r1588877046) */
 			("org.matrix.msc3916.stable".to_owned(), true), /* authenticated media (https://github.com/matrix-org/matrix-spec-proposals/pull/3916) */
 			("org.matrix.msc4180".to_owned(), true), /* stable flag for 3916 (https://github.com/matrix-org/matrix-spec-proposals/pull/4180) */
 			("uk.tcpip.msc4133".to_owned(), true), /* Extending User Profile API with Key:Value Pairs (https://github.com/matrix-org/matrix-spec-proposals/pull/4133) */
 			("us.cloke.msc4175".to_owned(), true), /* Profile field for user time zone (https://github.com/matrix-org/matrix-spec-proposals/pull/4175) */
 			("org.matrix.simplified_msc3575".to_owned(), true), /* Simplified Sliding sync (https://github.com/matrix-org/matrix-spec-proposals/pull/4186) */
 			("uk.timedout.msc4323".to_owned(), true), /* agnostic suspend (https://github.com/matrix-org/matrix-spec-proposals/pull/4323) */
+			("org.matrix.msc4155".to_owned(), true), /* invite filtering (https://github.com/matrix-org/matrix-spec-proposals/pull/4155) */
 		]),
 	};
 

src/api/client/well_known.rs

@@ -78,7 +78,7 @@ pub(crate) async fn well_known_support(
 			while let Some(user_id) = stream.next().await {
 				// Skip server user
 				if *user_id == services.globals.server_user {
-					break;
+					continue;
 				}
 				contacts.push(Contact {
 					role: role_value.clone(),

src/api/router.rs

@@ -22,12 +22,9 @@
 pub fn build(router: Router<State>, server: &Server) -> Router<State> {
 	let config = &server.config;
 	let mut router = router
-        .ruma_route(&client::get_timezone_key_route)
         .ruma_route(&client::get_profile_key_route)
         .ruma_route(&client::set_profile_key_route)
         .ruma_route(&client::delete_profile_key_route)
-        .ruma_route(&client::set_timezone_key_route)
-        .ruma_route(&client::delete_timezone_key_route)
         .ruma_route(&client::appservice_ping)
 		.ruma_route(&client::get_supported_versions_route)
 		.ruma_route(&client::get_register_available_route)
@@ -146,7 +143,6 @@ pub fn build(router: Router<State>, server: &Server) -> Router<State> {
 				.put(client::send_state_event_for_empty_key_route),
 		)
 		.ruma_route(&client::sync_events_route)
-		.ruma_route(&client::sync_events_v4_route)
 		.ruma_route(&client::sync_events_v5_route)
 		.ruma_route(&client::get_context_route)
 		.ruma_route(&client::get_message_events_route)
@@ -229,6 +225,7 @@ pub fn build(router: Router<State>, server: &Server) -> Router<State> {
 			.ruma_route(&server::well_known_server)
 			.ruma_route(&server::get_content_route)
 			.ruma_route(&server::get_content_thumbnail_route)
+			.ruma_route(&server::get_edutypes_route)
 			.route("/_conduwuit/local_user_count", get(client::conduwuit_local_user_count))
 			.route("/_continuwuity/local_user_count", get(client::conduwuit_local_user_count));
 	} else {

src/api/router/auth.rs

@@ -20,9 +20,7 @@
 		client::{
 			directory::get_public_rooms,
 			error::ErrorKind,
-			profile::{
-				get_avatar_url, get_display_name, get_profile, get_profile_key, get_timezone_key,
-			},
+			profile::{get_avatar_url, get_display_name, get_profile, get_profile_key},
 			voip::get_turn_server_info,
 		},
 		federation::{authentication::XMatrix, openid::get_openid_userinfo},
@@ -89,8 +87,7 @@ pub(super) async fn auth(
 			| &get_profile::v3::Request::METADATA
 			| &get_profile_key::unstable::Request::METADATA
 			| &get_display_name::v3::Request::METADATA
-			| &get_avatar_url::v3::Request::METADATA
-			| &get_timezone_key::unstable::Request::METADATA => {
+			| &get_avatar_url::v3::Request::METADATA => {
 				if services.server.config.require_auth_for_profile_requests {
 					match token {
 						| Token::Appservice(_) | Token::User(_) => {

src/api/router/request.rs

@@ -34,6 +34,19 @@ pub(super) async fn from(
 
 	let max_body_size = services.server.config.max_request_size;
 
+	// Check if the Content-Length header is present and valid, saves us streaming
+	// the response into memory
+	if let Some(content_length) = parts.headers.get(http::header::CONTENT_LENGTH) {
+		if let Ok(content_length) = content_length
+			.to_str()
+			.map(|s| s.parse::<usize>().unwrap_or_default())
+		{
+			if content_length > max_body_size {
+				return Err(err!(Request(TooLarge("Request body too large"))));
+			}
+		}
+	}
+
 	let body = axum::body::to_bytes(body, max_body_size)
 		.await
 		.map_err(|e| err!(Request(TooLarge("Request body too large: {e}"))))?;

src/api/server/backfill.rs

@@ -2,7 +2,7 @@
 
 use axum::extract::State;
 use conduwuit::{
-	PduCount, Result,
+	Event, PduCount, Result,
 	utils::{IterStream, ReadyExt, stream::TryTools},
 };
 use futures::{FutureExt, StreamExt, TryStreamExt};
@@ -68,7 +68,7 @@ pub(crate) async fn get_backfill_route(
 				Ok(services
 					.rooms
 					.state_accessor
-					.server_can_see_event(body.origin(), &pdu.room_id, &pdu.event_id)
+					.server_can_see_event(body.origin(), &pdu.room_id_or_hash(), &pdu.event_id)
 					.await
 					.then_some(pdu))
 			})

src/api/server/edutypes.rs

@@ -0,0 +1,19 @@
+use axum::extract::State;
+use conduwuit::Result;
+use ruma::api::federation::edutypes::get_edutypes;
+
+use crate::Ruma;
+
+/// # `GET /_matrix/federation/v1/edutypes`
+///
+/// Lists EDU types we wish to receive
+pub(crate) async fn get_edutypes_route(
+	State(services): State<crate::State>,
+	_body: Ruma<get_edutypes::unstable::Request>,
+) -> Result<get_edutypes::unstable::Response> {
+	Ok(get_edutypes::unstable::Response {
+		typing: services.config.allow_incoming_typing,
+		presence: services.config.allow_incoming_presence,
+		receipt: services.config.allow_incoming_read_receipts,
+	})
+}