Bump 1.4.7

Signed-off-by: Jason Volk <jason@zemos.net>

.github/workflows/lint.yml

@@ -106,7 +106,6 @@ jobs:
       excludes: ${{inputs.excludes}}
       includes: ${{inputs.includes}}
 
-  # disabled due to excessive build time issue installing cargo lychee
   lychee:
     if: >
       !failure() && !cancelled()
@@ -114,7 +113,6 @@ jobs:
       && contains(fromJSON(inputs.feat_sets), fromJSON('["all"]')[0])
       && contains(fromJSON(inputs.rust_toolchains), fromJSON('["nightly"]')[0])
       && contains(fromJSON(inputs.sys_targets), fromJSON('["x86_64-v1-linux-gnu"]')[0])
-      && false
 
     name: Lychee
     uses: ./.github/workflows/bake.yml

.github/workflows/main.yml

@@ -20,13 +20,13 @@ jobs:
     name: Init
     runs-on: ${{matrix.runner}}
     env:
-      default_cargo_profiles: '["test", "release"]'
+      default_cargo_profiles: '["test", "bench", "release"]'
       default_feat_sets: '["none", "default", "all"]'
       default_rust_toolchains: '["nightly", "stable"]'
       default_sys_names: '["debian"]'
       default_sys_versions: '["testing-slim"]'
       default_rust_targets: '["x86_64-unknown-linux-gnu"]'
-      default_sys_targets: '["x86_64-v1-linux-gnu"]'
+      default_sys_targets: '["x86_64-v1-linux-gnu", "x86_64-v3-linux-gnu"]'
       default_machines: '["X64"]'
 
     outputs:
@@ -38,14 +38,18 @@ jobs:
       sys_targets: ${{vars.SYS_TARGETS || env.default_sys_targets}}
       sys_versions: ${{vars.SYS_VERSIONS || env.default_sys_versions}}
       machines: ${{vars.MACHINES || env.default_machines}}
-      package: ${{vars.PACKAGE || !contains(github.ref, 'refs/pull/')}}
-      publish: ${{vars.PUBLISH || !contains(github.ref, 'refs/pull/')}}
-      build_pkgs: ${{vars.BUILD_PKGS || github.ref == 'refs/heads/main' || contains(github.ref, 'tags/v')}}
-      check_pkgs: ${{vars.CHECK_PKGS || 'false'}}
-      complement: ${{vars.COMPLEMENT || 'true'}}
+      package: ${{vars.PACKAGE != 'false'}}
+      publish: ${{vars.PUBLISH != 'false'}}
+      build_nix: ${{vars.BUILD_NIX != 'false'}}
+      build_pkgs: ${{vars.BUILD_PKGS || github.ref == 'refs/heads/main' || github.ref == 'refs/heads/test' || contains(github.ref, 'tags/v')}}
+      check_pkgs: ${{vars.CHECK_PKGS || github.ref == 'refs/heads/test'}}
+      complement: ${{vars.COMPLEMENT != 'false'}}
       complement_runner: 'het'
       docker_repo: ${{vars.DOCKER_REPO}}
       release_url: ${{steps.release.outputs.upload_url}}
+      pages_url: 'https://matrix-construct.github.io/tuwunel/'
+      rust_sdk_integ: ${{vars.RUST_SDK_INTEGRATION != 'false'}}
+      head_msg: ${{github.event.head_commit.message || github.event.workflow_run.head_commit.message}}
 
     strategy:
       fail-fast: true
@@ -58,8 +62,8 @@ jobs:
     - name: Initialize Builder
       env:
         runner: ${{matrix.runner}}
-        reserved_space: '{"het": "128GB", "aws": "48GB", "gcp": "160GB"}'
-        max_used_space: '{"het": "256GB", "aws": "64GB", "gcp": "192GB"}'
+        reserved_space: '{"het": "192GB", "aws": "48GB", "gcp": "160GB"}'
+        max_used_space: '{"het": "384GB", "aws": "64GB", "gcp": "192GB"}'
       run: |
         set +e
         docker buildx inspect "${GITHUB_ACTOR}"
@@ -114,6 +118,8 @@ jobs:
     if: >
       !failure() && !cancelled()
       && !contains(github.ref, 'refs/tags/v')
+      && !contains(needs.init.outputs.head_msg, '[ci no lint]')
+      && !contains(needs.init.outputs.head_msg, '[ci only it]')
 
     name: Lint
     needs: [init] #needs: [init, deps]
@@ -132,6 +138,7 @@ jobs:
           {"cargo_profile": "test", "feat_set": "logging"},
           {"cargo_profile": "test", "rust_toolchain": "stable", "feat_set": "none"},
           {"cargo_profile": "test", "rust_target": "aarch64-unknown-linux-gnu"},
+          {"cargo_profile": "bench"},
           {"cargo_profile": "release", "rust_toolchain": "nightly", "feat_set": "none"},
           {"cargo_profile": "release", "rust_toolchain": "nightly", "feat_set": "default"},
           {"cargo_profile": "release", "rust_toolchain": "nightly", "feat_set": "logging"},
@@ -161,11 +168,15 @@ jobs:
     if: >
       !failure() && !cancelled()
       && !contains(github.ref, 'refs/tags/v')
+      && !contains(needs.init.outputs.head_msg, '[ci no test]')
 
     name: Test
     needs: [init, lint]
     uses: ./.github/workflows/test.yml
     with:
+      head_msg: ${{needs.init.outputs.head_msg}}
+      build_nix: ${{fromJSON(needs.init.outputs.build_nix)}}
+      rust_sdk_integ: ${{fromJSON(needs.init.outputs.rust_sdk_integ)}}
       complement: ${{fromJSON(needs.init.outputs.complement)}}
       complement_runner: ${{needs.init.outputs.complement_runner}}
       cargo_profiles: ${{needs.init.outputs.cargo_profiles}}
@@ -178,7 +189,6 @@ jobs:
       machines: ${{needs.init.outputs.machines}}
       excludes: >
         [
-          {"feat_set": "logging"},
           {"cargo_profile": "test", "rust_toolchain": "stable", "feat_set": "none"},
           {"cargo_profile": "test", "rust_target": "aarch64-unknown-linux-gnu"},
           {"cargo_profile": "release-debuginfo"},
@@ -212,14 +222,19 @@ jobs:
     if: >
       !failure() && !cancelled()
       && needs.init.outputs.package
+      && !contains(github.ref, 'refs/pull')
+      && !contains(needs.init.outputs.head_msg, '[ci only it]')
+      && !contains(needs.init.outputs.head_msg, '[ci no build]')
+      && !contains(needs.init.outputs.head_msg, '[ci no package]')
 
     name: Package
-    needs: [init, lint]
+    needs: [init, test]
     uses: ./.github/workflows/package.yml
     with:
       release_url: ${{needs.init.outputs.release_url}}
       check_pkgs: ${{needs.init.outputs.check_pkgs}}
       build_pkgs: ${{needs.init.outputs.build_pkgs}}
+      build_nix: ${{fromJSON(needs.init.outputs.build_nix)}}
       cargo_profiles: ${{needs.init.outputs.cargo_profiles}}
       feat_sets: ${{needs.init.outputs.feat_sets}}
       rust_toolchains: ${{needs.init.outputs.rust_toolchains}}
@@ -232,6 +247,7 @@ jobs:
         [
           {"feat_set": "none"},
           {"cargo_profile": "test"},
+          {"cargo_profile": "bench"},
           {"cargo_profile": "release-native"},
           {"cargo_profile": "release-debuginfo", "feat_set": "default"},
           {"cargo_profile": "release-debuginfo", "feat_set": "logging"},
@@ -244,6 +260,7 @@ jobs:
           {"cargo_profile": "release", "rust_toolchain": "nightly"},
           {"rust_target": "aarch64-unknown-linux-gnu", "feat_set": "default"},
           {"rust_target": "aarch64-unknown-linux-gnu", "feat_set": "logging"},
+          {"rust_target": "aarch64-unknown-linux-gnu", "bake_target": "nix"},
           {"rust_target": "aarch64-unknown-linux-gnu", "sys_target": "x86_64-v1-linux-gnu"},
           {"rust_target": "aarch64-unknown-linux-gnu", "sys_target": "x86_64-v2-linux-gnu"},
           {"rust_target": "aarch64-unknown-linux-gnu", "sys_target": "x86_64-v3-linux-gnu"},
@@ -254,18 +271,26 @@ jobs:
           {"sys_target": "x86_64-v2-linux-gnu", "machine": "ARM64"},
           {"sys_target": "x86_64-v2-linux-gnu", "feat_set": "default"},
           {"sys_target": "x86_64-v2-linux-gnu", "feat_set": "logging"},
+          {"sys_target": "x86_64-v2-linux-gnu", "bake_target": "nix"},
           {"sys_target": "x86_64-v3-linux-gnu", "machine": "ARM64"},
           {"sys_target": "x86_64-v3-linux-gnu", "feat_set": "default"},
           {"sys_target": "x86_64-v3-linux-gnu", "feat_set": "logging"},
+          {"sys_target": "x86_64-v3-linux-gnu", "bake_target": "nix"},
           {"sys_target": "x86_64-v4-linux-gnu", "machine": "ARM64"},
           {"sys_target": "x86_64-v4-linux-gnu", "feat_set": "default"},
           {"sys_target": "x86_64-v4-linux-gnu", "feat_set": "logging"},
+          {"sys_target": "x86_64-v4-linux-gnu", "bake_target": "nix"},
         ]
 
   publish:
     if: >
       !failure() && !cancelled()
       && needs.init.outputs.publish
+      && !contains(github.ref, 'refs/pull')
+      && !contains(needs.init.outputs.head_msg, '[ci only it]')
+      && !contains(needs.init.outputs.head_msg, '[ci no build]')
+      && !contains(needs.init.outputs.head_msg, '[ci no package]')
+      && !contains(needs.init.outputs.head_msg, '[ci no publish]')
 
     name: Publish
     needs: [init, test, package]
@@ -273,6 +298,7 @@ jobs:
     with:
       docker_repo: ${{needs.init.outputs.docker_repo}}
       release_url: ${{needs.init.outputs.release_url}}
+      pages_url: ${{needs.init.outputs.pages_url}}
       cargo_profiles: ${{needs.init.outputs.cargo_profiles}}
       feat_sets: ${{needs.init.outputs.feat_sets}}
       rust_toolchains: ${{needs.init.outputs.rust_toolchains}}
@@ -284,12 +310,13 @@ jobs:
       excludes: >
         [
           {"feat_set": "none"},
-          {"feat_set": "logging"},
+          {"feat_set": "default"},
           {"cargo_profile": "test"},
+          {"cargo_profile": "bench"},
           {"cargo_profile": "release-debuginfo"},
           {"cargo_profile": "release-native"},
-          {"cargo_profile": "release", "rust_toolchain": "nightly"},
-          {"rust_target": "aarch64-unknown-linux-gnu", "feat_set": "default"},
+          {"rust_toolchain": "nightly"},
+          {"rust_target": "aarch64-unknown-linux-gnu", "feat_set": "logging"},
           {"rust_target": "aarch64-unknown-linux-gnu", "sys_target": "x86_64-v1-linux-gnu"},
           {"rust_target": "aarch64-unknown-linux-gnu", "sys_target": "x86_64-v2-linux-gnu"},
           {"rust_target": "aarch64-unknown-linux-gnu", "sys_target": "x86_64-v3-linux-gnu"},
@@ -298,11 +325,11 @@ jobs:
           {"sys_target": "aarch64-v8-linux-gnu", "machine": "X64"},
           {"sys_target": "x86_64-v1-linux-gnu", "machine": "ARM64"},
           {"sys_target": "x86_64-v2-linux-gnu", "machine": "ARM64"},
-          {"sys_target": "x86_64-v2-linux-gnu", "feat_set": "default"},
+          {"sys_target": "x86_64-v2-linux-gnu", "feat_set": "logging"},
           {"sys_target": "x86_64-v3-linux-gnu", "machine": "ARM64"},
-          {"sys_target": "x86_64-v3-linux-gnu", "feat_set": "default"},
+          {"sys_target": "x86_64-v3-linux-gnu", "feat_set": "logging"},
           {"sys_target": "x86_64-v4-linux-gnu", "machine": "ARM64"},
-          {"sys_target": "x86_64-v4-linux-gnu", "feat_set": "default"},
+          {"sys_target": "x86_64-v4-linux-gnu", "feat_set": "logging"},
         ]
 
     secrets:

.github/workflows/package.yml

@@ -44,6 +44,9 @@ on:
       check_pkgs:
         type: string
         default: 'false'
+      build_nix:
+        type: string
+        default: 'true'
 
 jobs:
   book:
@@ -155,7 +158,7 @@ jobs:
     name: Distro Packages
     uses: ./.github/workflows/bake.yml
     with:
-      bake_targets: '["deb", "rpm"]'
+      bake_targets: '["deb", "rpm", "nix"]'
       cargo_profiles: '["release"]'
       feat_sets: '["all"]'
       rust_toolchains: '["stable"]'
@@ -176,6 +179,12 @@ jobs:
           "rpm": {
             "dst": "tuwunel.rpm",
             "mime": "application/x-rpm"
+          },
+          "nix": {
+            "dst": "tuwunel.nix.tar.zst",
+            "src": "/opt/tuwunel.nix.tar",
+            "mime": "application/zstd",
+            "zstd": 11
           }
         }
 

.github/workflows/publish.yml

@@ -44,20 +44,27 @@ on:
       release_url:
         type: string
         description: For release assets
+      pages_url:
+        type: string
+        description: For pages deployment
 
 jobs:
   documents:
     if: >
       !failure() && !cancelled()
-      && contains(fromJSON(inputs.sys_targets), fromJSON('["X64"]')[0])
       && (github.ref == 'refs/heads/main' || contains(github.ref, 'refs/tags/v'))
       && !contains(github.ref, '-draft')
 
     name: Documents
     runs-on: ['X64', 'het']
     permissions:
-      pages: write
+      contents: read
       id-token: write
+      pages: write
+
+    environment:
+      name: github-pages
+      url: ${{inputs.pages_url}}
 
     steps:
       - id: book

.github/workflows/test.yml

@@ -38,16 +38,22 @@ on:
       complement:
         type: boolean
         default: true
-      complement_feat_sets:
-        type: string
-        default: '["all"]'
       complement_runner:
         type: string
+      rust_sdk_integ:
+        type: boolean
+        default: true
+      build_nix:
+        type: boolean
+        default: true
+      head_msg:
+        type: string
 
 jobs:
   docs:
     if: >
-      contains(fromJSON(inputs.cargo_profiles), fromJSON('["test"]')[0])
+      !contains(inputs.head_msg, '[ci only it]')
+      && contains(fromJSON(inputs.cargo_profiles), fromJSON('["test"]')[0])
       && contains(fromJSON(inputs.feat_sets), fromJSON('["all"]')[0])
       && contains(fromJSON(inputs.rust_toolchains), fromJSON('["nightly"]')[0])
       && contains(fromJSON(inputs.sys_targets), fromJSON('["x86_64-v1-linux-gnu"]')[0])
@@ -69,15 +75,16 @@ jobs:
 
   unit:
     if: >
-      contains(fromJSON(inputs.cargo_profiles), fromJSON('["test"]')[0])
+      !contains(inputs.head_msg, '[ci only it]')
+      && contains(fromJSON(inputs.cargo_profiles), fromJSON('["test"]')[0])
       && contains(fromJSON(inputs.feat_sets), fromJSON('["all"]')[0])
       && contains(fromJSON(inputs.rust_toolchains), fromJSON('["nightly"]')[0])
       && contains(fromJSON(inputs.sys_targets), fromJSON('["x86_64-v1-linux-gnu"]')[0])
 
-    name: Unit
+    name: Module
     uses: ./.github/workflows/bake.yml
     with:
-      bake_targets: '["unit"]'
+      bake_targets: '["unit", "integ"]'
       cargo_profiles: '["test"]'
       feat_sets: '["all"]'
       rust_toolchains: ${{inputs.rust_toolchains}}
@@ -89,9 +96,58 @@ jobs:
       excludes: ${{inputs.excludes}}
       includes: ${{inputs.includes}}
 
+  bench:
+    if: >
+      !contains(inputs.head_msg, '[ci only it]')
+      && contains(fromJSON(inputs.cargo_profiles), fromJSON('["bench"]')[0])
+      && contains(fromJSON(inputs.feat_sets), fromJSON('["all"]')[0])
+      && contains(fromJSON(inputs.rust_toolchains), fromJSON('["nightly"]')[0])
+      && contains(fromJSON(inputs.sys_targets), fromJSON('["x86_64-v3-linux-gnu"]')[0])
+
+    name: Bench
+    uses: ./.github/workflows/bake.yml
+    with:
+      bake_targets: '["unit", "integ"]'
+      cargo_profiles: '["bench"]'
+      feat_sets: '["all"]'
+      rust_toolchains: '["nightly"]'
+      sys_names: ${{inputs.sys_names}}
+      sys_versions: ${{inputs.sys_versions}}
+      rust_targets: ${{inputs.rust_targets}}
+      sys_targets: '["x86_64-v3-linux-gnu"]'
+      machines: ${{inputs.machines}}
+      excludes: ${{inputs.excludes}}
+      includes: ${{inputs.includes}}
+
+  memcheck:
+    if: >
+      !contains(inputs.head_msg, '[ci only it]')
+      && contains(fromJSON(inputs.cargo_profiles), fromJSON('["bench"]')[0])
+      && contains(fromJSON(inputs.feat_sets), fromJSON('["all"]')[0])
+      && contains(fromJSON(inputs.rust_toolchains), fromJSON('["nightly"]')[0])
+      && contains(fromJSON(inputs.sys_targets), fromJSON('["x86_64-v3-linux-gnu"]')[0])
+
+    name: Memcheck
+    uses: ./.github/workflows/bake.yml
+    with:
+      #bake_targets: '["unit-valgrind", "integ-valgrind"]'
+      bake_targets: '["integ-valgrind"]'
+      cargo_profiles: '["bench"]' # use bench not release for debug syms
+      feat_sets: '["all"]'
+      rust_toolchains: '["nightly"]'
+      sys_names: ${{inputs.sys_names}}
+      sys_versions: ${{inputs.sys_versions}}
+      rust_targets: ${{inputs.rust_targets}}
+      sys_targets: '["x86_64-v3-linux-gnu"]'
+      machines: ${{inputs.machines}}
+      includes: ${{inputs.includes}}
+      excludes: ${{inputs.excludes}}
+
   smoke:
     if: >
       !failure() && !cancelled()
+      && !contains(inputs.head_msg, '[ci only it]')
+      && !contains(inputs.head_msg, '[ci no build]')
       && inputs.cargo_profiles
       && inputs.machines
 
@@ -117,6 +173,8 @@ jobs:
           {"cargo_profile": "test", "rust_target": "aarch64-unknown-linux-gnu"},
           {"cargo_profile": "test", "sys_target": "x86_64-v2-linux-gnu"},
           {"cargo_profile": "test", "sys_target": "x86_64-v3-linux-gnu"},
+          {"cargo_profile": "test", "bake_target": "smoke-valgrind"},
+          {"cargo_profile": "bench"},
           {"cargo_profile": "release", "rust_toolchain": "nightly"},
           {"cargo_profile": "release", "rust_toolchain": "stable", "feat_set": "none"},
           {"cargo_profile": "release", "bake_target": "smoke-valgrind"},
@@ -160,26 +218,105 @@ jobs:
           {"sys_target": "x86_64-v4-linux-gnu", "bake_target": "smoke-valgrind"},
         ]
 
+  nix:
+    if: >
+      !failure() && !cancelled()
+      && !contains(inputs.head_msg, '[ci only it]')
+      && !contains(inputs.head_msg, '[ci no build]')
+      && !contains(github.ref, 'refs/pull')
+      && contains(fromJSON(inputs.cargo_profiles), fromJSON('["release"]')[0])
+      && contains(fromJSON(inputs.feat_sets), fromJSON('["all"]')[0])
+      && contains(fromJSON(inputs.rust_toolchains), fromJSON('["stable"]')[0])
+      && contains(fromJSON(inputs.sys_targets), fromJSON('["x86_64-v1-linux-gnu"]')[0])
+      && fromJSON(inputs.build_nix)
+
+    name: Smoke NixOS
+    uses: ./.github/workflows/bake.yml
+    with:
+      bake_targets: '["smoke-nix"]'
+      cargo_profiles: '["release"]'
+      feat_sets: '["all"]'
+      rust_toolchains: '["stable"]'
+      sys_names: ${{inputs.sys_names}}
+      sys_versions: ${{inputs.sys_versions}}
+      rust_targets: ${{inputs.rust_targets}}
+      sys_targets: '["x86_64-v1-linux-gnu"]'
+      machines: ${{inputs.machines}}
+      excludes: ${{inputs.excludes}}
+      includes: ${{inputs.includes}}
+
+  rust-sdk-integ:
+    if: >
+      !failure() && !cancelled()
+      && !contains(inputs.head_msg, '[ci no build]')
+      && inputs.rust_sdk_integ
+      && contains(fromJSON(inputs.feat_sets), fromJSON('["all"]')[0])
+      && contains(fromJSON(inputs.rust_toolchains), fromJSON('["nightly"]')[0])
+      && contains(fromJSON(inputs.rust_targets), fromJSON('["x86_64-unknown-linux-gnu"]')[0])
+
+    name: Matrix SDK Integration
+    needs: [smoke]
+    uses: ./.github/workflows/bake.yml
+    with:
+      #bake_targets: '["rust-sdk-integ", "rust-sdk-valgrind"]'
+      bake_targets: '["rust-sdk-integ"]'
+      cargo_profiles: ${{inputs.cargo_profiles}}
+      feat_sets: '["all"]'
+      rust_toolchains: '["nightly"]'
+      sys_names: ${{inputs.sys_names}}
+      sys_versions: ${{inputs.sys_versions}}
+      rust_targets: '["x86_64-unknown-linux-gnu"]'
+      sys_targets: ${{inputs.sys_targets}}
+      machines: '["X64"]'
+      runner: ${{inputs.complement_runner}}
+      includes: ${{inputs.includes}}
+      artifact: >
+        {
+          "rust-sdk-integ": {
+            "src": "/var/log/tuwunel.log",
+            "dst": "rust-sdk-integ.tuwunel.log",
+          },
+          "rust-sdk-valgrind": {
+            "src": "/var/log/tuwunel.log",
+            "dst": "rust-sdk-valgrind.tuwunel.log",
+          }
+        }
+      excludes: >
+        [
+          {"bake_target": "rust-sdk-valgrind", "cargo_profile": "test"},
+          {"feat_set": "none"},
+          {"feat_set": "logging"},
+          {"cargo_profile": "release"},
+          {"cargo_profile": "release-debuginfo"},
+          {"cargo_profile": "release-native"},
+          {"cargo_profile": "test", "sys_target": "x86_64-v2-linux-gnu"},
+          {"cargo_profile": "test", "sys_target": "x86_64-v3-linux-gnu"},
+          {"cargo_profile": "bench", "sys_target": "x86_64-v1-linux-gnu"},
+          {"cargo_profile": "bench", "sys_target": "x86_64-v2-linux-gnu"},
+          {"rust_target": "aarch64-unknown-linux-gnu"},
+          {"sys_target": "aarch64-v8-linux-gnu"},
+        ]
+
   complement:
     if: >
       !failure() && !cancelled()
+      && !contains(inputs.head_msg, '[ci no build]')
       && inputs.complement
-      && contains(fromJSON(inputs.cargo_profiles), fromJSON('["test"]')[0])
-      && contains(fromJSON(inputs.complement_feat_sets), fromJSON('["all"]')[0])
+      && contains(fromJSON(inputs.cargo_profiles), fromJSON('["bench"]')[0])
       && contains(fromJSON(inputs.rust_toolchains), fromJSON('["nightly"]')[0])
-      && contains(fromJSON(inputs.sys_targets), fromJSON('["x86_64-v1-linux-gnu"]')[0])
+      && contains(fromJSON(inputs.sys_targets), fromJSON('["x86_64-v3-linux-gnu"]')[0])
 
     name: Complement
     uses: ./.github/workflows/bake.yml
     with:
       bake_targets: '["complement-tester", "complement-testee"]'
-      cargo_profiles: '["test"]'
-      feat_sets: ${{inputs.complement_feat_sets}}
+      cargo_profiles: '["bench"]'
+      feat_sets: '["logging"]'
       rust_toolchains: '["nightly"]'
       sys_names: ${{inputs.sys_names}}
       sys_versions: ${{inputs.sys_versions}}
       rust_targets: ${{inputs.rust_targets}}
-      sys_targets: '["x86_64-v1-linux-gnu"]'
+      sys_targets: '["x86_64-v3-linux-gnu"]'
       machines: ${{inputs.machines}}
       runner: ${{inputs.complement_runner}}
       excludes: ${{inputs.excludes}}
@@ -188,15 +325,15 @@ jobs:
   compliance:
     if: >
       !failure() && !cancelled()
+      && !contains(inputs.head_msg, '[ci no build]')
       && inputs.complement
       && inputs.machines
-      && contains(fromJSON(inputs.cargo_profiles), fromJSON('["test"]')[0])
-      && contains(fromJSON(inputs.feat_sets), fromJSON('["all"]')[0])
+      && contains(fromJSON(inputs.cargo_profiles), fromJSON('["bench"]')[0])
       && contains(fromJSON(inputs.rust_toolchains), fromJSON('["nightly"]')[0])
-      && contains(fromJSON(inputs.sys_targets), fromJSON('["x86_64-v1-linux-gnu"]')[0])
+      && contains(fromJSON(inputs.sys_targets), fromJSON('["x86_64-v3-linux-gnu"]')[0])
 
     name: Matrix Compliance
-    needs: [complement]
+    needs: [complement, smoke]
     runs-on: ["${{matrix.machine}}", "${{inputs.complement_runner}}"]
     concurrency:
       group: complement-cant-walk-and-chew-bubblegum
@@ -205,13 +342,13 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        cargo_profile: ${{fromJSON('["test"]')}}
-        feat_set: ${{fromJSON(inputs.complement_feat_sets)}}
+        cargo_profile: ${{fromJSON('["bench"]')}}
+        feat_set: ${{fromJSON('["logging"]')}}
         rust_toolchain: ${{fromJSON('["nightly"]')}}
         sys_name: ${{fromJSON(inputs.sys_names)}}
         sys_version: ${{fromJSON(inputs.sys_versions)}}
         rust_target: ${{fromJSON(inputs.rust_targets)}}
-        sys_target: ${{fromJSON('["x86_64-v1-linux-gnu"]')}}
+        sys_target: ${{fromJSON('["x86_64-v3-linux-gnu"]')}}
         machine: ${{fromJSON(inputs.machines)}}
         exclude: ${{fromJSON(inputs.excludes)}}
         include: ${{fromJSON(inputs.includes)}}
@@ -242,7 +379,7 @@ jobs:
 
       run: |
         cid=$(cat "$name")
-        docker cp "$cid:/usr/src/complement/new_results.jsonl" tests/test_results/complement/test_results.jsonl
+        docker cp "$cid:/usr/src/complement/new_results.jsonl" tests/complement/results.jsonl
 
     - if: success() || failure() && steps.execute.outcome == 'failure'
       name: Upload New Results
@@ -250,7 +387,7 @@ jobs:
       uses: actions/upload-artifact@v4
       with:
         name: complement_results-${{matrix.feat_set}}-${{matrix.sys_name}}-${{matrix.sys_target}}.jsonl
-        path: ./tests/test_results/complement/test_results.jsonl
+        path: ./tests/complement/results.jsonl
 
     - if: failure() && steps.execute.outcome == 'failure'
       name: Upload Failure Output
@@ -258,7 +395,7 @@ jobs:
       uses: actions/upload-artifact@v4
       with:
         name: complement_output-${{matrix.feat_set}}-${{matrix.sys_name}}-${{matrix.sys_target}}.jsonl
-        path: ./complement.jsonl
+        path: ./tests/complement/logs.jsonl
         if-no-files-found: ignore
 
     - name: Accept

.gitignore

@@ -94,7 +94,7 @@ public/
 rustc-ice-*
 
 # complement test logs are huge
-tests/test_results/complement/test_logs.jsonl
+tests/complement/logs.jsonl
 
 # cargo profiles from -Z self-profile
 *.mm_profdata

Cargo.toml

@@ -28,8 +28,8 @@ keywords = [
 license = "Apache-2.0"
 readme = "README.md"
 repository = "https://github.com/matrix-construct/tuwunel"
-rust-version = "1.88.0"
-version = "1.4.2"
+rust-version = "1.89.0"
+version = "1.4.7"
 
 [workspace.metadata.crane]
 name = "tuwunel"
@@ -104,7 +104,7 @@ features = [
 version = "1.10"
 
 [workspace.dependencies.bytesize]
-version = "2.0"
+version = "2.1"
 
 [workspace.dependencies.cargo_toml]
 version = "0.22"
@@ -140,10 +140,19 @@ features = [
 version = "0.8.3"
 
 [workspace.dependencies.const-str]
-version = "0.6"
+version = "0.7"
+
+[workspace.dependencies.criterion]
+version = "0.7"
+default-features = false
+features = [
+	"cargo_bench_support",
+	"async_futures",
+	"async_tokio",
+]
 
 [workspace.dependencies.ctor]
-version = "0.4"
+version = "0.5"
 
 [workspace.dependencies.cyborgtime]
 version = "2.1"
@@ -183,7 +192,7 @@ version = "1.3"
 version = "0.1"
 
 [workspace.dependencies.hyper]
-version = "1.6"
+version = "1.7"
 default-features = false
 features = [
 	"server",
@@ -222,6 +231,10 @@ version = "0.1"
 [workspace.dependencies.itertools]
 version = "0.14"
 
+[workspace.dependencies.jevmalloc]
+git = "https://github.com/matrix-construct/jevmalloc"
+rev = "29fde02533ee2c6a5e9b87095324e6f7a5d43738"
+
 [workspace.dependencies.jsonwebtoken]
 version = "9.3"
 default-features = false
@@ -253,7 +266,7 @@ version = "0.1"
 version = "1.0"
 
 [workspace.dependencies.minicbor]
-version = "2.0"
+version = "2.1"
 features = ["std"]
 
 [workspace.dependencies.minicbor-serde]
@@ -263,13 +276,16 @@ features = ["std"]
 [workspace.dependencies.nix]
 version = "0.30"
 default-features = false
-features = ["resource"]
+features = [
+	"resource",
+	"user",
+]
 
 [workspace.dependencies.num-traits]
 version = "0.2"
 
 [workspace.dependencies.opentelemetry]
-version = "0.30.0"
+version = "0.31"
 
 # Disabled until they move to opentelemetry 0.30
 #[workspace.dependencies.opentelemetry-jaeger]
@@ -277,7 +293,7 @@ version = "0.30.0"
 #features = ["rt-tokio"]
 
 [workspace.dependencies.opentelemetry_sdk]
-version = "0.30"
+version = "0.31"
 features = ["rt-tokio"]
 
 [workspace.dependencies.proc-macro2]
@@ -290,7 +306,7 @@ version = "1.0"
 version = "0.8"
 
 [workspace.dependencies.regex]
-version = "1.11"
+version = "1.12"
 
 [workspace.dependencies.reqwest]
 version = "0.12"
@@ -308,7 +324,7 @@ default-features = false
 
 [workspace.dependencies.ruma]
 git = "https://github.com/matrix-construct/ruma"
-rev = "5682b88cf1bcaf0f47805d614b476b242ef075d4"
+rev = "f8aa292ce48f28ee6c82b29a811d26c6ccb5aad2"
 features = [
     "__compat",
     "appservice-api-c",
@@ -327,6 +343,7 @@ features = [
     "unstable-msc2870",
     "unstable-msc3026",
     "unstable-msc3061",
+    "unstable-msc3814",
     "unstable-msc3245",
     "unstable-msc3381", # polls
     "unstable-msc3489", # beacon / live location
@@ -336,11 +353,12 @@ features = [
     "unstable-msc4121",
     "unstable-msc4125",
     "unstable-msc4133",
+    "unstable-msc4143",
     "unstable-msc4186",
     "unstable-msc4203", # sending to-device events to appservices 
+    "unstable-msc4310",
     "unstable-msc4311",
     "unstable-extensible-events",
-    "unstable-hydra",
 ]
 
 [workspace.dependencies.rustls]
@@ -354,14 +372,16 @@ default-features = false
 
 [workspace.dependencies.rust-rocksdb]
 git = "https://github.com/matrix-construct/rust-rocksdb"
-rev = "225a42519276e502205bdc845cebdb22d70ee245"
+rev = "31e6f8dd4a531b379c132e82773efc9315b0489a"
 default-features = false
 features = [
 	"bzip2",
+	"lto",
 	"lz4",
 	"multi-threaded-cf",
-	"mt_static",
+	"serde1",
 	"zstd",
+	"zstd-static-linking-only",
 ]
 
 [workspace.dependencies.sanitize-filename]
@@ -372,7 +392,7 @@ version = "0.4"
 default-features = false
 
 [workspace.dependencies.sentry]
-version = "0.42"
+version = "0.45"
 default-features = false
 features = [
     "backtrace",
@@ -388,10 +408,10 @@ features = [
 ]
 
 [workspace.dependencies.sentry-tower]
-version = "0.42"
+version = "0.45"
 
 [workspace.dependencies.sentry-tracing]
-version = "0.42"
+version = "0.45"
 
 [workspace.dependencies.serde]
 version = "1.0"
@@ -454,39 +474,15 @@ features = [
 ]
 
 [workspace.dependencies.termimad]
-version = "0.33"
+version = "0.34"
 default-features = false
 
 [workspace.dependencies.thiserror]
 version = "2.0"
 default-features = false
 
-[workspace.dependencies.tikv-jemallocator]
-git = "https://github.com/matrix-construct/jemallocator"
-rev = "03bed96afbbc898bef4d4f7d335c0519e3d1afad"
-default-features = false
-features = [
-	"background_threads_runtime_support",
-	"unprefixed_malloc_on_supported_platforms",
-]
-
-[workspace.dependencies.tikv-jemalloc-ctl]
-git = "https://github.com/matrix-construct/jemallocator"
-rev = "03bed96afbbc898bef4d4f7d335c0519e3d1afad"
-default-features = false
-features = ["use_std"]
-
-[workspace.dependencies.tikv-jemalloc-sys]
-git = "https://github.com/matrix-construct/jemallocator"
-rev = "03bed96afbbc898bef4d4f7d335c0519e3d1afad"
-default-features = false
-features = [
-	"background_threads_runtime_support",
-	"unprefixed_malloc_on_supported_platforms",
-]
-
 [workspace.dependencies.tokio]
-version = "1.47"
+version = "1.48"
 default-features = false
 features = [
 	"fs",
@@ -539,7 +535,7 @@ default-features = false
 version = "0.2"
 
 [workspace.dependencies.tracing-opentelemetry]
-version = "0.31"
+version = "0.32"
 
 [workspace.dependencies.tracing-subscriber]
 version = "0.3"
@@ -691,7 +687,7 @@ inherits = "release-native.build-override"
 
 [profile.bench]
 debug = "limited"
-strip = false
+strip = "none"
 #rustflags = [
 #	"-Cremark=all",
 #	'-Ztime-passes',
@@ -711,7 +707,7 @@ strip = false
 # and can be raised if build times are tolerable.
 
 [profile.dev]
-debug = "full"
+debug = 0
 #rustflags = [
 #	'--cfg', 'tuwunel_mods',
 #	'-Ztime-passes',
@@ -771,7 +767,7 @@ inherits = "dev"
 
 [profile.dev.package.'*']
 inherits = "dev"
-debug = "limited"
+debug = 0
 incremental = false
 codegen-units = 1
 opt-level = 'z'
@@ -788,6 +784,11 @@ opt-level = 'z'
 #	'-Clink-arg=-Wl,-z,nodelete',
 #]
 
+# same as dev but slower.
+[profile.dbg]
+inherits = "dev"
+debug = "full"
+
 # primarily used for CI
 [profile.test]
 debug = "limited"
@@ -879,6 +880,9 @@ multiple_crate_versions = { level = "allow", priority = 1 }
 ###################
 complexity = { level = "warn", priority = -1 }
 
+# promotes forward-compat for literal ..default() construction
+needless_update = { level = "allow", priority = 1 }
+
 ###################
 correctness = { level = "warn", priority = -1 }
 
@@ -891,6 +895,7 @@ option_if_let_else = { level = "allow", priority = 1 }               # TODO
 redundant_pub_crate = { level = "allow", priority = 1 }              # TODO
 significant_drop_in_scrutinee = { level = "allow", priority = 1 }    # TODO
 significant_drop_tightening = { level = "allow", priority = 1 }      # TODO
+tuple_array_conversions = { level = "allow", priority = 1 }
 
 ###################
 pedantic = { level = "warn", priority = -1 }
@@ -905,6 +910,7 @@ if_then_some_else_none = { level = "allow", priority = 1 }
 inline_always = { level = "allow", priority = 1 }
 map_unwrap_or = { level = "allow", priority = 1 }
 match_bool = { level = "allow", priority = 1 }
+match_same_arms = { level = "allow", priority = 1 }
 missing_docs_in_private_items = { level = "allow", priority = 1 }
 missing_errors_doc = { level = "allow", priority = 1 }
 missing_panics_doc = { level = "allow", priority = 1 }
@@ -950,7 +956,6 @@ pub_without_shorthand = "warn"
 rc_buffer = "warn"
 rc_mutex = "warn"
 redundant_type_annotations = "warn"
-rest_pat_in_fully_bound_structs = "warn"
 semicolon_outside_block = "warn"
 str_to_string = "warn"
 string_lit_chars_any = "warn"

RELEASE.md

@@ -1,57 +1,39 @@
-# Tuwunel 1.4.2
+# Tuwunel 1.4.7
 
-September 12, 2025
+December 3, 2025
 
-Users running maubot, neochat, or any client or bridge not excluded below should update to this patch as soon as possible to reduce unnecessary resource consumption. (see: Bug Fixes)
+Tuwunel is now deployed at scale serving the citizens of Switzerland in production. Some optimizations were requested to reduce operating costs from projected growth over product lifespan: this release delivers with markedly reduced CPU usage and improves responsiveness. However, complications during routine dependency upgrades consumed valuable time planned for features and issues which could not be completed for this release.
 
 ### New Features
 
-- Requested by @alaviss an alternative DNS resolver has been implemented for use with appservices and other configured targets intended for local networks. This passthru performs minimal caching and cannot be used for federation. Enable with `dns_passthru_appservices = true` or specifying hosts in `dns_passthru_domains` (#158)
-
-- Contributed by @tototomate123 a nifty experimental feature can disable push notifications when you're active on one device from being sent to others. This can be enabled with `suppress_push_when_active`. Please thank them when your pocket stops vibrating while chatting on your desktop! (#150)
-
-- Thanks to a report by @DetermineAbsurd the `m.federate` field can be defaulted to false when creating a room using the new `federate_created_rooms` config option. (#151)
-
-- At the request of @grinapo verbose logging builds are now bundled with this release. These builds are found with the feature-set `-logging-` which is otherwise similar to `-all-`. This contains more messages at all levels optimized away in other release modes; it comes at some performance penalty.
-
-- JWT tokens can now be used for authentication on any endpoint which supports UIA. For example: an external forgot-password service can send a token to the `client/account/password` endpoint to reset a user's password. This feature was commissioned and made public by an enterprise sponsor.
+- Upgrade Support for Room Version 12 is now available. Though this room version has been supported for the entire 1.4 series, all Tuwunel servers have been protected by Hydra Backports on all room versions. As such, other work was able to be prioritized for the preceding releases.
 
 ### Enhancements
 
-- Sliding-sync has been significantly refactored. Performance has massively increased with many bugs and compliance issues also fixed. Please be aware we are tracking an issue related to read-marker behavior in Element X. The 🟢 dot does not unconditionally clear at every touch. Whether this is a feature or a bug, or both, is being investigated for v1.5.
+- Recursive relations have been enabled. This is an optimization which allows the server to gather more data using fewer client requests, for example, of a thread with many reactions and replies. The implementation is now optimal and utilizes the full capabilities of Tuwunel's asynchronous database.
 
-- Hydra backports are now enabled by default. The change should be completely transparent. If you do notice any increased load try to increase the `cache_capacity_modifier` above default.
+- Several miscellaneous but significant optimizations took place at the direction of memory profiling. This reduced load on the allocator for database queries and JSON serialization of complex objects. Heroes calculations and the joined room hot-path on sync v3 were further optimized to reduce the database query load itself.
 
-- Room deletions now also purge synctokens which can be significant to the overall storage consumed by a room. Users who have already deleted rooms please be assured an update planned for v1.5 will deal with cleansing synctokens in general.
+- Jemalloc has been repackaged with platform-specific optimizations enhancing the build. The upgrade to the dev branch of libjemalloc itself was considered as too much variability for the same release, it is planned for an upcoming release.
 
-- Room version 1 and 2 support took a step forward, possibly working for some rooms but is not yet considered adequately supported and the ticket remains open. (#12)
-
-- Thanks to @AreYouLoco for contributing an updated Kubernetes [Helm Chart](https://github.com/AreYouLoco/tuwunel-helm); link added to docs.
+- Thanks to element-hq/synapse#18970 by @dasha-uwu, we have very slightly turned down the amount of randomness when selecting join-servers, More retries also occur within a single request if necessary. Thanks to @gogo199432 and @lifeofguenter for reporting problems in (#128) and (#205) respectively. More opportunities are still available to make large room joins robust.
 
 ### Bug Fixes
 
-- **Special thanks to @frebib for investigating a bug which triggers the uploading of unnecessary encryption one-time-keys.** Running over ten maubot instances it became obvious after observing increased resources and laggy bot response. This update removes any excess keys for a device. Thanks to @duckbuster for confirming neochat is affected. Clients confirmed unaffected include: Element, Element X, Nheko. Fractal, Cinny, matrix-rust-sdk and matrix-js-sdk clients and bots are probably unaffected. Mautrix-based bridges are probably affected. Users of unaffected clients should still upgrade.
+- Special thanks to @yefimg for fixing LDAP logout in (#231) from a report kindly made by @orhtej2 in (#97); thank you for your patience waiting for domain expertise to assist here.
 
-- Thanks @dasha_uwu for refactoring alias resolution logic with fixes to remain compatible with the upcoming element-web release. This was an incredibly valuable contribution which will spare all of us from impending grief; the kind of ahead-of-the-game initiative I don't think a project like this could exist without. (adadafa88f3)
+- Thanks to @Radiant-Xyz the example configurations have been updated to remove `allow_check_for_updates`. This fixes any warnings for the item no longer existing. (#221)
 
-- Room deletions now preserve a small number of records to properly synchronize with local clients and remote servers after the room vanishes. Prior behavior is maintained with a `--force` flag added to the command.
+- Thanks again to @Radiant-Xyz reporting in (#219) the `/whoami` endpoint is now returns spec-compliant errors for Mautrix bridges (fe12daead9). Thanks also to @bobobo1618 for confirming the fix is working.
 
-- Thanks @scvalex for once again cleaning up our mess after Nix found the github CI was not running doctests. Thank you for contributing the patch 🙏  (#152).
+- Relations responses were sometimes incorrect in the forwards direction. This was fixed by (5147b541) bringing those responses into full compliance. Note the prior release had also fixed compliance issues but in the backwards direction.
 
-- Thanks @Tronde for reporting a broken link to the CoC in the mdbook documentation. (#155)
+- Server selection for backfill struggled sometimes for version 12 rooms. These rooms might fail to load history after join. Additional servers are now found using `creators` and `additional_creators` instead.
 
-- Specification compliance required the `/joined_rooms` endpoint be restricted to current members rather than including past members. (4b49aaad53a)
+- Room leave compliance has been fixed for an edge-case where a room becomes empty except for a locally invited user which does not have its leave event sent down `/sync`.
 
-- Specification compliance required state events be made visible to prior members of a room where `history_visibility=shared`. (86781522b68)
+- Thanks to @grinapo for a report which lead to the discovery of events acquired over backfill not being checked for whether they already exist.
 
-- The `limit` parameter to the `/context` endpoint is now divided with de facto compatibility (matrix-org/matrix-spec#2202)
+### Upcoming
 
-- The room avatar in sliding sync is now computed with greater compliance to the specification (3deebeab78f). This builds off earlier work done by @tmayoff in (a340e6786db).
-
-- The canonical alias for a room is considered invalid if the primary alias is missing or removed (7221d466ce8). This is a T&S concern and we encourage reports for any other contexts where this condition should be applied.
-
-- Presence is no longer updated by the private read-receipt or read-marker paths, only public receipts.
-
-### Deprecations
-
-- Hardened Malloc support had to be removed after the build broke. We will gladly add support back upon request or contribution.
+- As stated in the summary, several planned items could not be cut into this release. These include SSO/OIDC support (#7), Element Call setup assistance and documentation (#217)(#215), User-level Admin Room and Media deletion (#192), and any other assigned issue. These items are on the short-list for the next cycle and mean a lot to us; to all participants: your issues are not being ignored and we hear you.

arch/tuwunel.service

@@ -4,7 +4,6 @@ Wants=network-online.target
 After=network-online.target
 Documentation=https://tuwunel.chat/
 RequiresMountsFor=/var/lib/private/tuwunel
-Alias=matrix-tuwunel.service
 
 [Service]
 DynamicUser=yes
@@ -77,3 +76,4 @@ StartLimitBurst=5
 
 [Install]
 WantedBy=multi-user.target
+Alias=matrix-tuwunel.service

book.toml

@@ -5,7 +5,6 @@ authors = [
 ]
 description = "Tuwunel, a high performance successor to Conduit and Conduwuit"
 language = "en"
-multilingual = false
 src = "docs"
 title = "Tuwunel One"
 text-direction = "ltr"
@@ -15,7 +14,9 @@ build-dir = "public"
 create-missing = true
 extra-watch-dirs = [
 	"debian",
+	"docker",
 	"docs",
+	"rpm",
 ]
 
 [rust]
@@ -24,7 +25,7 @@ edition = "2024"
 [output.html]
 git-repository-url = "https://github.com/matrix-construct/tuwunel"
 edit-url-template = "https://github.com/matrix-construct/tuwunel/edit/main/{path}"
-git-repository-icon = "fa-github-square"
+git-repository-icon = "fab-github"
 
 [output.html.search]
 limit-results = 15

clippy.toml

@@ -1,11 +1,11 @@
+stack-size-threshold = 393216
+future-size-threshold = 24576
 array-size-threshold = 4096
-cognitive-complexity-threshold = 100        # TODO reduce me ALARA
-excessive-nesting-threshold = 8
-future-size-threshold = 8192
-stack-size-threshold = 196608               # TODO reduce me ALARA
-too-many-lines-threshold = 780              # TODO reduce me to <= 100
-type-complexity-threshold = 250             # reduce me to ~200
 large-error-threshold = 256                 # TODO reduce me ALARA
+too-many-lines-threshold = 780              # TODO reduce me to <= 100
+excessive-nesting-threshold = 8
+type-complexity-threshold = 250             # reduce me to ~200
+cognitive-complexity-threshold = 100        # TODO reduce me ALARA
 
 #disallowed-macros = [
 #	{ path = "log::error", reason = "use tuwunel_core::error" },

debian/tuwunel.service

@@ -2,11 +2,9 @@
 Description=Tuwunel Matrix homeserver
 Wants=network-online.target
 After=network-online.target
-Alias=matrix-tuwunel.service
 Documentation=https://tuwunel.chat/
 
 [Service]
-DynamicUser=yes
 User=tuwunel
 Group=tuwunel
 Type=notify
@@ -64,3 +62,4 @@ StartLimitBurst=5
 
 [Install]
 WantedBy=multi-user.target
+Alias=matrix-tuwunel.service

docker/Dockerfile.cargo

@@ -12,10 +12,12 @@ ARG CARGO_TARGET_DIR
 ARG cargo_target_profile
 ARG cargo_target_artifact
 ARG cargo_target_share
+ARG cargo_share
 ARG CARGO_TERM_VERBOSE=0
 ARG RUST_BACKTRACE
 ARG JEMALLOC_OVERRIDE
 ARG ROCKSDB_LIB_DIR
+ARG VALGRINDFLAGS=""
 ARG CARGO_BUILD_RUSTFLAGS
 ARG CARGO_PROFILE_TEST_DEBUG
 ARG CARGO_PROFILE_TEST_INCREMENTAL
@@ -32,6 +34,8 @@ ARG color_args="--color always"
 ARG recipe_args=""
 ARG cargo_args=""
 ARG git_checkout
+ARG targ_dir="${CARGO_TARGET_DIR}/${cargo_target_profile}"
+ARG targ_targ_dir="${CARGO_TARGET_DIR}/${rust_target}/${cargo_target_profile}"
 
 WORKDIR /
 COPY --link --from=input . .
@@ -42,11 +46,13 @@ WORKDIR /usr/lib/${sys_triple}
 COPY --link --from=rocksdb . .
 
 WORKDIR /usr/src/tuwunel
+SHELL ["/bin/bash", "-c"]
 ENV PATH="${CARGO_HOME}/bin:$PATH"
 ENV CARGO_TERM_VERBOSE="${CARGO_TERM_VERBOSE}"
 ENV RUST_BACKTRACE="${RUST_BACKTRACE}"
 ENV JEMALLOC_OVERRIDE="${JEMALLOC_OVERRIDE}"
 ENV ROCKSDB_LIB_DIR="${ROCKSDB_LIB_DIR}"
+ENV VALGRINDFLAGS="${VALGRINDFLAGS}"
 ENV CARGO_PROFILE_TEST_DEBUG="${CARGO_PROFILE_TEST_DEBUG}"
 ENV CARGO_PROFILE_TEST_INCREMENTAL="${CARGO_PROFILE_TEST_INCREMENTAL}"
 ENV CARGO_PROFILE_BENCH_DEBUG="${CARGO_PROFILE_BENCH_DEBUG}"
@@ -56,17 +62,15 @@ ENV CARGO_PROFILE_RELEASE_DEBUGINFO_DEBUG="${CARGO_PROFILE_RELEASE_DEBUGINFO_DEB
 ENV CARGO_PROFILE_RELEASE_DEBUGINFO_LTO="${CARGO_PROFILE_RELEASE_DEBUGINFO_LTO}"
 ENV CARGO_BUILD_RUSTFLAGS="${CARGO_BUILD_RUSTFLAGS}"
 ENV CARGO_TARGET_DIR="${CARGO_TARGET_DIR}"
-ENV targ_dir="${CARGO_TARGET_DIR}/${cargo_target_profile}"
-ENV targ_targ_dir="${CARGO_TARGET_DIR}/${rust_target}/${cargo_target_profile}"
 RUN \
 --mount=type=cache,dst=${RUSTUP_HOME}/downloads,sharing=shared \
 --mount=type=cache,dst=${CARGO_HOME}/registry,sharing=shared \
 --mount=type=cache,dst=${CARGO_HOME}/git,sharing=shared \
---mount=type=cache,dst=${targ_dir}/deps,id=${cargo_target_share}/deps,sharing=locked \
---mount=type=cache,dst=${targ_dir}/build,id=${cargo_target_share}/build,sharing=locked \
---mount=type=cache,dst=${targ_dir}/examples,id=${cargo_target_share}/examples,sharing=locked \
---mount=type=cache,dst=${targ_dir}/incremental,id=${cargo_target_share}/incremental,sharing=locked \
---mount=type=cache,dst=${targ_dir}/.fingerprint,id=${cargo_target_share}/fingerprint,sharing=locked \
+--mount=type=cache,dst=${targ_dir}/deps,id=${cargo_share}/deps,sharing=locked \
+--mount=type=cache,dst=${targ_dir}/build,id=${cargo_share}/build,sharing=locked \
+--mount=type=cache,dst=${targ_dir}/examples,id=${cargo_share}/examples,sharing=locked \
+--mount=type=cache,dst=${targ_dir}/incremental,id=${cargo_share}/incremental,sharing=locked \
+--mount=type=cache,dst=${targ_dir}/.fingerprint,id=${cargo_share}/fingerprint,sharing=locked \
 --mount=type=cache,dst=${targ_targ_dir}/deps,id=${cargo_target_share}/deps,sharing=locked \
 --mount=type=cache,dst=${targ_targ_dir}/build,id=${cargo_target_share}/build,sharing=locked \
 --mount=type=cache,dst=${targ_targ_dir}/examples,id=${cargo_target_share}/examples,sharing=locked \
@@ -74,7 +78,9 @@ RUN \
 --mount=type=cache,dst=${targ_targ_dir}/.fingerprint,id=${cargo_target_share}/fingerprint,sharing=locked \
 <<EOF
     set -eux
-    rustup run ${rust_toolchain} \
+    ulimit -n 65535
+
+    rustup run "${rust_toolchain}" \
         cargo ${cargo_cmd} \
             --verbose \
             --locked \

docker/Dockerfile.cargo.deb

@@ -10,9 +10,12 @@ ARG CARGO_TARGET_DIR
 ARG cargo_target_profile
 ARG cargo_target_artifact
 ARG cargo_target_share
+ARG cargo_share
 ARG cargo_profile
 ARG cargo_features
 ARG cargo_spec_features
+ARG targ_dir="${CARGO_TARGET_DIR}/${cargo_target_profile}"
+ARG targ_targ_dir="${CARGO_TARGET_DIR}/${rust_target}/${cargo_target_profile}"
 ARG pkg_dir
 ARG deb_args=""
 
@@ -20,17 +23,15 @@ WORKDIR /
 COPY --link --from=input . .
 
 WORKDIR /usr/src/tuwunel
-ENV targ_dir="${CARGO_TARGET_DIR}/${cargo_target_profile}"
-ENV targ_targ_dir="${CARGO_TARGET_DIR}/${rust_target}/${cargo_target_profile}"
 RUN \
 --mount=type=cache,dst=${RUSTUP_HOME}/downloads,sharing=shared \
 --mount=type=cache,dst=${CARGO_HOME}/registry,sharing=shared \
 --mount=type=cache,dst=${CARGO_HOME}/git,sharing=shared \
---mount=type=cache,dst=${targ_dir}/deps,id=${cargo_target_share}/deps,sharing=locked \
---mount=type=cache,dst=${targ_dir}/build,id=${cargo_target_share}/build,sharing=locked \
---mount=type=cache,dst=${targ_dir}/examples,id=${cargo_target_share}/examples,sharing=locked \
---mount=type=cache,dst=${targ_dir}/incremental,id=${cargo_target_share}/incremental,sharing=locked \
---mount=type=cache,dst=${targ_dir}/.fingerprint,id=${cargo_target_share}/fingerprint,sharing=locked \
+--mount=type=cache,dst=${targ_dir}/deps,id=${cargo_share}/deps,sharing=locked \
+--mount=type=cache,dst=${targ_dir}/build,id=${cargo_share}/build,sharing=locked \
+--mount=type=cache,dst=${targ_dir}/examples,id=${cargo_share}/examples,sharing=locked \
+--mount=type=cache,dst=${targ_dir}/incremental,id=${cargo_share}/incremental,sharing=locked \
+--mount=type=cache,dst=${targ_dir}/.fingerprint,id=${cargo_share}/fingerprint,sharing=locked \
 --mount=type=cache,dst=${targ_targ_dir}/deps,id=${cargo_target_share}/deps,sharing=locked \
 --mount=type=cache,dst=${targ_targ_dir}/build,id=${cargo_target_share}/build,sharing=locked \
 --mount=type=cache,dst=${targ_targ_dir}/examples,id=${cargo_target_share}/examples,sharing=locked \

docker/Dockerfile.cargo.rpm

@@ -1,6 +1,7 @@
 # syntax = docker/dockerfile:1.11-labs
 
 FROM input AS build-rpm
+ARG rust_target
 ARG rust_toolchain
 ARG RUSTUP_HOME
 ARG CARGO_HOME
@@ -9,9 +10,12 @@ ARG CARGO_TARGET_DIR
 ARG cargo_target_profile
 ARG cargo_target_artifact
 ARG cargo_target_share
+ARG cargo_share
 ARG cargo_profile
 ARG cargo_features
 ARG cargo_spec_features
+ARG targ_dir="${CARGO_TARGET_DIR}/${cargo_target_profile}"
+ARG targ_targ_dir="${CARGO_TARGET_DIR}/${rust_target}/${cargo_target_profile}"
 ARG pkg_dir
 ARG gen_rpm_args=""
 
@@ -23,11 +27,11 @@ RUN \
 --mount=type=cache,dst=${RUSTUP_HOME}/downloads,sharing=shared \
 --mount=type=cache,dst=${CARGO_HOME}/registry,sharing=shared \
 --mount=type=cache,dst=${CARGO_HOME}/git,sharing=shared \
---mount=type=cache,dst=${targ_dir}/deps,id=${cargo_target_share}/deps,sharing=locked \
---mount=type=cache,dst=${targ_dir}/build,id=${cargo_target_share}/build,sharing=locked \
---mount=type=cache,dst=${targ_dir}/examples,id=${cargo_target_share}/examples,sharing=locked \
---mount=type=cache,dst=${targ_dir}/incremental,id=${cargo_target_share}/incremental,sharing=locked \
---mount=type=cache,dst=${targ_dir}/.fingerprint,id=${cargo_target_share}/fingerprint,sharing=locked \
+--mount=type=cache,dst=${targ_dir}/deps,id=${cargo_share}/deps,sharing=locked \
+--mount=type=cache,dst=${targ_dir}/build,id=${cargo_share}/build,sharing=locked \
+--mount=type=cache,dst=${targ_dir}/examples,id=${cargo_share}/examples,sharing=locked \
+--mount=type=cache,dst=${targ_dir}/incremental,id=${cargo_share}/incremental,sharing=locked \
+--mount=type=cache,dst=${targ_dir}/.fingerprint,id=${cargo_share}/fingerprint,sharing=locked \
 --mount=type=cache,dst=${targ_targ_dir}/deps,id=${cargo_target_share}/deps,sharing=locked \
 --mount=type=cache,dst=${targ_targ_dir}/build,id=${cargo_target_share}/build,sharing=locked \
 --mount=type=cache,dst=${targ_targ_dir}/examples,id=${cargo_target_share}/examples,sharing=locked \

docker/Dockerfile.complement

@@ -1,22 +1,6 @@
 # syntax = docker/dockerfile:1.11-labs
 
-FROM input AS key-gen-base
-ARG var_cache
-ARG var_lib_apt
-
-WORKDIR /
-COPY --link --from=input . .
-
-RUN \
---mount=type=cache,dst=/var/cache,id=${var_cache},sharing=locked \
---mount=type=cache,dst=/var/lib/apt,id=${var_lib_apt},sharing=locked \
-<<EOF
-    set -eux
-    apt-get -y -U install --no-install-recommends openssl gawk
-EOF
-
-
-FROM key-gen-base AS key-gen
+FROM input AS key-gen
 
 WORKDIR /complement
 COPY <<EOF v3.ext
@@ -62,12 +46,11 @@ RUN [ -f certificate.crt ] && [ -f private_key.pem ]
 FROM scratch AS complement-config
 WORKDIR /complement
 COPY --from=key-gen /complement/* .
-COPY --from=source /usr/src/tuwunel/tests/test_results/complement/test_results.jsonl old_results.jsonl
+COPY --from=source /usr/src/tuwunel/tests/complement/results.jsonl old_results.jsonl
 COPY <<EOF complement.toml
     [global]
     address = "0.0.0.0"
     admin_room_notices = false
-    allow_check_for_updates = false
     allow_device_name_federation = true
     allow_guest_registration = true
     allow_invalid_tls_certificates = true
@@ -89,7 +72,6 @@ COPY <<EOF complement.toml
     log_thread_ids = true
     media_compat_file_link = false
     media_startup_check = true
-    only_query_trusted_key_servers = false
     port = [8008, 8448]
     prune_missing_media = true
     query_trusted_key_servers_first = false
@@ -121,9 +103,6 @@ EOF
 
 FROM input AS complement-testee
 
-WORKDIR /
-COPY --link --from=input . .
-
 EXPOSE 8008 8448
 RUN mkdir /database
 COPY --from=complement-config * /complement/
@@ -133,9 +112,6 @@ ENTRYPOINT tuwunel -Oserver_name=\""$SERVER_NAME\""
 
 FROM input AS complement-testee-valgrind
 
-WORKDIR /
-COPY --link --from=input . .
-
 EXPOSE 8008 8448
 RUN mkdir /database
 COPY --from=complement-config * /complement/
@@ -152,22 +128,13 @@ ENTRYPOINT valgrind \
 FROM input AS complement-base
 ARG var_cache
 ARG var_lib_apt
+ARG complement_ref="4d3130f06d0dc3f794b5d48fbdba0b466792b52b"
 ARG complement_tags="conduwuit_blacklist"
 ARG complement_tests="./tests/..."
 ARG complement_run=".*"
 
-WORKDIR /
-RUN \
---mount=type=cache,dst=/var/cache,id=${var_cache},sharing=locked \
---mount=type=cache,dst=/var/lib/apt,id=${var_lib_apt},sharing=locked \
---mount=type=cache,dst=/go/pkg/mod/cache,sharing=locked \
-<<EOF
-    set -eux
-    apt-get -y -U install --no-install-recommends golang-go jq
-EOF
-
 WORKDIR /usr/src
-ADD https://github.com/matrix-construct/complement.git#403840348f6bcc9cc8ed1671dc2f638c2b1ce4ac complement
+ADD https://github.com/matrix-construct/complement.git#${complement_ref} complement
 
 WORKDIR /usr/src/complement
 ENV COMPLEMENT_BASE_IMAGE="complement-testee"
@@ -183,8 +150,9 @@ EOF
 FROM input AS complement-tester
 ARG complement_verbose=0
 ARG complement_debug=0
+ARG complement_dirty=0
 ARG complement_count=1
-ARG complement_parallel=16
+ARG complement_parallel=1
 ARG complement_shuffle=1337
 ARG complement_timeout="1h"
 ARG complement_run=".*"
@@ -193,9 +161,6 @@ ARG complement_tags="conduwuit_blacklist"
 ARG complement_tests="./tests/..."
 ARG complement_base_image
 
-WORKDIR /
-COPY --link --from=input . .
-
 WORKDIR /usr/src/complement
 ENV COMPLEMENT_DEBUG=$complement_debug
 ENV complement_parallel="$complement_parallel"
@@ -207,6 +172,7 @@ ENV complement_tests="$complement_tests"
 ENV complement_skip="$complement_skip"
 ENV complement_run="$complement_run"
 ENV complement_tests="$complement_tests"
+ENV COMPLEMENT_ENABLE_DIRTY_RUNS="$complement_dirty"
 ENV COMPLEMENT_ALWAYS_PRINT_SERVER_LOGS="$complement_verbose"
 ENV COMPLEMENT_HOSTNAME_RUNNING_COMPLEMENT="host.docker.internal"
 ENV COMPLEMENT_HOST_MOUNTS="/var/run/docker.sock:/var/run/docker.sock"

docker/Dockerfile.install

@@ -15,15 +15,17 @@ ENV src_path="${CARGO_TARGET_DIR}/${rust_target}/${cargo_target_profile}/tuwunel
 ENV dst_path="${install_prefix}/bin/tuwunel"
 COPY --from=bins $src_path $dst_path
 RUN <<EOF
-    ret=$(ldd "${dst_path}")
+    ldd -v "${dst_path}"
+    ret=$?
     if [ "$ret" = "0" ] && [ "$assert_linkage" = "static" ]; then
+        echo "($ret) expected a static binary"
         exit 1
     elif [ "$ret" != "0" ] && [ "$assert_linkage" = "dynamic" ]; then
+        echo "($ret) expected a dynamic binary"
         exit 1
     fi
 
     set -eux
-    ldd -v ${dst_path} || true
-    du -h ${dst_path}
-    sha1sum ${dst_path}
+    du -h "${dst_path}"
+    sha1sum "${dst_path}"
 EOF

docker/Dockerfile.matrix-rust-sdk

@@ -0,0 +1,100 @@
+# syntax = docker/dockerfile:1.11-labs
+
+FROM input AS rust-sdk-integration
+ARG sys_name
+ARG sys_version
+ARG feat_set
+ARG rust_target
+ARG rust_toolchain
+ARG cargo_profile
+ARG RUSTUP_HOME
+ARG CARGO_HOME
+ARG CARGO_TARGET
+ARG MRSDK_TARGET_DIR="/usr/src/matrix-rust-sdk/target"
+ARG mrsdk_target_share
+#ARG mrsdk_ref="integration"
+ARG mrsdk_ref="tuwunel-changes"
+ARG mrsdk_test_args=""
+ARG mrsdk_test_opts=""
+ARG mrsdk_skip_list=""
+ARG mrsdk_parallel=2
+ARG mrsdk_startup_delay="10s"
+ARG mrsdk_testee="/usr/bin/tuwunel"
+
+WORKDIR /usr/src
+ADD --link https://github.com/matrix-construct/matrix-rust-sdk.git#${mrsdk_ref} matrix-rust-sdk
+
+WORKDIR /etc
+COPY <<EOF tuwunel.toml
+    [global]
+    admin_room_notices = false
+    allow_device_name_federation = true
+    allow_guest_registration = true
+    allow_legacy_media = true
+    allow_public_room_directory_over_federation = true
+    allow_public_room_directory_without_auth = true
+    allow_registration = true
+    create_admin_room = false
+    ip_range_denylist = []
+    log = "debug,tuwunel=trace,h2=warn,hyper=warn"
+    log_colors = false
+    log_guest_registrations = false
+    log_span_events = "NONE"
+    log_thread_ids = true
+    media_compat_file_link = false
+    media_startup_check = true
+    query_trusted_key_servers_first = false
+    query_trusted_key_servers_first_on_join = false
+    rocksdb_log_level = "debug"
+    rocksdb_max_log_files = 1
+    rocksdb_paranoid_file_checks = true
+    rocksdb_recovery_mode = 0
+    trusted_servers = []
+    url_preview_domain_contains_allowlist = ["*"]
+    url_preview_domain_explicit_denylist = ["*"]
+    yes_i_am_very_very_sure_i_want_an_open_registration_server_prone_to_abuse = true
+EOF
+
+WORKDIR /usr/lib
+COPY --link --from=install /usr/lib .
+
+WORKDIR /usr/bin
+COPY --link --from=install /usr/bin/tuwunel .
+
+WORKDIR /usr/src/matrix-rust-sdk
+SHELL ["/bin/bash", "-c"]
+ENV RUST_BACKTRACE="full"
+ENV TUWUNEL_CONFIG="/etc/tuwunel.toml"
+ENV TUWUNEL_DATABASE_PATH="/var/db/tuwunel"
+ENV TUWUNEL_SERVER_NAME="localhost"
+ENV TUWUNEL_PORT="[8448]"
+ENV HOMESERVER_URL="http://localhost:8448"
+RUN \
+--mount=type=cache,dst=${RUSTUP_HOME}/downloads,sharing=shared \
+--mount=type=cache,dst=${CARGO_HOME}/registry,sharing=shared \
+--mount=type=cache,dst=${CARGO_HOME}/git,sharing=shared \
+--mount=type=cache,dst=${MRSDK_TARGET_DIR},id=${mrsdk_target_share},sharing=locked \
+<<EOF
+    set -eux
+
+    nohup ${mrsdk_testee[@]} 1> /var/log/tuwunel.log &
+    PID=$!; trap "sleep 10s; set +e; kill -QUIT ${PID}; wait ${PID}" EXIT
+    sleep "${mrsdk_startup_delay}"
+
+    rustup run ${rust_toolchain} \
+        cargo test \
+            --locked \
+            --release \
+            "--color=always" \
+            "--features=default" \
+            "--target=${rust_target}" \
+            "--target-dir=${MRSDK_TARGET_DIR}" \
+            "--package=matrix-sdk-integration-testing" \
+            ${mrsdk_test_args[@]} \
+            -- \
+            "--color=always" \
+            "--test-threads=${mrsdk_parallel}" \
+            ${mrsdk_skip_list[@]} \
+            ${mrsdk_test_opts[@]} \
+            ;
+EOF

docker/Dockerfile.nix

@@ -0,0 +1,91 @@
+# syntax = docker/dockerfile:1.11-labs
+
+FROM input AS nix-base
+
+WORKDIR /
+COPY --link --from=input . .
+
+RUN \
+--mount=type=cache,dst=/nix,sharing=shared \
+--mount=type=cache,dst=/root/.cache/nix,sharing=shared \
+--mount=type=cache,dst=/root/.local/state/nix,sharing=shared \
+<<EOF
+	set -eux
+	curl --proto '=https' --tlsv1.2 -L https://nixos.org/nix/install > nix-install
+	sh ./nix-install --daemon
+	rm nix-install
+EOF
+
+
+FROM nix-base AS build-nix
+
+WORKDIR /usr/src/tuwunel
+COPY --link --from=source /usr/src/tuwunel .
+RUN \
+--mount=type=cache,dst=/nix,sharing=shared \
+--mount=type=cache,dst=/root/.cache/nix,sharing=shared \
+--mount=type=cache,dst=/root/.local/state/nix,sharing=shared \
+<<EOF
+	set -eux
+
+	nix-build \
+		--verbose \
+		--cores 0 \
+		--max-jobs $(nproc) \
+		--log-format raw \
+		.
+
+	cp -afRL --copy-contents result /opt/tuwunel
+EOF
+
+
+FROM input AS smoke-nix
+
+WORKDIR /
+COPY --link --from=nix-base . .
+
+WORKDIR /usr/src/tuwunel
+COPY --link --from=source /usr/src/tuwunel .
+ENV TUWUNEL_DATABASE_PATH="/tmp/tuwunel/smoketest.db"
+ENV TUWUNEL_LOG="info"
+RUN \
+--mount=type=cache,dst=/nix,sharing=shared \
+--mount=type=cache,dst=/root/.cache/nix,sharing=shared \
+--mount=type=cache,dst=/root/.local/state/nix,sharing=shared \
+<<EOF
+    set -eux
+    alias nix="nix --extra-experimental-features nix-command --extra-experimental-features flakes"
+
+    nix run \
+        --verbose \
+        --cores 0 \
+        --max-jobs $(nproc) \
+        --log-format raw \
+        .#all-features \
+            -- \
+            -Otest='["smoke", "fresh"]' \
+            -Oserver_name=\"localhost\" \
+EOF
+
+
+FROM input AS nix-pkg
+
+WORKDIR /
+COPY --link --from=nix-base . .
+
+WORKDIR /usr/src/tuwunel
+COPY --link --from=source /usr/src/tuwunel .
+RUN \
+--mount=type=cache,dst=/nix,sharing=shared \
+--mount=type=cache,dst=/root/.cache/nix,sharing=shared \
+--mount=type=cache,dst=/root/.local/state/nix,sharing=shared \
+<<EOF
+    set -eux
+    alias nix="nix --extra-experimental-features nix-command --extra-experimental-features flakes"
+
+    ID=$(nix-store --realise $(nix path-info --derivation))
+
+    mkdir -p tuwunel
+    nix-store --export $ID > tuwunel/tuwunel.drv
+    tar -cvf /opt/tuwunel.nix.tar tuwunel
+EOF

docker/Dockerfile.rocksdb

@@ -28,10 +28,12 @@ FROM input AS rocksdb-build
 ARG rocksdb_shared=0
 ARG rocksdb_portable="1"
 ARG rocksdb_opt_level="3"
+ARG rocksdb_lto="-flto -ffat-lto-objects"
 ARG rocksdb_build_type="Release"
 ARG rocksdb_cxx_flags="-ftls-model=initial-exec"
 ARG rocksdb_make_verbose="ON"
 ARG rocksdb_make_rule_messages="OFF"
+ARG rocksdb_numa=0
 ARG rocksdb_jemalloc=1
 ARG rocksdb_iouring=1
 ARG rocksdb_zstd=1
@@ -59,7 +61,7 @@ RUN <<EOF
         "-DBUILD_SHARED_LIBS=${rocksdb_shared}" \
         "-DROCKSDB_BUILD_SHARED=${rocksdb_shared}" \
         "-DCMAKE_CXX_FLAGS:STRING=${rocksdb_cxx_flags}" \
-        "-DCMAKE_CXX_FLAGS_RELEASE:STRING=-g0 -O${rocksdb_opt_level} -DNDEBUG" \
+        "-DCMAKE_CXX_FLAGS_RELEASE:STRING=-g0 -O${rocksdb_opt_level} -DNDEBUG ${rocksdb_lto}" \
         "-DPORTABLE=${rocksdb_portable}" \
         "-DFAIL_ON_WARNINGS=0" \
         "-DUSE_RTTI=0" \
@@ -73,6 +75,7 @@ RUN <<EOF
         "-DWITH_TOOLS=0" \
         "-DWITH_TESTS=0" \
         "-DWITH_GFLAGS=0" \
+        "-DWITH_NUMA=${rocksdb_numa}" \
         "-DWITH_LIBURING=${rocksdb_iouring}" \
         "-DWITH_JEMALLOC=${rocksdb_jemalloc}" \
         "-DWITH_ZSTD=${rocksdb_zstd}" \

docker/Dockerfile.rust

@@ -6,9 +6,6 @@ ARG rust_target
 ARG rustup_version="1.28.2"
 ARG rustup_profile="minimal"
 
-WORKDIR /
-COPY --link --from=input . .
-
 WORKDIR ${RUST_HOME}
 RUN <<EOF
     set -eux
@@ -32,9 +29,6 @@ ARG CARGO_TERM_VERBOSE
 ARG rustup_components
 ARG cargo_installs
 
-WORKDIR /
-COPY --link --from=input . .
-
 WORKDIR ${RUST_HOME}
 ENV CARGO_TARGET="${rust_target}"
 ENV RUSTUP_HOME="${RUSTUP_HOME}"

docker/Dockerfile.source

@@ -3,9 +3,6 @@
 FROM input AS source
 ARG git_checkout
 
-WORKDIR /
-COPY --link --from=input . .
-
 ADD --link --keep-git-dir . /usr/src/tuwunel
 WORKDIR /usr/src/tuwunel
 RUN <<EOF
@@ -30,7 +27,6 @@ ARG JEMALLOC_OVERRIDE
 ARG ROCKSDB_LIB_DIR
 
 WORKDIR /
-COPY --link --from=input . .
 COPY --link --from=rust ${RUST_HOME} ${RUST_HOME}
 COPY --link --from=source /usr/src/tuwunel /usr/src/tuwunel
 
@@ -67,9 +63,6 @@ ARG RUSTUP_HOME
 ARG CARGO_HOME
 ARG CARGO_TARGET
 
-WORKDIR /
-COPY --link --from=input . .
-
 WORKDIR /usr/src/tuwunel
 RUN \
 --mount=type=cache,dst=${RUSTUP_HOME}/downloads,sharing=locked \

docker/Dockerfile.system

@@ -12,9 +12,6 @@ ARG var_lib_apt
 ARG packages
 ARG DEBIAN_FRONTEND
 
-WORKDIR /
-COPY --link --from=input . .
-
 ENV DEBIAN_FRONTEND="${DEBIAN_FRONTEND}"
 RUN \
 --mount=type=cache,dst=/var/cache,id=${var_cache},sharing=locked \

docker/bake.hcl

@@ -1,4 +1,4 @@
-variable "CI" {}
+
 variable "GITHUB_ACTOR" {}
 variable "GITHUB_REPOSITORY" {}
 variable "GITHUB_REF" {}
@@ -84,6 +84,9 @@ variable "rocksdb_build_type" {
 variable "rocksdb_make_verbose" {
     default = "ON"
 }
+variable "rocksdb_numa" {
+    default = "0"
+}
 
 # Complement options
 variable "complement_count" {
@@ -184,7 +187,6 @@ dynamic_libs = [
 
 nightly_rustflags = [
     "--cfg tokio_unstable",
-    "--cfg tuwunel_bench",
     "--allow=unstable-features",
     "-Z crate-attr=feature(test)",
     "-Z enforce-type-length-limit",
@@ -240,7 +242,15 @@ group "tests" {
         "docs",
         "unit",
         "smoke",
+        "integration",
+        "matrix-compliance",
+    ]
+}
+
+group "matrix-compliance" {
+    targets = [
         "complement",
+        "rust-sdk-integ",
     ]
 }
 
@@ -408,7 +418,7 @@ target "complement-base" {
         elem("complement-config", [sys_name, sys_version, sys_target])
     ]
     contexts = {
-        input = elem("target:base", [sys_name, sys_version, sys_target])
+        input = elem("target:builder", [sys_name, sys_version, sys_target])
     }
     args = complement_args
 }
@@ -429,6 +439,116 @@ target "complement-config" {
     }
 }
 
+#
+# Integration tests
+#
+
+group "integration" {
+    targets = [
+        "integ",
+        "rust-sdk-integ",
+    ]
+}
+
+variable "valgrind_flags" {
+    default = "--error-exitcode=1 --exit-on-first-error=yes --undef-value-errors=no --leak-check=no"
+}
+
+target "rust-sdk-valgrind" {
+    name = elem("rust-sdk-valgrind", [cargo_profile, rust_toolchain, rust_target, feat_set, sys_name, sys_version, sys_target])
+    tags = [
+        elem_tag("rust-sdk-valgrind", [cargo_profile, rust_toolchain, rust_target, feat_set, sys_name, sys_version, sys_target], "latest"),
+    ]
+    matrix = cargo_rust_feat_sys
+    inherits = [
+        elem("rust-sdk-integ", [cargo_profile, rust_toolchain, rust_target, feat_set, sys_name, sys_version, sys_target])
+    ]
+    contexts = {
+        input = elem("target:rust", [rust_toolchain, rust_target, sys_name, sys_version, sys_target])
+        install = elem("target:install", [cargo_profile, rust_toolchain, rust_target, feat_set, sys_name, sys_version, sys_target])
+    }
+    args = {
+        VALGRINDFLAGS = "${valgrind_flags}"
+        mrsdk_testee = "valgrind ${valgrind_flags} /usr/bin/tuwunel"
+        mrsdk_test_args = ""
+        mrsdk_startup_delay = "30s"
+        mrsdk_skip_list =<<EOF
+            --skip test_delayed_invite_response_and_sent_message_decryption
+            --skip test_history_share_on_invite_pin_violation
+EOF
+    }
+}
+
+target "rust-sdk-integ" {
+    name = elem("rust-sdk-integ", [cargo_profile, rust_toolchain, rust_target, feat_set, sys_name, sys_version, sys_target])
+    tags = [
+        elem_tag("rust-sdk-integ", [cargo_profile, rust_toolchain, rust_target, feat_set, sys_name, sys_version, sys_target], "latest"),
+    ]
+    output = ["type=docker,compression=zstd,mode=max,compression-level=${zstd_image_compress_level}"]
+    cache_to = ["type=local,compression=zstd,mode=max,compression-level=${cache_compress_level}"]
+    target = "rust-sdk-integration"
+    dockerfile = "${docker_dir}/Dockerfile.matrix-rust-sdk"
+    matrix = cargo_rust_feat_sys
+    inherits = [
+        elem("rust", [rust_toolchain, rust_target, sys_name, sys_version, sys_target]),
+        elem("integ", [cargo_profile, rust_toolchain, rust_target, feat_set, sys_name, sys_version, sys_target]),
+    ]
+    contexts = {
+        input = elem("target:rust", [rust_toolchain, rust_target, sys_name, sys_version, sys_target])
+        install = elem("target:install", [cargo_profile, rust_toolchain, rust_target, feat_set, sys_name, sys_version, sys_target])
+    }
+    args = {
+        mrsdk_target_share = "/usr/src/matrix-rust-sdk/target/${sys_name}/${sys_version}/${rust_target}/${rust_toolchain}/_shared_cache"
+
+        mrsdk_testee = "/usr/bin/tuwunel"
+        mrsdk_test_args = "--no-fail-fast"
+
+        mrsdk_skip_list =<<EOF
+            --skip test_delayed_invite_response_and_sent_message_decryption
+EOF
+    }
+}
+
+target "integ-valgrind" {
+    name = elem("integ-valgrind", [cargo_profile, rust_toolchain, rust_target, feat_set, sys_name, sys_version, sys_target])
+    tags = [
+        elem_tag("integ-valgrind", [cargo_profile, rust_toolchain, rust_target, feat_set, sys_name, sys_version, sys_target], "latest"),
+    ]
+    matrix = cargo_rust_feat_sys
+    inherits = [
+        elem("integ", [cargo_profile, rust_toolchain, rust_target, feat_set, sys_name, sys_version, sys_target]),
+    ]
+    contexts = {
+        input = elem("target:build-tests", [cargo_profile, rust_toolchain, rust_target, feat_set, sys_name, sys_version, sys_target])
+    }
+    args = {
+        VALGRINDFLAGS = "${valgrind_flags}"
+        cargo_cmd = "valgrind test"
+        cargo_args = "--test=*"
+    }
+}
+
+target "integ" {
+    name = elem("integ", [cargo_profile, rust_toolchain, rust_target, feat_set, sys_name, sys_version, sys_target])
+    tags = [
+        elem_tag("integ", [cargo_profile, rust_toolchain, rust_target, feat_set, sys_name, sys_version, sys_target], "latest"),
+    ]
+    matrix = cargo_rust_feat_sys
+    inherits = [
+        elem("build-tests", [cargo_profile, rust_toolchain, rust_target, feat_set, sys_name, sys_version, sys_target]),
+    ]
+    contexts = {
+        input = elem("target:build-tests", [cargo_profile, rust_toolchain, rust_target, feat_set, sys_name, sys_version, sys_target])
+    }
+    args = {
+        TUWUNEL_DATABASE_PATH = "/tmp/integration.test.db"
+        cargo_cmd = (cargo_profile == "bench"? "bench": "test")
+        cargo_args = (cargo_profile == "bench"?
+            "--no-fail-fast --bench=*": "--no-fail-fast --test=*"
+        )
+    }
+}
+
 #
 # Smoke tests
 #
@@ -437,11 +557,26 @@ group "smoke" {
     targets = [
         "smoke-version",
         "smoke-startup",
+        #"smoke-nix",
         #"smoke-valgrind",
         #"smoke-perf",
     ]
 }
 
+target "smoke-nix" {
+    name = elem("smoke-nix", [cargo_profile, rust_toolchain, rust_target, feat_set, sys_name, sys_version, sys_target])
+    tags = [
+        elem_tag("smoke-nix", [cargo_profile, rust_toolchain, rust_target, feat_set, sys_name, sys_version, sys_target], "latest"),
+    ]
+    output = ["type=cacheonly,compression=zstd,mode=min,compression-level=${cache_compress_level}"]
+    dockerfile = "${docker_dir}/Dockerfile.nix"
+    target = "smoke-nix"
+    matrix = cargo_rust_feat_sys
+    inherits = [
+        elem("build-nix", [cargo_profile, rust_toolchain, rust_target, feat_set, sys_name, sys_version, sys_target]),
+    ]
+}
+
 target "smoke-valgrind" {
     name = elem("smoke-valgrind", [cargo_profile, rust_toolchain, rust_target, feat_set, sys_name, sys_version, sys_target])
     tags = [
@@ -514,6 +649,70 @@ target "tests-smoke" {
     }
 }
 
+#
+# Unit tests
+#
+
+target "unit-valgrind" {
+    name = elem("unit-valgrind", [cargo_profile, rust_toolchain, rust_target, feat_set, sys_name, sys_version, sys_target])
+    tags = [
+        elem_tag("unit-valgrind", [cargo_profile, rust_toolchain, rust_target, feat_set, sys_name, sys_version, sys_target], "latest"),
+    ]
+    target = "cargo"
+    matrix = cargo_rust_feat_sys
+    inherits = [
+        elem("unit", [cargo_profile, rust_toolchain, rust_target, feat_set, sys_name, sys_version, sys_target]),
+    ]
+    contexts = {
+        input = elem("target:unit", [cargo_profile, rust_toolchain, rust_target, feat_set, sys_name, sys_version, sys_target])
+    }
+    args = {
+        VALGRINDFLAGS = "${valgrind_flags}"
+        cargo_cmd = "valgrind test"
+        cargo_args = "--lib --bins"
+    }
+}
+
+target "unit" {
+    name = elem("unit", [cargo_profile, rust_toolchain, rust_target, feat_set, sys_name, sys_version, sys_target])
+    tags = [
+        elem_tag("unit", [cargo_profile, rust_toolchain, rust_target, feat_set, sys_name, sys_version, sys_target], "latest"),
+    ]
+    target = "cargo"
+    matrix = cargo_rust_feat_sys
+    inherits = [
+        elem("build-tests", [cargo_profile, rust_toolchain, rust_target, feat_set, sys_name, sys_version, sys_target]),
+    ]
+    contexts = {
+        input = elem("target:build-tests", [cargo_profile, rust_toolchain, rust_target, feat_set, sys_name, sys_version, sys_target])
+    }
+    args = {
+        cargo_cmd = (cargo_profile == "bench"? "bench": "test")
+        cargo_args = (cargo_profile == "bench"?
+            "--no-fail-fast --lib": "--no-fail-fast --lib --bins"
+        )
+    }
+}
+
+target "docs" {
+    name = elem("docs", [cargo_profile, rust_toolchain, rust_target, feat_set, sys_name, sys_version, sys_target])
+    tags = [
+        elem_tag("docs", [cargo_profile, rust_toolchain, rust_target, feat_set, sys_name, sys_version, sys_target], "latest"),
+    ]
+    target = "cargo"
+    matrix = cargo_rust_feat_sys
+    inherits = [
+        elem("build-tests", [cargo_profile, rust_toolchain, rust_target, feat_set, sys_name, sys_version, sys_target]),
+    ]
+    contexts = {
+        input = elem("target:build-tests", [cargo_profile, rust_toolchain, rust_target, feat_set, sys_name, sys_version, sys_target])
+    }
+    args = {
+        cargo_cmd = "test"
+        cargo_args = "--doc --no-fail-fast"
+    }
+}
+
 #
 # Installation
 #
@@ -669,6 +868,12 @@ target "install" {
     }
     args = {
         install_prefix = install_prefix
+        assert_linkage = (
+            substr(cargo_profile, 0, 5) == "bench"?     "static":
+            substr(cargo_profile, 0, 7) == "release"?   "static":
+            substr(rust_toolchain, 0, 6) == "stable"?   "static":
+            ""
+        )
     }
 }
 
@@ -678,6 +883,7 @@ target "install" {
 
 group "pkg" {
     targets = [
+        "nix",
         "deb",
         "rpm",
         "deb-install",
@@ -789,47 +995,36 @@ target "build-deb" {
     }
 }
 
-#
-# Unit tests
-#
-
-target "unit" {
-    name = elem("unit", [cargo_profile, rust_toolchain, rust_target, feat_set, sys_name, sys_version, sys_target])
+target "nix" {
+    name = elem("nix", [cargo_profile, rust_toolchain, rust_target, feat_set, sys_name, sys_version, sys_target])
     tags = [
-        elem_tag("unit", [cargo_profile, rust_toolchain, rust_target, feat_set, sys_name, sys_version, sys_target], "latest"),
+        elem_tag("nix", [cargo_profile, rust_toolchain, rust_target, feat_set, sys_name, sys_version, sys_target], "latest"),
     ]
-    target = "cargo"
+    output = ["type=docker,compression=zstd,mode=min,compression-level=${zstd_image_compress_level}"]
+    target = "nix-pkg"
     matrix = cargo_rust_feat_sys
     inherits = [
-        elem("build-tests", [cargo_profile, rust_toolchain, rust_target, feat_set, sys_name, sys_version, sys_target]),
+        elem("build-nix", [cargo_profile, rust_toolchain, rust_target, feat_set, sys_name, sys_version, sys_target]),
     ]
-    contexts = {
-        input = elem("target:build-tests", [cargo_profile, rust_toolchain, rust_target, feat_set, sys_name, sys_version, sys_target])
-    }
-    args = {
-        cargo_cmd = (cargo_profile == "bench"? "bench": "test")
-        cargo_args = (rust_toolchain == "nightly"?
-            "--no-fail-fast --all-targets": "--no-fail-fast --bins --tests"
-        )
-    }
 }
 
-target "docs" {
-    name = elem("docs", [cargo_profile, rust_toolchain, rust_target, feat_set, sys_name, sys_version, sys_target])
+target "build-nix" {
+    name = elem("build-nix", [cargo_profile, rust_toolchain, rust_target, feat_set, sys_name, sys_version, sys_target])
     tags = [
-        elem_tag("docs", [cargo_profile, rust_toolchain, rust_target, feat_set, sys_name, sys_version, sys_target], "latest"),
+        elem_tag("build-nix", [cargo_profile, rust_toolchain, rust_target, feat_set, sys_name, sys_version, sys_target], "latest"),
     ]
-    target = "cargo"
+    output = ["type=cacheonly,compression=zstd,mode=min,compression-level=${cache_compress_level}"]
+    cache_to = ["type=local,compression=zstd,mode=max,compression-level=${cache_compress_level}"]
+    dockerfile = "${docker_dir}/Dockerfile.nix"
+    target = "build-nix"
     matrix = cargo_rust_feat_sys
     inherits = [
-        elem("build-tests", [cargo_profile, rust_toolchain, rust_target, feat_set, sys_name, sys_version, sys_target]),
+        elem("builder", [sys_name, sys_version, sys_target]),
+        elem("source", [sys_name, sys_version, sys_target]),
     ]
     contexts = {
-        input = elem("target:build-tests", [cargo_profile, rust_toolchain, rust_target, feat_set, sys_name, sys_version, sys_target])
-    }
-    args = {
-        cargo_cmd = "test"
-        cargo_args = "--doc --no-fail-fast"
+        input = elem("target:builder", [sys_name, sys_version, sys_target]),
+        source = elem("target:source", [sys_name, sys_version, sys_target]),
     }
 }
 
@@ -854,7 +1049,6 @@ target "book" {
     }
     dockerfile-inline =<<EOF
         FROM input AS book
-        COPY --link --from=input . .
         RUN ["mdbook", "build", "-d", "/book", "/usr/src/tuwunel"]
 EOF
 }
@@ -922,7 +1116,7 @@ target "build-tests" {
     }
     args = {
         cargo_cmd = (cargo_profile == "bench"? "bench": "test")
-        cargo_args = "--no-run"
+        cargo_args = (cargo_profile == "bench"? "--no-run --benches": "--no-run --tests")
     }
 }
 
@@ -1200,6 +1394,8 @@ target "deps-base" {
         # cache key for unique artifact area
         cargo_target_artifact = "${cargo_tgt_dir_base}/${sys_name}/${sys_version}/${rust_target}/${rust_toolchain}/${cargo_profile}/${feat_set}/${git_ref_sha}"
         # cache key for hashed subdirs
+        cargo_share = "${cargo_tgt_dir_base}/${sys_name}/${sys_version}/${rust_toolchain}/${cargo_profile}/_shared_cache"
+        # cache key for hashed subdirs
         cargo_target_share = "${cargo_tgt_dir_base}/${sys_name}/${sys_version}/${rust_target}/${rust_toolchain}/${cargo_profile}/_shared_cache"
         # cased name of profile subdir within target complex
         cargo_target_profile = (
@@ -1210,8 +1406,8 @@ target "deps-base" {
 
         CARGO_PROFILE_TEST_DEBUG = "false"
         CARGO_PROFILE_TEST_INCREMENTAL = "false"
-        CARGO_PROFILE_BENCH_DEBUG = "limited"
-        CARGO_PROFILE_BENCH_LTO = "false"
+        CARGO_PROFILE_BENCH_DEBUG = "false"
+        CARGO_PROFILE_BENCH_LTO = "thin"
         CARGO_PROFILE_RELEASE_LTO = "thin"
         CARGO_PROFILE_RELEASE_DEBUGINFO_DEBUG = "limited"
         CARGO_PROFILE_RELEASE_DEBUGINFO_LTO = "off"
@@ -1224,7 +1420,7 @@ target "deps-base" {
                     join(" ", static_rustflags),
                     join(" ", static_nightly_rustflags),
                     join(" ", native_rustflags),
-                    "-C link-arg=-L/usr/lib/gcc/${sys_target_triple(sys_target)}/14", #FIXME
+                    "-C link-arg=-L/usr/lib/gcc/${sys_target_triple(sys_target)}/15", #FIXME
                     contains(split(",", cargo_feat_sets[feat_set]), "bzip2_compression")?
                         "-C link-arg=-l:libbz2.a": "",
                     contains(split(",", cargo_feat_sets[feat_set]), "lz4_compression")?
@@ -1238,7 +1434,7 @@ target "deps-base" {
                         "-C link-arg=-l:libgcc.a": "",
                 ]):
 
-            cargo_profile == "release" && rust_toolchain == "nightly"?
+            (cargo_profile == "release" || cargo_profile == "bench") && substr(rust_toolchain, 0, 7) == "nightly"?
                 join(" ", [
                     join(" ", rustflags),
                     join(" ", nightly_rustflags),
@@ -1246,7 +1442,7 @@ target "deps-base" {
                     join(" ", static_nightly_rustflags),
                     sys_target_triple(sys_target) == "x86_64-linux-gnu"?
                         "-C target-cpu=${sys_target_isa(sys_target)}": "",
-                    "-C link-arg=-L/usr/lib/gcc/${sys_target_triple(sys_target)}/14", #FIXME
+                    "-C link-arg=-L/usr/lib/gcc/${sys_target_triple(sys_target)}/15", #FIXME
                     contains(split(",", cargo_feat_sets[feat_set]), "bzip2_compression")?
                         "-C link-arg=-l:libbz2.a": "",
                     contains(split(",", cargo_feat_sets[feat_set]), "lz4_compression")?
@@ -1260,13 +1456,13 @@ target "deps-base" {
                         "-C link-arg=-l:libgcc.a": "",
                 ]):
 
-            cargo_profile == "release" || cargo_profile == "release-debuginfo"?
+            cargo_profile == "release" || cargo_profile == "release-debuginfo" || cargo_profile == "bench"?
                 join(" ", [
                     join(" ", rustflags),
                     join(" ", static_rustflags),
                     sys_target_triple(sys_target) == "x86_64-linux-gnu"?
                         "-C target-cpu=${sys_target_isa(sys_target)}": "",
-                    "-C link-arg=-L/usr/lib/gcc/${sys_target_triple(sys_target)}/14", #FIXME
+                    "-C link-arg=-L/usr/lib/gcc/${sys_target_triple(sys_target)}/15", #FIXME
                     contains(split(",", cargo_feat_sets[feat_set]), "bzip2_compression")?
                         "-C link-arg=-l:libbz2.a": "",
                     contains(split(",", cargo_feat_sets[feat_set]), "lz4_compression")?
@@ -1280,13 +1476,13 @@ target "deps-base" {
                         "-C link-arg=-l:libgcc.a": "",
                 ]):
 
-            rust_toolchain == "stable"?
+            substr(rust_toolchain, 0, 6) == "stable"?
                 join(" ", [
                     join(" ", rustflags),
                     join(" ", static_rustflags),
                     sys_target_triple(sys_target) == "x86_64-linux-gnu"?
                         "-C target-cpu=${sys_target_isa(sys_target)}": "",
-                    "-C link-arg=-L/usr/lib/gcc/${sys_target_triple(sys_target)}/14", #FIXME
+                    "-C link-arg=-L/usr/lib/gcc/${sys_target_triple(sys_target)}/15", #FIXME
                     contains(split(",", cargo_feat_sets[feat_set]), "bzip2_compression")?
                         "-C link-arg=-l:libbz2.a": "",
                     contains(split(",", cargo_feat_sets[feat_set]), "lz4_compression")?
@@ -1300,7 +1496,7 @@ target "deps-base" {
                         "-C link-arg=-l:libgcc.a": "",
                 ]):
 
-            rust_toolchain == "nightly"?
+            substr(rust_toolchain, 0, 7) == "nightly"?
                 join(" ", [
                     join(" ", rustflags),
                     join(" ", nightly_rustflags),
@@ -1365,6 +1561,7 @@ target "rocksdb-build" {
         rocksdb_zstd = contains(split(",", cargo_feat_sets[feat_set]), "zstd_compression")? 1: 0
         rocksdb_jemalloc = contains(split(",", cargo_feat_sets[feat_set]), "jemalloc")? 1: 0
         rocksdb_iouring = contains(split(",", cargo_feat_sets[feat_set]), "io_uring")? 1: 0
+        rocksdb_numa = rocksdb_numa
         rocksdb_shared = 0
         rocksdb_opt_level = rocksdb_opt_level
         rocksdb_build_type = rocksdb_build_type
@@ -1523,12 +1720,13 @@ rustup_components = [
 ]
 
 cargo_installs = [
-    "cargo-chef",
     "cargo-audit",
-    "cargo-deb",
     #"cargo-arch",
+    "cargo-chef",
+    "cargo-deb",
     "cargo-generate-rpm",
-    #"lychee",
+    "cargo-valgrind",
+    "lychee",
     "mdbook",
     "typos-cli",
 ]
@@ -1611,13 +1809,22 @@ kitchen_packages = [
     "clang",
     "cmake",
     "curl",
+    "gawk",
     "git",
+    "golang-go",
     "gzip",
+    "jq",
     "libc6-dev",
+    "libclang-dev",
+    "libnuma-dev",
     "libssl-dev",
+    "libsqlite3-dev",
     "make",
+    "nix-bin",
+    "openssl",
     "pkg-config",
     "pkgconf",
+    "valgrind",
     "xz-utils",
 ]
 

docker/complement.sh

@@ -16,6 +16,7 @@ default_sys_target="x86_64-v1-linux-gnu"
 default_sys_version="testing-slim"
 
 default_complement_verbose=0
+default_complement_dirty=0
 default_complement_count=1
 default_complement_parallel=1
 default_complement_shuffle=0
@@ -31,9 +32,7 @@ skip="${skip}|TestRoomCreate/Parallel/POST_/createRoom_makes_a_room_with_a_topic
 skip="${skip}|TestLogin/parallel/POST_/"
 skip="${skip}|TestUnbanViaInvite"
 skip="${skip}|TestRoomState/Parallel/GET_/publicRooms_lists_newly-created_room"
-# flakes due to timeout in debug-mode
-skip="${skip}|TestMSC4297StateResolutionV2_1_starts_from_empty_set"
-skip="${skip}|TestMSC4297StateResolutionV2_1_includes_conflicted_subgraph"
+skip="${skip}|TestThreadReceiptsInSyncMSC4102"
 
 set -a
 cargo_profile="${cargo_profile:-$default_cargo_profile}"
@@ -53,6 +52,7 @@ set +a
 envs=""
 envs="$envs -e complement_verbose=${complement_verbose:-$default_complement_verbose}"
 envs="$envs -e complement_count=${complement_count:-$default_complement_count}"
+envs="$envs -e complement_dirty=${complement_dirty:-$default_complement_dirty}"
 envs="$envs -e complement_parallel=${complement_parallel:-$default_complement_parallel}"
 envs="$envs -e complement_shuffle=${complement_shuffle:-$default_complement_shuffle}"
 envs="$envs -e complement_timeout=${complement_timeout:-$default_complement_timeout}"
@@ -82,13 +82,13 @@ if test "$CI" = "true"; then
 fi
 
 output_src="$cid:/usr/src/complement/full_output.jsonl"
-output_dst="complement.jsonl"
+output_dst="tests/complement/logs.jsonl"
 extract_output() {
 	docker cp "$output_src" "$output_dst"
 }
 
 result_src="$cid:/usr/src/complement/new_results.jsonl"
-result_dst="tests/test_results/complement/test_results.jsonl"
+result_dst="tests/complement/results.jsonl"
 extract_results() {
 	docker cp "$result_src" "$result_dst"
 }

docs/SUMMARY.md

@@ -5,6 +5,8 @@ # Summary
   - [Examples](configuration/examples.md)
 - [Deploying](deploying.md)
   - [Generic](deploying/generic.md)
+    - [Reverse Proxy - Caddy](deploying/reverse-proxy-caddy.md)
+    - [Reverse Proxy - Nginx](deploying/reverse-proxy-nginx.md)
   - [NixOS](deploying/nixos.md)
   - [Docker](deploying/docker.md)
   - [Kubernetes](deploying/kubernetes.md)

docs/deploying/docker-compose.for-traefik.yml

@@ -20,7 +20,6 @@ services:
       TUWUNEL_REGISTRATION_TOKEN: 'YOUR_TOKEN' # A registration token is required when registration is allowed.
       #TUWUNEL_YES_I_AM_VERY_VERY_SURE_I_WANT_AN_OPEN_REGISTRATION_SERVER_PRONE_TO_ABUSE: 'true'
       TUWUNEL_ALLOW_FEDERATION: 'true'
-      TUWUNEL_ALLOW_CHECK_FOR_UPDATES: 'true'
       TUWUNEL_TRUSTED_SERVERS: '["matrix.org"]'
       #TUWUNEL_LOG: warn,state_res=warn
       TUWUNEL_ADDRESS: 0.0.0.0

docs/deploying/docker-compose.with-caddy.yml

@@ -36,7 +36,6 @@ services:
             TUWUNEL_REGISTRATION_TOKEN: 'YOUR_TOKEN' # A registration token is required when registration is allowed.
             #TUWUNEL_YES_I_AM_VERY_VERY_SURE_I_WANT_AN_OPEN_REGISTRATION_SERVER_PRONE_TO_ABUSE: 'true'
             TUWUNEL_ALLOW_FEDERATION: 'true'
-            TUWUNEL_ALLOW_CHECK_FOR_UPDATES: 'true'
             TUWUNEL_TRUSTED_SERVERS: '["matrix.org"]'
             #TUWUNEL_LOG: warn,state_res=warn
             TUWUNEL_ADDRESS: 0.0.0.0

docs/deploying/docker-compose.with-traefik.yml

@@ -26,7 +26,6 @@ services:
       # TUWUNEL_LOG: info  # default is: "warn,state_res=warn"
       # TUWUNEL_ALLOW_ENCRYPTION: 'true'
       # TUWUNEL_ALLOW_FEDERATION: 'true'
-      # TUWUNEL_ALLOW_CHECK_FOR_UPDATES: 'true'
       # TUWUNEL_ALLOW_INCOMING_PRESENCE: true
       # TUWUNEL_ALLOW_OUTGOING_PRESENCE: true
       # TUWUNEL_ALLOW_LOCAL_PRESENCE: true

docs/deploying/docker-compose.yml

@@ -20,7 +20,6 @@ services:
             TUWUNEL_REGISTRATION_TOKEN: 'YOUR_TOKEN' # A registration token is required when registration is allowed.
             #TUWUNEL_YES_I_AM_VERY_VERY_SURE_I_WANT_AN_OPEN_REGISTRATION_SERVER_PRONE_TO_ABUSE: 'true'
             TUWUNEL_ALLOW_FEDERATION: 'true'
-            TUWUNEL_ALLOW_CHECK_FOR_UPDATES: 'true'
             TUWUNEL_TRUSTED_SERVERS: '["matrix.org"]'
             #TUWUNEL_LOG: warn,state_res=warn
             TUWUNEL_ADDRESS: 0.0.0.0

docs/deploying/generic.md

@@ -1,9 +1,9 @@
 # Generic deployment documentation
 
-> ### Getting help
+> [!TIP]
 >
-> If you run into any problems while setting up Tuwunel [open an issue on
-> GitHub](https://github.com/matrix-construct/tuwunel/issues/new).
+> Getting help: If you run into any problems while setting up Tuwunel
+> [open an issue on GitHub](https://github.com/matrix-construct/tuwunel/issues/new).
 
 ## Installing Tuwunel
 
@@ -12,24 +12,23 @@ ### Static prebuilt binary
 You may simply download the binary that fits your machine architecture (x86_64
 or aarch64). Run `uname -m` to see what you need.
 
-Prebuilt fully static musl binaries can be downloaded from the latest tagged
+Prebuilt fully static binaries can be downloaded from the latest tagged
 release [here](https://github.com/matrix-construct/tuwunel/releases/latest) or
-`main` CI branch workflow artifact output. These also include Debian/Ubuntu
-packages.
+`main` CI branch workflow artifact output. These also include `.deb` packages
+for Debian or Ubuntu and `.rpm` packages for Red Hat or Fedora.
 
-These can be curl'd directly from. `ci-bins` are CI workflow binaries by commit
-hash/revision, and `releases` are tagged releases. Sort by descending last
-modified for the latest.
+For the **best** performance; if using an `x86_64` CPU made in the last ~10 years,
+we recommend using the `-v3-` optimised packages. See below for a command to check
+what your system supports. If the server refuses to start or exits with an "Illegal
+Instruction" error you will need `-v2-` or `-v1-` packages instead. The database
+backend, RocksDB, benefits from `-v2-` or greater as it features performance
+critical hardware accelerated CRC32 hashing/checksumming.
 
-These binaries have jemalloc and io_uring statically linked and included with
-them, so no additional dynamic dependencies need to be installed.
-
-For the **best** performance; if using an `x86_64` CPU made in the last ~15 years,
-we recommend using the `-haswell-` optimised binaries. This sets
-`-march=haswell` which is the most compatible and highest performance with
-optimised binaries. The database backend, RocksDB, most benefits from this as it
-will then use hardware accelerated CRC32 hashing/checksumming which is critical
-for performance.
+Linux users can run this script to display which optimization levels they may
+choose:
+```
+cat /proc/cpuinfo | grep -Po '(avx|sse)[235]' | sort -u | sed 's/avx5/v4/;s/avx2/v3/;s/sse3/v2/;s/sse2/v1/' | sort
+```
 
 ### Compiling
 
@@ -142,69 +141,48 @@ ## Setting the correct file permissions
 
 ## Setting up the Reverse Proxy
 
-We recommend Caddy as a reverse proxy, as it is trivial to use, handling TLS certificates, reverse proxy headers, etc transparently with proper defaults.
-For other software, please refer to their respective documentation or online guides.
+We recommend Caddy as a reverse proxy, as it is trivial to use, handling TLS certificates, reverse proxy headers, etc. transparently with proper defaults. However, Nginx is also well-supported and widely used.
 
-### Caddy
+**Choose your reverse proxy:**
 
-After installing Caddy via your preferred method, create `/etc/caddy/conf.d/tuwunel_caddyfile`
-and enter this (substitute for your server name).
+- **[Caddy Setup Guide](reverse-proxy-caddy.md)** - Recommended for ease of use and automatic TLS
+- **[Nginx Setup Guide](reverse-proxy-nginx.md)** - Popular choice with extensive documentation
 
-```caddyfile
-your.server.name, your.server.name:8448 {
-    # TCP reverse_proxy
-    reverse_proxy localhost:8008
-    # UNIX socket
-    #reverse_proxy unix//run/tuwunel/tuwunel.sock
-}
-```
+### Quick Overview
 
-That's it! Just start and enable the service and you're set.
+Regardless of which reverse proxy you choose, you will need to:
 
-```bash
-sudo systemctl enable --now caddy
-```
+1. **Reverse proxy the following routes:**
+   - `/_matrix/` - core Matrix C-S and S-S APIs
+   - `/_tuwunel/` - ad-hoc Tuwunel routes such as `/local_user_count` and `/server_version`
 
-### Other Reverse Proxies
+2. **Optionally reverse proxy (recommended):**
+   - `/.well-known/matrix/client` and `/.well-known/matrix/server` if using Tuwunel to perform delegation (see the `[global.well_known]` config section)
+   - `/.well-known/matrix/support` if using Tuwunel to send the homeserver admin contact and support page (formerly known as MSC1929)
+   - `/` if you would like to see `hewwo from tuwunel woof!` at the root
 
-As we would prefer our users to use Caddy, we will not provide configuration files for other proxys.
+3. **Handle ports:**
+   - Port 443 (HTTPS) for client-server API
+   - Port 8448 for federation (if federating with other homeservers)
 
-You will need to reverse proxy everything under following routes:
-- `/_matrix/` - core Matrix C-S and S-S APIs
-- `/_tuwunel/` - ad-hoc Tuwunel routes such as `/local_user_count` and
-`/server_version`
-
-You can optionally reverse proxy the following individual routes:
-- `/.well-known/matrix/client` and `/.well-known/matrix/server` if using
-Tuwunel to perform delegation (see the `[global.well_known]` config section)
-- `/.well-known/matrix/support` if using Tuwunel to send the homeserver admin
-contact and support page (formerly known as MSC1929)
-- `/` if you would like to see `hewwo from tuwunel woof!` at the root
-
-See the following spec pages for more details on these files:
+See the following spec pages for more details on well-known files:
 - [`/.well-known/matrix/server`](https://spec.matrix.org/latest/client-server-api/#getwell-knownmatrixserver)
 - [`/.well-known/matrix/client`](https://spec.matrix.org/latest/client-server-api/#getwell-knownmatrixclient)
 - [`/.well-known/matrix/support`](https://spec.matrix.org/latest/client-server-api/#getwell-knownmatrixsupport)
 
 Examples of delegation:
-- <https://puppygock.gay/.well-known/matrix/server>
-- <https://puppygock.gay/.well-known/matrix/client>
+- <https://matrix.org/.well-known/matrix/server>
+- <https://matrix.org/.well-known/matrix/client>
 
-For Apache and Nginx there are many examples available online.
+### Other Reverse Proxies
 
-Lighttpd is not supported as it seems to mess with the `X-Matrix` Authorization
-header, making federation non-functional. If a workaround is found, feel free to share to get it added to the documentation here.
+_Specific contributions for other proxies are welcome!_
 
-If using Apache, you need to use `nocanon` in your `ProxyPass` directive to prevent httpd from messing with the `X-Matrix` header (note that Apache isn't very good as a general reverse proxy and we discourage the usage of it if you can).
+**Not Recommended:**
+- **Apache**: While possible, Apache requires special configuration (`nocanon` in `ProxyPass`) to prevent corruption of the `X-Matrix` header.
+- **Lighttpd**: Its proxy module alters the `X-Matrix` authorization header, breaking federation functionality.
 
-If using Nginx, you need to give Tuwunel the request URI using `$request_uri`, or like so:
-- `proxy_pass http://127.0.0.1:6167$request_uri;`
-- `proxy_pass http://127.0.0.1:6167;`
-
-Nginx users need to increase `client_max_body_size` (default is 1M) to match
-`max_request_size` defined in tuwunel.toml.
-
-## You're done
+## You are done
 
 Now you can start Tuwunel with:
 

docs/deploying/reverse-proxy-caddy.md

@@ -0,0 +1,48 @@
+# Reverse Proxy Setup - Caddy
+
+[<= Back to Generic Deployment Guide](generic.md#setting-up-the-reverse-proxy)
+
+We recommend Caddy as a reverse proxy, as it is trivial to use, handling TLS certificates, reverse proxy headers, etc. transparently with proper defaults.
+
+## Installation
+
+Install Caddy via your preferred method. Refer to the [official Caddy installation guide](https://caddyserver.com/docs/install) for your distribution.
+
+## Configuration
+
+After installing Caddy, create `/etc/caddy/conf.d/tuwunel_caddyfile` and enter this (substitute `your.server.name` with your actual server name):
+
+```caddyfile
+your.server.name, your.server.name:8448 {
+    # TCP reverse_proxy
+    reverse_proxy localhost:8008
+    # UNIX socket (alternative - comment out the line above and uncomment this)
+    #reverse_proxy unix//run/tuwunel/tuwunel.sock
+}
+```
+
+### What this does
+
+- Handles both port 443 (HTTPS) and port 8448 (Matrix federation) automatically
+- Automatically provisions and renews TLS certificates via Let's Encrypt
+- Sets all necessary reverse proxy headers correctly
+- Routes all traffic to Tuwunel listening on `localhost:8008`
+
+That's it! Just start and enable the service and you're set.
+
+```bash
+sudo systemctl enable --now caddy
+```
+
+## Verification
+
+After starting Caddy, verify it's working by checking:
+
+```bash
+curl https://your.server.name/_tuwunel/server_version
+curl https://your.server.name:8448/_tuwunel/server_version
+```
+
+---
+
+[=> Continue with "You're Done"](generic.md#you-are-done)

docs/deploying/reverse-proxy-nginx.md

@@ -0,0 +1,164 @@
+# Reverse Proxy Setup - Nginx
+
+[<= Back to Generic Deployment Guide](generic.md#setting-up-the-reverse-proxy)
+
+This guide shows you how to configure Nginx as a reverse proxy for Tuwunel with TLS support.
+
+## Installation
+
+Install Nginx via your preferred method. Most distributions include Nginx in their package repositories:
+
+```bash
+# Debian/Ubuntu
+sudo apt install nginx
+
+# Red Hat/Fedora
+sudo dnf install nginx
+
+# Arch Linux
+sudo pacman -S nginx
+```
+
+## Configuration
+
+Create a new configuration file at `/etc/nginx/sites-available/tuwunel` (or `/etc/nginx/conf.d/tuwunel.conf` on some distributions):
+
+```nginx
+# Client-Server API over HTTPS (port 443)
+server {
+  listen 443 ssl http2;
+  listen [::]:443 ssl http2;
+  server_name matrix.example.com;
+
+  # Nginx standard body size is 1MB, which is quite small for media uploads
+  # Increase this to match the max_request_size in your tuwunel.toml
+  client_max_body_size 100M;
+
+  # Forward requests to Tuwunel (listening on 127.0.0.1:8008)
+  location / {
+    proxy_pass http://127.0.0.1:8008;
+
+    # Preserve host and scheme - critical for proper Matrix operation
+    proxy_set_header Host $host;
+    proxy_set_header X-Forwarded-For $remote_addr;
+    proxy_set_header X-Forwarded-Proto https;
+  }
+
+  # TLS configuration (Let's Encrypt example using certbot)
+  ssl_certificate /etc/letsencrypt/live/matrix.example.com/fullchain.pem;
+  ssl_certificate_key /etc/letsencrypt/live/matrix.example.com/privkey.pem;
+}
+
+# Matrix Federation over HTTPS (port 8448)
+# Only needed if you want to federate with other homeservers
+# Don't forget to open port 8448 in your firewall!
+server {
+  listen 8448 ssl http2;
+  listen [::]:8448 ssl http2;
+  server_name matrix.example.com;
+
+  # Same body size increase for larger files
+  client_max_body_size 100M;
+
+  # Forward to the same local port as client-server API
+  location / {
+    proxy_pass http://127.0.0.1:8008;
+    proxy_set_header Host $host;
+    proxy_set_header X-Forwarded-For $remote_addr;
+    proxy_set_header X-Forwarded-Proto https;
+  }
+
+  # TLS configuration (same certificates as above)
+  ssl_certificate /etc/letsencrypt/live/matrix.example.com/fullchain.pem;
+  ssl_certificate_key /etc/letsencrypt/live/matrix.example.com/privkey.pem;
+}
+```
+
+### Important Notes
+
+- **Replace `matrix.example.com`** with your actual server name
+- **`client_max_body_size`**: Must match or exceed `max_request_size` in your `tuwunel.toml`
+- **Do NOT use `$request_uri`** in `proxy_pass` - while some guides suggest this, it's not necessary for Tuwunel and can cause issues
+- **IPv6**: The `listen [::]:443` and `listen [::]:8448` lines enable IPv6 support. Remove them if you don't need IPv6
+
+### TLS Certificates
+
+The example above uses Let's Encrypt certificates via certbot. To obtain certificates:
+
+```bash
+sudo certbot certonly --nginx -d matrix.example.com
+```
+
+Certbot will automatically handle renewal. Make sure to reload Nginx after certificate renewal:
+
+```bash
+sudo systemctl reload nginx
+```
+
+### Optional: Timeout Configuration
+
+The default Nginx timeouts are usually sufficient for Matrix operations. Element's long-polling `/sync` requests typically run for 30 seconds, which is within Nginx's default timeouts.
+
+However, if you experience federation retries or dropped long-poll connections, you can extend the timeouts by adding these lines inside your `location /` blocks:
+
+```nginx
+location / {
+  proxy_pass http://127.0.0.1:8008;
+  proxy_set_header Host $host;
+  proxy_set_header X-Forwarded-For $remote_addr;
+  proxy_set_header X-Forwarded-Proto https;
+
+  # Optional: Extend timeouts if experiencing issues
+  proxy_read_timeout 300s;
+  proxy_send_timeout 300s;
+}
+```
+
+## Enable the Configuration
+
+If using sites-available/sites-enabled structure:
+
+```bash
+sudo ln -s /etc/nginx/sites-available/tuwunel /etc/nginx/sites-enabled/
+```
+
+Test the configuration:
+
+```bash
+sudo nginx -t
+```
+
+If the test passes, reload Nginx:
+
+```bash
+sudo systemctl reload nginx
+```
+
+Enable Nginx to start on boot:
+
+```bash
+sudo systemctl enable nginx
+```
+
+## Verification
+
+After configuring Nginx, verify it's working by checking:
+
+```bash
+curl https://matrix.example.com/_tuwunel/server_version
+curl https://matrix.example.com:8448/_tuwunel/server_version
+```
+
+## Troubleshooting
+
+### Apache Compatibility Note
+
+If you're considering Apache instead of Nginx: Apache is not well-suited as a reverse proxy for Matrix homeservers. If you must use Apache, you need to use `nocanon` in your `ProxyPass` directive to prevent httpd from corrupting the `X-Matrix` authorization header, which will break federation.
+
+### Lighttpd is Not Supported
+
+Lighttpd has known issues with the `X-Matrix` authorization header, making federation non-functional. We do not recommend using Lighttpd with Tuwunel.
+
+---
+
+[=> Continue with "You're Done"](generic.md#you-are-done)

docs/deploying/tuwunel.container

@@ -1,5 +1,5 @@
 [Container]
-Environment=TUWUNEL_SERVER_NAME=your.domain.here TUWUNEL_DATABASE_PATH=/var/lib/tuwunel TUWUNEL_PORT=6167 TUWUNEL_MAX_REQUEST_SIZE=20000000 TUWUNEL_ALLOW_REGISTRATION=true TUWUNEL_REGISTRATION_TOKEN=YOUR_TOKEN TUWUNEL_ALLOW_FEDERATION=true TUWUNEL_ALLOW_CHECK_FOR_UPDATES=true TUWUNEL_TRUSTED_SERVERS=["matrix.org"] TUWUNEL_ADDRESS=0.0.0.0 # Add TUWUNEL_CONFIG: '/etc/tuwunel.toml' if the config is mapped
+Environment=TUWUNEL_SERVER_NAME=your.domain.here TUWUNEL_DATABASE_PATH=/var/lib/tuwunel TUWUNEL_PORT=6167 TUWUNEL_MAX_REQUEST_SIZE=20000000 TUWUNEL_ALLOW_REGISTRATION=true TUWUNEL_REGISTRATION_TOKEN=YOUR_TOKEN TUWUNEL_ALLOW_FEDERATION=true TUWUNEL_TRUSTED_SERVERS=["matrix.org"] TUWUNEL_ADDRESS=0.0.0.0 # Add TUWUNEL_CONFIG: '/etc/tuwunel.toml' if the config is mapped
 Image=docker.io/jevolk/tuwunel:latest
 PublishPort=8448:6167
 Volume=/path/to/db:/var/lib/tuwunel

flake.lock

@@ -10,11 +10,11 @@
         "nixpkgs-stable": "nixpkgs-stable"
       },
       "locked": {
-        "lastModified": 1748532342,
-        "narHash": "sha256-CvaKOUq8G10sghKpZhEB2UYjJoWhEkrDFggDgi7piUI=",
+        "lastModified": 1758711588,
+        "narHash": "sha256-0nZlCCDC5PfndsQJXXtcyrtrfW49I3KadGMDlutzaGU=",
         "owner": "zhaofengli",
         "repo": "attic",
-        "rev": "ce9373715fe3fac7a174a65a7e6d6baeba8cb4f9",
+        "rev": "12cbeca141f46e1ade76728bce8adc447f2166c6",
         "type": "github"
       },
       "original": {
@@ -29,14 +29,14 @@
         "devenv": "devenv",
         "flake-compat": "flake-compat_2",
         "git-hooks": "git-hooks",
-        "nixpkgs": "nixpkgs_4"
+        "nixpkgs": "nixpkgs_2"
       },
       "locked": {
-        "lastModified": 1748883665,
-        "narHash": "sha256-R0W7uAg+BLoHjMRMQ8+oiSbTq8nkGz5RDpQ+ZfxxP3A=",
+        "lastModified": 1763236786,
+        "narHash": "sha256-JB19RGXDr6loKSdqwvA15jhRHwf6+9Crq2glqqVar84=",
         "owner": "cachix",
         "repo": "cachix",
-        "rev": "f707778d902af4d62d8dd92c269f8e70de09acbe",
+        "rev": "938a275857047c300596092beaabaee6d892e243",
         "type": "github"
       },
       "original": {
@@ -58,16 +58,21 @@
         ],
         "git-hooks": [
           "cachix",
-          "devenv"
+          "devenv",
+          "git-hooks"
         ],
-        "nixpkgs": "nixpkgs_2"
+        "nixpkgs": [
+          "cachix",
+          "devenv",
+          "nixpkgs"
+        ]
       },
       "locked": {
-        "lastModified": 1744206633,
-        "narHash": "sha256-pb5aYkE8FOoa4n123slgHiOf1UbNSnKe5pEZC+xXD5g=",
+        "lastModified": 1752264895,
+        "narHash": "sha256-1zBPE/PNAkPNUsOWFET4J0cjlvziH8DOekesDmjND+w=",
         "owner": "cachix",
         "repo": "cachix",
-        "rev": "8a60090640b96f9df95d1ab99e5763a586be1404",
+        "rev": "47053aef762f452e816e44eb9a23fbc3827b241a",
         "type": "github"
       },
       "original": {
@@ -80,11 +85,11 @@
     "complement": {
       "flake": false,
       "locked": {
-        "lastModified": 1741891349,
-        "narHash": "sha256-YvrzOWcX7DH1drp5SGa+E/fc7wN3hqFtPbqPjZpOu1Q=",
+        "lastModified": 1761739261,
+        "narHash": "sha256-XdzSBbJIYG6thrHbo44/qBiMu5R4bayfy/dlWo9AXBA=",
         "owner": "matrix-construct",
         "repo": "complement",
-        "rev": "e587b3df569cba411aeac7c20b6366d03c143745",
+        "rev": "350d7666cab14cb0051ef53da7a1b0b3216d7269",
         "type": "github"
       },
       "original": {
@@ -95,18 +100,12 @@
       }
     },
     "crane": {
-      "inputs": {
-        "nixpkgs": [
-          "attic",
-          "nixpkgs"
-        ]
-      },
       "locked": {
-        "lastModified": 1722960479,
-        "narHash": "sha256-NhCkJJQhD5GUib8zN9JrmYGMwt4lCRp6ZVNzIiYCl0Y=",
+        "lastModified": 1751562746,
+        "narHash": "sha256-smpugNIkmDeicNz301Ll1bD7nFOty97T79m4GUMUczA=",
         "owner": "ipetkov",
         "repo": "crane",
-        "rev": "4c6c77920b8d44cd6660c1621dea6b3fc4b4c4f4",
+        "rev": "aed2020fd3dc26e1e857d4107a5a67a33ab6c1fd",
         "type": "github"
       },
       "original": {
@@ -117,11 +116,11 @@
     },
     "crane_2": {
       "locked": {
-        "lastModified": 1748970125,
-        "narHash": "sha256-UDyigbDGv8fvs9aS95yzFfOKkEjx1LO3PL3DsKopohA=",
+        "lastModified": 1763511871,
+        "narHash": "sha256-KKZWi+ij7oT0Ag8yC6MQkzfHGcytyjMJDD+47ZV1YNU=",
         "owner": "ipetkov",
         "repo": "crane",
-        "rev": "323b5746d89e04b22554b061522dfce9e4c49b18",
+        "rev": "099f9014bc8d0cd6e445470ea1df0fd691d5a548",
         "type": "github"
       },
       "original": {
@@ -138,6 +137,7 @@
           "cachix",
           "flake-compat"
         ],
+        "flake-parts": "flake-parts_2",
         "git-hooks": [
           "cachix",
           "git-hooks"
@@ -149,11 +149,11 @@
         ]
       },
       "locked": {
-        "lastModified": 1748273445,
-        "narHash": "sha256-5V0dzpNgQM0CHDsMzh+ludYeu1S+Y+IMjbaskSSdFh0=",
+        "lastModified": 1760560333,
+        "narHash": "sha256-goJQdVl9oDgCxF9CggPUw1DvB4gsot1jzMmz9px8Du8=",
         "owner": "cachix",
         "repo": "devenv",
-        "rev": "668a50d8b7bdb19a0131f53c9f6c25c9071e1ffb",
+        "rev": "0a4043938f540027e562c5a0feebbe6be872c3ea",
         "type": "github"
       },
       "original": {
@@ -170,11 +170,11 @@
         "rust-analyzer-src": "rust-analyzer-src"
       },
       "locked": {
-        "lastModified": 1749883145,
-        "narHash": "sha256-RlcGw3vAnbI3cfZn8aFaovNUd7312VZh+/FDWkqdA7E=",
+        "lastModified": 1763707297,
+        "narHash": "sha256-Bd9VGavwFBLpyU4pjiWfv73gUibNj8dc3xmOW8ff3bI=",
         "owner": "nix-community",
         "repo": "fenix",
-        "rev": "a804172f150bcf81262655324e583bb0cd0f28dd",
+        "rev": "7c2d3a165a4a080fdcb6c191d8f9768281c99f75",
         "type": "github"
       },
       "original": {
@@ -187,11 +187,11 @@
     "flake-compat": {
       "flake": false,
       "locked": {
-        "lastModified": 1696426674,
-        "narHash": "sha256-kvjfFW7WAETZlt09AgDn1MrtKzP7t90Vf7vypd3OL1U=",
+        "lastModified": 1747046372,
+        "narHash": "sha256-CIVLLkVgvHYbgI2UpXvIIBJ12HWgX+fjA8Xf8PUmqCY=",
         "owner": "edolstra",
         "repo": "flake-compat",
-        "rev": "0f9255e01c2351cc7d116c072cb317785dd33b33",
+        "rev": "9100a0f413b0c601e0533d1d94ffd501ce2e7885",
         "type": "github"
       },
       "original": {
@@ -219,11 +219,11 @@
     "flake-compat_3": {
       "flake": false,
       "locked": {
-        "lastModified": 1747046372,
-        "narHash": "sha256-CIVLLkVgvHYbgI2UpXvIIBJ12HWgX+fjA8Xf8PUmqCY=",
+        "lastModified": 1761588595,
+        "narHash": "sha256-XKUZz9zewJNUj46b4AJdiRZJAvSZ0Dqj2BNfXvFlJC4=",
         "owner": "edolstra",
         "repo": "flake-compat",
-        "rev": "9100a0f413b0c601e0533d1d94ffd501ce2e7885",
+        "rev": "f387cd2afec9419c8ee37694406ca490c3f34ee5",
         "type": "github"
       },
       "original": {
@@ -241,11 +241,11 @@
         ]
       },
       "locked": {
-        "lastModified": 1722555600,
-        "narHash": "sha256-XOQkdLafnb/p9ij77byFQjDf5m5QYl9b2REiVClC+x4=",
+        "lastModified": 1751413152,
+        "narHash": "sha256-Tyw1RjYEsp5scoigs1384gIg6e0GoBVjms4aXFfRssQ=",
         "owner": "hercules-ci",
         "repo": "flake-parts",
-        "rev": "8471fe90ad337a8074e957b69ca4d0089218391d",
+        "rev": "77826244401ea9de6e3bac47c2db46005e1f30b5",
         "type": "github"
       },
       "original": {
@@ -259,16 +259,15 @@
         "nixpkgs-lib": [
           "cachix",
           "devenv",
-          "nix",
           "nixpkgs"
         ]
       },
       "locked": {
-        "lastModified": 1712014858,
-        "narHash": "sha256-sB4SWl2lX95bExY2gMFG5HIzvva5AVMJd4Igm+GpZNw=",
+        "lastModified": 1756770412,
+        "narHash": "sha256-+uWLQZccFHwqpGqr2Yt5VsW/PbeJVTn9Dk6SHWhNRPw=",
         "owner": "hercules-ci",
         "repo": "flake-parts",
-        "rev": "9126214d0a59633752a136528f5f3b9aa8565b7d",
+        "rev": "4524271976b625a4a605beefd893f270620fd751",
         "type": "github"
       },
       "original": {
@@ -309,11 +308,11 @@
         ]
       },
       "locked": {
-        "lastModified": 1747372754,
-        "narHash": "sha256-2Y53NGIX2vxfie1rOW0Qb86vjRZ7ngizoo+bnXU9D9k=",
+        "lastModified": 1760392170,
+        "narHash": "sha256-WftxJgr2MeDDFK47fQKywzC72L2jRc/PWcyGdjaDzkw=",
         "owner": "cachix",
         "repo": "git-hooks.nix",
-        "rev": "80479b6ec16fefd9c1db3ea13aeb038c60530f46",
+        "rev": "46d55f0aeb1d567a78223e69729734f3dca25a85",
         "type": "github"
       },
       "original": {
@@ -344,30 +343,14 @@
         "type": "github"
       }
     },
-    "libgit2": {
-      "flake": false,
-      "locked": {
-        "lastModified": 1697646580,
-        "narHash": "sha256-oX4Z3S9WtJlwvj0uH9HlYcWv+x1hqp8mhXl7HsLu2f0=",
-        "owner": "libgit2",
-        "repo": "libgit2",
-        "rev": "45fd9ed7ae1a9b74b957ef4f337bc3c8b3df01b5",
-        "type": "github"
-      },
-      "original": {
-        "owner": "libgit2",
-        "repo": "libgit2",
-        "type": "github"
-      }
-    },
     "liburing": {
       "flake": false,
       "locked": {
-        "lastModified": 1749816983,
-        "narHash": "sha256-p5hXfDe53Y4MVwL2+wKZYpy4OPGvqFFnOEvkMsFAO6c=",
+        "lastModified": 1763758538,
+        "narHash": "sha256-cDsxLOqeC7imBcArolTHvejSnWoadgpvDy1DJ2/3MOw=",
         "owner": "axboe",
         "repo": "liburing",
-        "rev": "ad83d3ab64894c16eaf21ef869656a5bddb93ca4",
+        "rev": "e1ef1e680ee38ed9116989155fca47921698c25f",
         "type": "github"
       },
       "original": {
@@ -381,11 +364,24 @@
       "inputs": {
         "flake-compat": [
           "cachix",
-          "devenv"
+          "devenv",
+          "flake-compat"
+        ],
+        "flake-parts": [
+          "cachix",
+          "devenv",
+          "flake-parts"
+        ],
+        "git-hooks-nix": [
+          "cachix",
+          "devenv",
+          "git-hooks"
+        ],
+        "nixpkgs": [
+          "cachix",
+          "devenv",
+          "nixpkgs"
         ],
-        "flake-parts": "flake-parts_2",
-        "libgit2": "libgit2",
-        "nixpkgs": "nixpkgs_3",
         "nixpkgs-23-11": [
           "cachix",
           "devenv"
@@ -393,34 +389,30 @@
         "nixpkgs-regression": [
           "cachix",
           "devenv"
-        ],
-        "pre-commit-hooks": [
-          "cachix",
-          "devenv"
         ]
       },
       "locked": {
-        "lastModified": 1745930071,
-        "narHash": "sha256-bYyjarS3qSNqxfgc89IoVz8cAFDkF9yPE63EJr+h50s=",
-        "owner": "domenkozar",
+        "lastModified": 1758763079,
+        "narHash": "sha256-Bx1A+lShhOWwMuy3uDzZQvYiBKBFcKwy6G6NEohhv6A=",
+        "owner": "cachix",
         "repo": "nix",
-        "rev": "b455edf3505f1bf0172b39a735caef94687d0d9c",
+        "rev": "6f0140527c2b0346df4afad7497baa08decb929f",
         "type": "github"
       },
       "original": {
-        "owner": "domenkozar",
-        "ref": "devenv-2.24",
+        "owner": "cachix",
+        "ref": "devenv-2.30.5",
         "repo": "nix",
         "type": "github"
       }
     },
     "nix-filter": {
       "locked": {
-        "lastModified": 1731533336,
-        "narHash": "sha256-oRam5PS1vcrr5UPgALW0eo1m/5/pls27Z/pabHNy2Ms=",
+        "lastModified": 1757882181,
+        "narHash": "sha256-+cCxYIh2UNalTz364p+QYmWHs0P+6wDhiWR4jDIKQIU=",
         "owner": "numtide",
         "repo": "nix-filter",
-        "rev": "f7653272fd234696ae94229839a99b73c9ab7de0",
+        "rev": "59c44d1909c72441144b93cf0f054be7fe764de5",
         "type": "github"
       },
       "original": {
@@ -438,11 +430,11 @@
         ]
       },
       "locked": {
-        "lastModified": 1729742964,
-        "narHash": "sha256-B4mzTcQ0FZHdpeWcpDYPERtyjJd/NIuaQ9+BV1h+MpA=",
+        "lastModified": 1737420293,
+        "narHash": "sha256-F1G5ifvqTpJq7fdkT34e/Jy9VCyzd5XfJ9TO8fHhJWE=",
         "owner": "nix-community",
         "repo": "nix-github-actions",
-        "rev": "e04df33f62cdcf93d73e9a04142464753a16db67",
+        "rev": "f4158fa080ef4503c8f4c820967d946c2af31ec9",
         "type": "github"
       },
       "original": {
@@ -453,11 +445,11 @@
     },
     "nixpkgs": {
       "locked": {
-        "lastModified": 1726042813,
-        "narHash": "sha256-LnNKCCxnwgF+575y0pxUdlGZBO/ru1CtGHIqQVfvjlA=",
+        "lastModified": 1751949589,
+        "narHash": "sha256-mgFxAPLWw0Kq+C8P3dRrZrOYEQXOtKuYVlo9xvPntt8=",
         "owner": "NixOS",
         "repo": "nixpkgs",
-        "rev": "159be5db480d1df880a0135ca0bfed84c2f88353",
+        "rev": "9b008d60392981ad674e04016d25619281550a9d",
         "type": "github"
       },
       "original": {
@@ -469,27 +461,27 @@
     },
     "nixpkgs-stable": {
       "locked": {
-        "lastModified": 1724316499,
-        "narHash": "sha256-Qb9MhKBUTCfWg/wqqaxt89Xfi6qTD3XpTzQ9eXi3JmE=",
+        "lastModified": 1751741127,
+        "narHash": "sha256-t75Shs76NgxjZSgvvZZ9qOmz5zuBE8buUaYD28BMTxg=",
         "owner": "NixOS",
         "repo": "nixpkgs",
-        "rev": "797f7dc49e0bc7fab4b57c021cdf68f595e47841",
+        "rev": "29e290002bfff26af1db6f64d070698019460302",
         "type": "github"
       },
       "original": {
         "owner": "NixOS",
-        "ref": "nixos-24.05",
+        "ref": "nixos-25.05",
         "repo": "nixpkgs",
         "type": "github"
       }
     },
     "nixpkgs_2": {
       "locked": {
-        "lastModified": 1733212471,
-        "narHash": "sha256-M1+uCoV5igihRfcUKrr1riygbe73/dzNnzPsmaLCmpo=",
+        "lastModified": 1760524057,
+        "narHash": "sha256-EVAqOteLBFmd7pKkb0+FIUyzTF61VKi7YmvP1tw4nEw=",
         "owner": "NixOS",
         "repo": "nixpkgs",
-        "rev": "55d15ad12a74eb7d4646254e13638ad0c4128776",
+        "rev": "544961dfcce86422ba200ed9a0b00dd4b1486ec5",
         "type": "github"
       },
       "original": {
@@ -501,43 +493,11 @@
     },
     "nixpkgs_3": {
       "locked": {
-        "lastModified": 1717432640,
-        "narHash": "sha256-+f9c4/ZX5MWDOuB1rKoWj+lBNm0z0rs4CK47HBLxy1o=",
+        "lastModified": 1763618868,
+        "narHash": "sha256-v5afmLjn/uyD9EQuPBn7nZuaZVV9r+JerayK/4wvdWA=",
         "owner": "NixOS",
         "repo": "nixpkgs",
-        "rev": "88269ab3044128b7c2f4c7d68448b2fb50456870",
-        "type": "github"
-      },
-      "original": {
-        "owner": "NixOS",
-        "ref": "release-24.05",
-        "repo": "nixpkgs",
-        "type": "github"
-      }
-    },
-    "nixpkgs_4": {
-      "locked": {
-        "lastModified": 1748190013,
-        "narHash": "sha256-R5HJFflOfsP5FBtk+zE8FpL8uqE7n62jqOsADvVshhE=",
-        "owner": "NixOS",
-        "repo": "nixpkgs",
-        "rev": "62b852f6c6742134ade1abdd2a21685fd617a291",
-        "type": "github"
-      },
-      "original": {
-        "owner": "NixOS",
-        "ref": "nixos-unstable",
-        "repo": "nixpkgs",
-        "type": "github"
-      }
-    },
-    "nixpkgs_5": {
-      "locked": {
-        "lastModified": 1749871736,
-        "narHash": "sha256-K9yBph93OLTNw02Q6e9CYFGrUhvEXnh45vrZqIRWfvQ=",
-        "owner": "NixOS",
-        "repo": "nixpkgs",
-        "rev": "6afe187897bef7933475e6af374c893f4c84a293",
+        "rev": "a8d610af3f1a5fb71e23e08434d8d61a466fc942",
         "type": "github"
       },
       "original": {
@@ -550,11 +510,11 @@
     "rocksdb": {
       "flake": false,
       "locked": {
-        "lastModified": 1749358049,
-        "narHash": "sha256-ZSjvAZBfZkJrBIpw8ANZMbJVb8AeuogvuAipGVE4Qe4=",
+        "lastModified": 1763593074,
+        "narHash": "sha256-aOV/jJjRjNJ3hrRqhCsXlIz05NvEhDF/j5Q5UOQuvp8=",
         "owner": "matrix-construct",
         "repo": "rocksdb",
-        "rev": "cf7f65d0b377af019661c240f9165b3ef60640c3",
+        "rev": "9a3a213b55df0b11408102c899a940675c0d90e4",
         "type": "github"
       },
       "original": {
@@ -575,18 +535,18 @@
         "flake-utils": "flake-utils",
         "liburing": "liburing",
         "nix-filter": "nix-filter",
-        "nixpkgs": "nixpkgs_5",
+        "nixpkgs": "nixpkgs_3",
         "rocksdb": "rocksdb"
       }
     },
     "rust-analyzer-src": {
       "flake": false,
       "locked": {
-        "lastModified": 1749829309,
-        "narHash": "sha256-t6x6/PKg8Shnkd3htrxf3WMgycfRLRWvN9JHAmGWf+s=",
+        "lastModified": 1763648203,
+        "narHash": "sha256-/WJdebbRD+m5vr2xy/bJdCpqd7YHSMapjuXAM/0lvtA=",
         "owner": "rust-lang",
         "repo": "rust-analyzer",
-        "rev": "a497f4114ccf24978accb56190e60d1e1659e0c7",
+        "rev": "eaaa2da9fbbfd7a79ff501e0563351cb2004574a",
         "type": "github"
       },
       "original": {

flake.nix

@@ -26,7 +26,7 @@
         file = ./rust-toolchain.toml;
 
         # See also `rust-toolchain.toml`
-        sha256 = "sha256-Qxt8XAuaUR2OMdKbN4u8dBJOhSHxS+uS06Wl9+flVEk=";
+        sha256 = "sha256-+9FmLhAOezBZCOziO0Qct1NOrfpjNsXxc/8I0c7BdKE=";
       };
 
       mkScope = pkgs: pkgs.lib.makeScope pkgs.newScope (self: {

nix/pkgs/complement/config.toml

@@ -21,7 +21,6 @@ media_startup_check = true
 prune_missing_media = true
 log_colors = true
 admin_room_notices = false
-allow_check_for_updates = false
 intentionally_unknown_config_option_for_testing = true
 rocksdb_log_level = "info"
 rocksdb_max_log_files = 1

nix/pkgs/main/default.nix

@@ -129,6 +129,7 @@ buildDepsOnlyEnv =
 
 buildPackageEnv = {
   TUWUNEL_VERSION_EXTRA = inputs.self.shortRev or inputs.self.dirtyShortRev or "";
+  TUWUNEL_DATABASE_PATH = "/var/tmp/tuwunel.db";
 } // buildDepsOnlyEnv // {
   # Only needed in static stdenv because these are transitive dependencies of rocksdb
   CARGO_BUILD_RUSTFLAGS = buildDepsOnlyEnv.CARGO_BUILD_RUSTFLAGS
@@ -201,8 +202,27 @@ craneLib.buildPackage ( commonAttrs // {
     env = buildDepsOnlyEnv;
   });
 
+  nativeCheckInputs = [
+    pkgsBuildHost.libredirect.hook
+  ];
+
+  preCheck =
+    let
+      fakeResolvConf = pkgsBuildHost.writeTextFile {
+        name = "resolv.conf";
+        text = ''
+          nameserver 0.0.0.0
+        '';
+      };
+    in
+    ''
+      export NIX_REDIRECTS="/etc/resolv.conf=${fakeResolvConf}"
+      export TUWUNEL_DATABASE_PATH="$(mktemp -d)/smoketest.db"
+    '';
   doCheck = true;
 
+  doBenchmark = false;
+
   cargoExtraArgs = "--no-default-features --locked "
     + lib.optionalString
       (features'' != [])

rpm/tuwunel.service

@@ -2,11 +2,9 @@
 Description=Tuwunel Matrix homeserver
 Wants=network-online.target
 After=network-online.target
-Alias=matrix-tuwunel.service
 Documentation=https://tuwunel.chat/
 
 [Service]
-DynamicUser=yes
 User=tuwunel
 Group=tuwunel
 Type=notify
@@ -63,3 +61,4 @@ StartLimitBurst=5
 
 [Install]
 WantedBy=multi-user.target
+Alias=matrix-tuwunel.service

rust-toolchain.toml

@@ -9,7 +9,7 @@
 # If you're having trouble making the relevant changes, bug a maintainer.
 
 [toolchain]
-channel = "1.88.0"
+channel = "1.89.0"
 profile = "minimal"
 components = [
     # For rust-analyzer

src/admin/Cargo.toml

@@ -11,6 +11,7 @@ version.workspace = true
 
 [lib]
 path = "mod.rs"
+bench = false
 crate-type = [
 	"rlib",
 #	"dylib",

src/admin/debug/commands.rs

@@ -28,10 +28,7 @@
 	},
 	warn,
 };
-use tuwunel_service::rooms::{
-	short::{ShortEventId, ShortRoomId},
-	state_compressor::HashSetCompressStateEvent,
-};
+use tuwunel_service::rooms::{short::ShortRoomId, state_compressor::HashSetCompressStateEvent};
 
 use crate::admin_command;
 
@@ -138,16 +135,8 @@ pub(super) async fn get_pdu(&self, event_id: OwnedEventId) -> Result {
 }
 
 #[admin_command]
-pub(super) async fn get_short_pdu(
-	&self,
-	shortroomid: ShortRoomId,
-	shorteventid: ShortEventId,
-) -> Result {
-	let pdu_id: RawPduId = PduId {
-		shortroomid,
-		shorteventid: shorteventid.into(),
-	}
-	.into();
+pub(super) async fn get_short_pdu(&self, shortroomid: ShortRoomId, count: i64) -> Result {
+	let pdu_id: RawPduId = PduId { shortroomid, count: count.into() }.into();
 
 	let pdu_json = self
 		.services
@@ -247,8 +236,8 @@ pub(super) async fn get_remote_pdu(
 
 	match self
 		.services
-		.sending
-		.send_federation_request(&server, ruma::api::federation::event::get_event::v1::Request {
+		.federation
+		.execute(&server, ruma::api::federation::event::get_event::v1::Request {
 			event_id: event_id.clone(),
 		})
 		.await
@@ -304,7 +293,7 @@ pub(super) async fn get_remote_pdu(
 
 #[admin_command]
 pub(super) async fn get_room_state(&self, room: OwnedRoomOrAliasId) -> Result {
-	let room_id = self.services.alias.resolve(&room).await?;
+	let room_id = self.services.alias.maybe_resolve(&room).await?;
 	let room_state: Vec<Raw<AnyStateEvent>> = self
 		.services
 		.state_accessor
@@ -338,11 +327,8 @@ pub(super) async fn ping(&self, server: OwnedServerName) -> Result {
 
 	match self
 		.services
-		.sending
-		.send_federation_request(
-			&server,
-			ruma::api::federation::discovery::get_server_version::v1::Request {},
-		)
+		.federation
+		.execute(&server, ruma::api::federation::discovery::get_server_version::v1::Request {})
 		.await
 	{
 		| Err(e) => {
@@ -582,8 +568,8 @@ pub(super) async fn force_set_room_state_from_server(
 
 	let remote_state_response = self
 		.services
-		.sending
-		.send_federation_request(&server_name, get_room_state::v1::Request {
+		.federation
+		.execute(&server_name, get_room_state::v1::Request {
 			room_id: room_id.clone(),
 			event_id: first_pdu.event_id().to_owned(),
 		})
@@ -918,7 +904,7 @@ pub(super) async fn database_files(&self, map: Option<String>, level: Option<i32
 	let mut files: Vec<_> = self
 		.services
 		.db
-		.db
+		.engine
 		.file_list()
 		.collect::<Result<_>>()?;
 
@@ -1023,7 +1009,7 @@ pub(super) async fn resync_database(&self) -> Result {
 
 	self.services
 		.db
-		.db
+		.engine
 		.update()
 		.map_err(|e| err!("Failed to update from primary: {e:?}"))
 }

src/admin/debug/mod.rs

@@ -4,7 +4,7 @@
 use clap::Subcommand;
 use ruma::{OwnedEventId, OwnedRoomId, OwnedRoomOrAliasId, OwnedServerName};
 use tuwunel_core::Result;
-use tuwunel_service::rooms::short::{ShortEventId, ShortRoomId};
+use tuwunel_service::rooms::short::ShortRoomId;
 
 use self::tester::TesterCommand;
 use crate::admin_command_dispatch;
@@ -43,8 +43,8 @@ pub(super) enum DebugCommand {
 		/// Shortroomid integer
 		shortroomid: ShortRoomId,
 
-		/// Shorteventid integer
-		shorteventid: ShortEventId,
+		/// PduCount integer
+		count: i64,
 	},
 
 	/// - Attempts to retrieve a PDU from a remote server. Inserts it into our

src/admin/query/account_data.rs

@@ -2,6 +2,7 @@
 use futures::StreamExt;
 use ruma::{OwnedRoomId, OwnedUserId};
 use tuwunel_core::Result;
+use tuwunel_database::Deserialized;
 
 use crate::{admin_command, admin_command_dispatch};
 
@@ -46,7 +47,7 @@ async fn changes_since(
 		.await;
 	let query_time = timer.elapsed();
 
-	self.write_str(&format!("Query completed in {query_time:?}:\n\n```rs\n{results:#?}\n```"))
+	self.write_str(&format!("Query completed in {query_time:?}:\n\n```rs\n{results:?}\n```"))
 		.await
 }
 
@@ -58,11 +59,12 @@ async fn account_data_get(
 	room_id: Option<OwnedRoomId>,
 ) -> Result {
 	let timer = tokio::time::Instant::now();
-	let results = self
+	let results: serde_json::Value = self
 		.services
 		.account_data
 		.get_raw(room_id.as_deref(), &user_id, &kind)
-		.await;
+		.await
+		.deserialized()?;
 	let query_time = timer.elapsed();
 
 	self.write_str(&format!("Query completed in {query_time:?}:\n\n```rs\n{results:#?}\n```"))

src/admin/query/mod.rs

@@ -10,6 +10,7 @@
 mod room_timeline;
 mod sending;
 mod short;
+mod sync;
 mod users;
 
 use clap::Subcommand;
@@ -20,7 +21,7 @@
 	presence::PresenceCommand, pusher::PusherCommand, raw::RawCommand, resolver::ResolverCommand,
 	room_alias::RoomAliasCommand, room_state_cache::RoomStateCacheCommand,
 	room_timeline::RoomTimelineCommand, sending::SendingCommand, short::ShortCommand,
-	users::UsersCommand,
+	sync::SyncCommand, users::UsersCommand,
 };
 use crate::admin_command_dispatch;
 
@@ -76,6 +77,10 @@ pub(super) enum QueryCommand {
 	#[command(subcommand)]
 	Short(ShortCommand),
 
+	/// - sync service
+	#[command(subcommand)]
+	Sync(SyncCommand),
+
 	/// - raw service
 	#[command(subcommand)]
 	Raw(RawCommand),

src/admin/query/raw.rs

@@ -16,16 +16,16 @@
 
 use crate::{admin_command, admin_command_dispatch};
 
-#[admin_command_dispatch]
+#[admin_command_dispatch(handler_prefix = "raw")]
 #[derive(Debug, Subcommand)]
 #[allow(clippy::enum_variant_names)]
 /// Query tables from database
 pub(crate) enum RawCommand {
 	/// - List database maps
-	RawMaps,
+	Maps,
 
 	/// - Raw database query
-	RawGet {
+	Get {
 		/// Map name
 		map: String,
 
@@ -37,17 +37,8 @@ pub(crate) enum RawCommand {
 		base64: bool,
 	},
 
-	/// - Raw database delete (for string keys)
-	RawDel {
-		/// Map name
-		map: String,
-
-		/// Key
-		key: String,
-	},
-
 	/// - Raw database keys iteration
-	RawKeys {
+	Keys {
 		/// Map name
 		map: String,
 
@@ -56,7 +47,7 @@ pub(crate) enum RawCommand {
 	},
 
 	/// - Raw database key size breakdown
-	RawKeysSizes {
+	KeysSizes {
 		/// Map name
 		map: Option<String>,
 
@@ -65,7 +56,7 @@ pub(crate) enum RawCommand {
 	},
 
 	/// - Raw database keys total bytes
-	RawKeysTotal {
+	KeysTotal {
 		/// Map name
 		map: Option<String>,
 
@@ -74,7 +65,7 @@ pub(crate) enum RawCommand {
 	},
 
 	/// - Raw database values size breakdown
-	RawValsSizes {
+	ValsSizes {
 		/// Map name
 		map: Option<String>,
 
@@ -83,7 +74,7 @@ pub(crate) enum RawCommand {
 	},
 
 	/// - Raw database values total bytes
-	RawValsTotal {
+	ValsTotal {
 		/// Map name
 		map: Option<String>,
 
@@ -92,7 +83,7 @@ pub(crate) enum RawCommand {
 	},
 
 	/// - Raw database items iteration
-	RawIter {
+	Iter {
 		/// Map name
 		map: String,
 
@@ -101,7 +92,7 @@ pub(crate) enum RawCommand {
 	},
 
 	/// - Raw database keys iteration
-	RawKeysFrom {
+	KeysFrom {
 		/// Map name
 		map: String,
 
@@ -114,7 +105,7 @@ pub(crate) enum RawCommand {
 	},
 
 	/// - Raw database items iteration
-	RawIterFrom {
+	IterFrom {
 		/// Map name
 		map: String,
 
@@ -127,7 +118,7 @@ pub(crate) enum RawCommand {
 	},
 
 	/// - Raw database record count
-	RawCount {
+	Count {
 		/// Map name
 		map: Option<String>,
 
@@ -135,7 +126,26 @@ pub(crate) enum RawCommand {
 		prefix: Option<String>,
 	},
 
-	/// - Compact database
+	/// - Raw database delete (for string keys) DANGER!!!
+	Del {
+		/// Map name
+		map: String,
+
+		/// Key
+		key: String,
+	},
+
+	/// - Clear database table DANGER!!!
+	Clear {
+		/// Map name
+		map: String,
+
+		/// Confirm
+		#[arg(long)]
+		confirm: bool,
+	},
+
+	/// - Compact database DANGER!!!
 	Compact {
 		#[arg(short, long, alias("column"))]
 		map: Option<Vec<String>>,
@@ -165,7 +175,7 @@ pub(crate) enum RawCommand {
 }
 
 #[admin_command]
-pub(super) async fn compact(
+pub(super) async fn raw_compact(
 	&self,
 	map: Option<Vec<String>>,
 	start: Option<String>,
@@ -425,6 +435,28 @@ pub(super) async fn raw_del(&self, map: String, key: String) -> Result {
 		.await
 }
 
+#[admin_command]
+pub(super) async fn raw_clear(&self, map: String, confirm: bool) -> Result {
+	let map = self.services.db.get(&map)?;
+
+	if !confirm {
+		return Err!("Are you really sure you want to clear all data? Add the --confirm option.");
+	}
+
+	let timer = Instant::now();
+	let cork = self.services.db.cork();
+	map.raw_keys()
+		.ignore_err()
+		.ready_for_each(|key| map.remove(&key))
+		.boxed()
+		.await;
+
+	drop(cork);
+	let query_time = timer.elapsed();
+	self.write_str(&format!("Operation completed in {query_time:?}"))
+		.await
+}
+
 #[admin_command]
 pub(super) async fn raw_get(&self, map: String, key: String, base64: bool) -> Result {
 	let map = self.services.db.get(&map)?;

src/admin/query/room_alias.rs

@@ -8,6 +8,13 @@
 #[derive(Debug, Subcommand)]
 /// All the getters and iterators from src/database/key_value/rooms/alias.rs
 pub(crate) enum RoomAliasCommand {
+	/// - Resolve any local or remote alias.
+	ResolveAlias {
+		/// Full room alias
+		alias: OwnedRoomAliasId,
+	},
+
+	/// - Resolve an alias on this server.
 	ResolveLocalAlias {
 		/// Full room alias
 		alias: OwnedRoomAliasId,
@@ -28,6 +35,13 @@ pub(super) async fn process(subcommand: RoomAliasCommand, context: &Context<'_>)
 	let services = context.services;
 
 	match subcommand {
+		| RoomAliasCommand::ResolveAlias { alias } => {
+			let timer = tokio::time::Instant::now();
+			let results = services.alias.resolve_alias(&alias).await;
+			let query_time = timer.elapsed();
+
+			write!(context, "Query completed in {query_time:?}:\n\n```rs\n{results:#?}\n```")
+		},
 		| RoomAliasCommand::ResolveLocalAlias { alias } => {
 			let timer = tokio::time::Instant::now();
 			let results = services.alias.resolve_local_alias(&alias).await;

src/admin/query/room_state_cache.rs

@@ -74,6 +74,10 @@ pub(crate) enum RoomStateCacheCommand {
 		user_id: OwnedUserId,
 		room_id: OwnedRoomId,
 	},
+
+	UserMemberships {
+		user_id: OwnedUserId,
+	},
 }
 
 pub(super) async fn process(subcommand: RoomStateCacheCommand, context: &Context<'_>) -> Result {
@@ -282,7 +286,7 @@ pub(super) async fn process(subcommand: RoomStateCacheCommand, context: &Context
 			let timer = tokio::time::Instant::now();
 			let results: Vec<_> = services
 				.state_cache
-				.rooms_invited(&user_id)
+				.rooms_invited_state(&user_id)
 				.collect()
 				.await;
 			let query_time = timer.elapsed();
@@ -297,7 +301,7 @@ pub(super) async fn process(subcommand: RoomStateCacheCommand, context: &Context
 			let timer = tokio::time::Instant::now();
 			let results: Vec<_> = services
 				.state_cache
-				.rooms_left(&user_id)
+				.rooms_left_state(&user_id)
 				.collect()
 				.await;
 			let query_time = timer.elapsed();
@@ -316,6 +320,22 @@ pub(super) async fn process(subcommand: RoomStateCacheCommand, context: &Context
 				.await;
 			let query_time = timer.elapsed();
 
+			context
+				.write_str(&format!(
+					"Query completed in {query_time:?}:\n\n```rs\n{results:#?}\n```"
+				))
+				.await
+		},
+		| RoomStateCacheCommand::UserMemberships { user_id } => {
+			let timer = tokio::time::Instant::now();
+			let results = services
+				.state_cache
+				.all_user_memberships(&user_id)
+				.map(|(membership, room_id)| (membership, room_id.to_owned()))
+				.collect::<Vec<_>>()
+				.await;
+			let query_time = timer.elapsed();
+
 			context
 				.write_str(&format!(
 					"Query completed in {query_time:?}:\n\n```rs\n{results:#?}\n```"

src/admin/query/room_timeline.rs

@@ -25,7 +25,11 @@ pub(crate) enum RoomTimelineCommand {
 
 #[admin_command]
 pub(super) async fn last(&self, room_id: OwnedRoomOrAliasId) -> Result {
-	let room_id = self.services.alias.resolve(&room_id).await?;
+	let room_id = self
+		.services
+		.alias
+		.maybe_resolve(&room_id)
+		.await?;
 
 	let result = self
 		.services
@@ -43,7 +47,11 @@ pub(super) async fn pdus(
 	from: Option<String>,
 	limit: Option<usize>,
 ) -> Result {
-	let room_id = self.services.alias.resolve(&room_id).await?;
+	let room_id = self
+		.services
+		.alias
+		.maybe_resolve(&room_id)
+		.await?;
 
 	let from: Option<PduCount> = from.as_deref().map(str::parse).transpose()?;
 

src/admin/query/short.rs

@@ -30,7 +30,11 @@ pub(super) async fn short_event_id(&self, event_id: OwnedEventId) -> Result {
 
 #[admin_command]
 pub(super) async fn short_room_id(&self, room_id: OwnedRoomOrAliasId) -> Result {
-	let room_id = self.services.alias.resolve(&room_id).await?;
+	let room_id = self
+		.services
+		.alias
+		.maybe_resolve(&room_id)
+		.await?;
 
 	let shortid = self
 		.services

src/admin/query/sync.rs

@@ -0,0 +1,82 @@
+use clap::Subcommand;
+use ruma::{OwnedDeviceId, OwnedUserId};
+use tuwunel_core::Result;
+use tuwunel_service::sync::into_connection_key;
+
+use crate::{admin_command, admin_command_dispatch};
+
+#[admin_command_dispatch]
+#[derive(Debug, Subcommand)]
+/// Query sync service state
+pub(crate) enum SyncCommand {
+	/// List sliding-sync connections.
+	ListConnections,
+
+	/// Show details of sliding sync connection by ID.
+	ShowConnection {
+		user_id: OwnedUserId,
+		device_id: OwnedDeviceId,
+		conn_id: Option<String>,
+	},
+
+	/// Drop connections for a user, device, or all.
+	DropConnections {
+		user_id: Option<OwnedUserId>,
+		device_id: Option<OwnedDeviceId>,
+		conn_id: Option<String>,
+	},
+}
+
+#[admin_command]
+pub(super) async fn list_connections(&self) -> Result {
+	let connections = self.services.sync.list_loaded_connections().await;
+
+	for connection_key in connections {
+		self.write_str(&format!("{connection_key:?}\n"))
+			.await?;
+	}
+
+	Ok(())
+}
+
+#[admin_command]
+pub(super) async fn show_connection(
+	&self,
+	user_id: OwnedUserId,
+	device_id: OwnedDeviceId,
+	conn_id: Option<String>,
+) -> Result {
+	let key = into_connection_key(user_id, device_id, conn_id);
+	let cache = self
+		.services
+		.sync
+		.get_loaded_connection(&key)
+		.await?;
+
+	let out;
+	{
+		let cached = cache.lock().await;
+		out = format!("{cached:#?}");
+	};
+
+	self.write_str(out.as_str()).await
+}
+
+#[admin_command]
+pub(super) async fn drop_connections(
+	&self,
+	user_id: Option<OwnedUserId>,
+	device_id: Option<OwnedDeviceId>,
+	conn_id: Option<String>,
+) -> Result {
+	self.services
+		.sync
+		.clear_connections(
+			user_id.as_deref(),
+			device_id.as_deref(),
+			conn_id.map(Into::into).as_ref(),
+		)
+		.await;
+
+	Ok(())
+}

src/admin/room/moderation.rs

@@ -48,7 +48,7 @@ pub(crate) enum RoomModerationCommand {
 async fn ban_room(&self, room: OwnedRoomOrAliasId) -> Result {
 	debug!("Got room alias or ID: {}", room);
 
-	let admin_room_alias = &self.services.globals.admin_alias;
+	let admin_room_alias = &self.services.admin.admin_alias;
 
 	if let Ok(admin_room_id) = self.services.admin.get_admin_room().await {
 		if room.to_string().eq(&admin_room_id) || room.to_string().eq(admin_room_alias) {
@@ -105,7 +105,7 @@ async fn ban_room(&self, room: OwnedRoomOrAliasId) -> Result {
 				match self
 					.services
 					.alias
-					.resolve_alias(room_alias, None)
+					.resolve_alias(room_alias)
 					.await
 				{
 					| Ok((room_id, servers)) => {
@@ -209,7 +209,7 @@ async fn ban_list_of_rooms(&self) -> Result {
 		.drain(1..self.body.len().saturating_sub(1))
 		.collect::<Vec<_>>();
 
-	let admin_room_alias = &self.services.globals.admin_alias;
+	let admin_room_alias = &self.services.admin.admin_alias;
 
 	let mut room_ban_count: usize = 0;
 	let mut room_ids: Vec<OwnedRoomId> = Vec::new();
@@ -260,7 +260,7 @@ async fn ban_list_of_rooms(&self) -> Result {
 									match self
 										.services
 										.alias
-										.resolve_alias(room_alias, None)
+										.resolve_alias(room_alias)
 										.await
 									{
 										| Ok((room_id, servers)) => {
@@ -423,7 +423,7 @@ async fn unban_room(&self, room: OwnedRoomOrAliasId) -> Result {
 				match self
 					.services
 					.alias
-					.resolve_alias(room_alias, None)
+					.resolve_alias(room_alias)
 					.await
 				{
 					| Ok((room_id, servers)) => {

src/admin/server/commands.rs

@@ -67,7 +67,7 @@ pub(super) async fn list_features(&self, available: bool, enabled: bool, comma:
 #[admin_command]
 pub(super) async fn memory_usage(&self) -> Result {
 	let services_usage = self.services.memory_usage().await?;
-	let database_usage = self.services.db.db.memory_usage()?;
+	let database_usage = self.services.db.engine.memory_usage()?;
 	let allocator_usage = tuwunel_core::alloc::memory_usage()
 		.map_or(String::new(), |s| format!("\nAllocator:\n{s}"));
 
@@ -88,7 +88,7 @@ pub(super) async fn clear_caches(&self) -> Result {
 pub(super) async fn list_backups(&self) -> Result {
 	self.services
 		.db
-		.db
+		.engine
 		.backup_list()?
 		.try_stream()
 		.try_for_each(|result| write!(self, "{result}"))
@@ -102,13 +102,13 @@ pub(super) async fn backup_database(&self) -> Result {
 		.services
 		.server
 		.runtime()
-		.spawn_blocking(move || match db.db.backup() {
+		.spawn_blocking(move || match db.engine.backup() {
 			| Ok(()) => "Done".to_owned(),
 			| Err(e) => format!("Failed: {e}"),
 		})
 		.await?;
 
-	let count = self.services.db.db.backup_count()?;
+	let count = self.services.db.engine.backup_count()?;
 	self.write_str(&format!("{result}. Currently have {count} backups."))
 		.await
 }

src/admin/user/commands.rs

@@ -2,7 +2,7 @@
 
 use futures::{FutureExt, StreamExt};
 use ruma::{
-	Int, OwnedEventId, OwnedRoomId, OwnedRoomOrAliasId, OwnedUserId, UserId,
+	Int, OwnedDeviceId, OwnedEventId, OwnedRoomId, OwnedRoomOrAliasId, OwnedUserId, UserId,
 	events::{
 		RoomAccountDataEventType, StateEventType,
 		room::{
@@ -119,7 +119,7 @@ pub(super) async fn create_user(&self, username: String, password: Option<String
 		.is_empty()
 	{
 		for room in &self.services.server.config.auto_join_rooms {
-			let Ok(room_id) = self.services.alias.resolve(room).await else {
+			let Ok(room_id) = self.services.alias.maybe_resolve(room).await else {
 				error!(
 					%user_id,
 					"Failed to resolve room alias to room ID when attempting to auto join {room}, skipping"
@@ -224,6 +224,25 @@ pub(super) async fn deactivate(&self, no_leave_rooms: bool, user_id: String) ->
 		.await
 }
 
+#[admin_command]
+pub(super) async fn delete_device(
+	&self,
+	user_id: OwnedUserId,
+	device_id: OwnedDeviceId,
+) -> Result {
+	if !self.services.globals.user_is_local(&user_id) {
+		return Err!("Cannot delete device of remote user");
+	}
+
+	self.services
+		.users
+		.remove_device(&user_id, &device_id)
+		.await;
+
+	self.write_str(&format!("User {user_id}'s device {device_id} removed."))
+		.await
+}
+
 #[admin_command]
 pub(super) async fn reset_password(&self, username: String, password: Option<String>) -> Result {
 	let user_id = parse_local_user_id(self.services, &username)?;
@@ -408,7 +427,7 @@ pub(super) async fn force_join_list_of_local_users(
 	let (room_id, servers) = self
 		.services
 		.alias
-		.resolve_with_servers(&room_id, None)
+		.maybe_resolve_with_servers(&room_id, None)
 		.await?;
 
 	if !self
@@ -532,7 +551,7 @@ pub(super) async fn force_join_all_local_users(
 	let (room_id, servers) = self
 		.services
 		.alias
-		.resolve_with_servers(&room_id, None)
+		.maybe_resolve_with_servers(&room_id, None)
 		.await?;
 
 	if !self
@@ -617,7 +636,7 @@ pub(super) async fn force_join_room(
 	let (room_id, servers) = self
 		.services
 		.alias
-		.resolve_with_servers(&room_id, None)
+		.maybe_resolve_with_servers(&room_id, None)
 		.await?;
 
 	assert!(
@@ -645,7 +664,11 @@ pub(super) async fn force_leave_room(
 	room_id: OwnedRoomOrAliasId,
 ) -> Result {
 	let user_id = parse_local_user_id(self.services, &user_id)?;
-	let room_id = self.services.alias.resolve(&room_id).await?;
+	let room_id = self
+		.services
+		.alias
+		.maybe_resolve(&room_id)
+		.await?;
 
 	assert!(
 		self.services.globals.user_is_local(&user_id),
@@ -678,7 +701,11 @@ pub(super) async fn force_leave_room(
 #[admin_command]
 pub(super) async fn force_demote(&self, user_id: String, room_id: OwnedRoomOrAliasId) -> Result {
 	let user_id = parse_local_user_id(self.services, &user_id)?;
-	let room_id = self.services.alias.resolve(&room_id).await?;
+	let room_id = self
+		.services
+		.alias
+		.maybe_resolve(&room_id)
+		.await?;
 
 	assert!(
 		self.services.globals.user_is_local(&user_id),
@@ -744,7 +771,11 @@ pub(super) async fn force_promote(
 	room_id: OwnedRoomOrAliasId,
 ) -> Result {
 	let target_id = parse_user_id(self.services, &target_id)?;
-	let room_id = self.services.alias.resolve(&room_id).await?;
+	let room_id = self
+		.services
+		.alias
+		.maybe_resolve(&room_id)
+		.await?;
 
 	let state_lock = self.services.state.mutex.lock(&room_id).await;
 

src/admin/user/mod.rs

@@ -1,7 +1,7 @@
 mod commands;
 
 use clap::Subcommand;
-use ruma::{OwnedEventId, OwnedRoomId, OwnedRoomOrAliasId};
+use ruma::{OwnedDeviceId, OwnedEventId, OwnedRoomId, OwnedRoomOrAliasId, OwnedUserId};
 use tuwunel_core::Result;
 
 use crate::admin_command_dispatch;
@@ -59,6 +59,12 @@ pub(super) enum UserCommand {
 		force: bool,
 	},
 
+	/// - Deletes a user's device.
+	DeleteDevice {
+		user_id: OwnedUserId,
+		device_id: OwnedDeviceId,
+	},
+
 	/// - List local users in the database
 	#[clap(alias = "list")]
 	ListUsers,

src/api/Cargo.toml

@@ -11,6 +11,7 @@ version.workspace = true
 
 [lib]
 path = "mod.rs"
+bench = false
 crate-type = [
 	"rlib",
 #	"dylib",

src/api/client/account.rs

@@ -5,7 +5,7 @@
 	ThirdPartyIdRemovalStatus, change_password, deactivate, get_3pids,
 	request_3pid_management_token_via_email, request_3pid_management_token_via_msisdn, whoami,
 };
-use tuwunel_core::{Err, Result, info, utils::ReadyExt};
+use tuwunel_core::{Err, Result, err, info, utils::ReadyExt};
 
 use crate::{Ruma, router::auth_uiaa};
 
@@ -73,10 +73,12 @@ pub(crate) async fn whoami_route(
 	Ok(whoami::v3::Response {
 		user_id: body.sender_user().to_owned(),
 		device_id: body.sender_device.clone(),
-		is_guest: services
-			.users
-			.is_deactivated(body.sender_user())
-			.await? && body.appservice_info.is_none(),
+		is_guest: body.appservice_info.is_none()
+			&& services
+				.users
+				.is_deactivated(body.sender_user())
+				.await
+				.map_err(|_| err!(Request(Forbidden("User does not exist."))))?,
 	})
 }
 

src/api/client/alias.rs

@@ -5,7 +5,7 @@
 	OwnedServerName, RoomAliasId, RoomId,
 	api::client::alias::{create_alias, delete_alias, get_alias},
 };
-use tuwunel_core::{Err, Result, debug};
+use tuwunel_core::{Err, Result, debug, err};
 use tuwunel_service::Services;
 
 use crate::Ruma;
@@ -26,8 +26,8 @@ pub(crate) async fn create_alias_route(
 	// this isn't apart of alias_checks or delete alias route because we should
 	// allow removing forbidden room aliases
 	if services
-		.globals
-		.forbidden_alias_names()
+		.config
+		.forbidden_alias_names
 		.is_match(body.room_alias.alias())
 	{
 		return Err!(Request(Forbidden("Room alias is forbidden.")));
@@ -83,13 +83,11 @@ pub(crate) async fn get_alias_route(
 ) -> Result<get_alias::v3::Response> {
 	let room_alias = body.body.room_alias;
 
-	let Ok((room_id, servers)) = services
+	let (room_id, servers) = services
 		.alias
-		.resolve_alias(&room_alias, None)
+		.resolve_alias(&room_alias)
 		.await
-	else {
-		return Err!(Request(NotFound("Room with alias not found.")));
-	};
+		.map_err(|_| err!(Request(NotFound("Room with alias not found."))))?;
 
 	let servers = room_available_servers(&services, &room_id, &room_alias, servers).await;
 	debug!(?room_alias, ?room_id, "available servers: {servers:?}");

src/api/client/appservice.rs

@@ -37,13 +37,10 @@ pub(crate) async fn appservice_ping(
 	let timer = tokio::time::Instant::now();
 
 	let _response = services
-		.sending
-		.send_appservice_request(
-			appservice_info.registration.clone(),
-			ping::send_ping::v1::Request {
-				transaction_id: body.transaction_id.clone(),
-			},
-		)
+		.appservice
+		.send_request(appservice_info.registration.clone(), ping::send_ping::v1::Request {
+			transaction_id: body.transaction_id.clone(),
+		})
 		.await?
 		.expect("We already validated if an appservice URL exists above");
 

src/api/client/context.rs

@@ -134,7 +134,7 @@ pub(crate) async fn get_context_route(
 		.map_or_else(|| body.event_id.as_ref(), |pdu| pdu.event_id.as_ref());
 
 	let state_ids = services
-		.state_accessor
+		.state
 		.pdu_shortstatehash(state_at)
 		.or_else(|_| services.state.get_room_shortstatehash(room_id))
 		.map_ok(|shortstatehash| {

src/api/client/dehydrated_device.rs

@@ -0,0 +1,132 @@
+use axum::extract::State;
+use axum_client_ip::InsecureClientIp;
+use futures::StreamExt;
+use ruma::api::client::dehydrated_device::{
+	delete_dehydrated_device::unstable as delete_dehydrated_device,
+	get_dehydrated_device::unstable as get_dehydrated_device, get_events::unstable as get_events,
+	put_dehydrated_device::unstable as put_dehydrated_device,
+};
+use tuwunel_core::{Err, Result, at, utils::result::IsErrOr};
+
+use crate::Ruma;
+
+const MAX_BATCH_EVENTS: usize = 50;
+
+/// # `PUT /_matrix/client/../dehydrated_device`
+///
+/// Creates or overwrites the user's dehydrated device.
+#[tracing::instrument(skip_all, fields(%client))]
+pub(crate) async fn put_dehydrated_device_route(
+	State(services): State<crate::State>,
+	InsecureClientIp(client): InsecureClientIp,
+	body: Ruma<put_dehydrated_device::Request>,
+) -> Result<put_dehydrated_device::Response> {
+	let sender_user = body
+		.sender_user
+		.as_deref()
+		.expect("AccessToken authentication required");
+
+	let device_id = body.body.device_id.clone();
+
+	services
+		.users
+		.set_dehydrated_device(sender_user, body.body)
+		.await?;
+
+	Ok(put_dehydrated_device::Response { device_id })
+}
+
+/// # `DELETE /_matrix/client/../dehydrated_device`
+///
+/// Deletes the user's dehydrated device without replacement.
+#[tracing::instrument(skip_all, fields(%client))]
+pub(crate) async fn delete_dehydrated_device_route(
+	State(services): State<crate::State>,
+	InsecureClientIp(client): InsecureClientIp,
+	body: Ruma<delete_dehydrated_device::Request>,
+) -> Result<delete_dehydrated_device::Response> {
+	let sender_user = body.sender_user();
+
+	let device_id = services
+		.users
+		.get_dehydrated_device_id(sender_user)
+		.await?;
+
+	services
+		.users
+		.remove_device(sender_user, &device_id)
+		.await;
+
+	Ok(delete_dehydrated_device::Response { device_id })
+}
+
+/// # `GET /_matrix/client/../dehydrated_device`
+///
+/// Gets the user's dehydrated device
+#[tracing::instrument(skip_all, fields(%client))]
+pub(crate) async fn get_dehydrated_device_route(
+	State(services): State<crate::State>,
+	InsecureClientIp(client): InsecureClientIp,
+	body: Ruma<get_dehydrated_device::Request>,
+) -> Result<get_dehydrated_device::Response> {
+	let sender_user = body.sender_user();
+
+	let device = services
+		.users
+		.get_dehydrated_device(sender_user)
+		.await?;
+
+	Ok(get_dehydrated_device::Response {
+		device_id: device.device_id,
+		device_data: device.device_data,
+	})
+}
+
+/// # `GET /_matrix/client/../dehydrated_device/{device_id}/events`
+///
+/// Paginates the events of the dehydrated device.
+#[tracing::instrument(skip_all, fields(%client))]
+pub(crate) async fn get_dehydrated_events_route(
+	State(services): State<crate::State>,
+	InsecureClientIp(client): InsecureClientIp,
+	body: Ruma<get_events::Request>,
+) -> Result<get_events::Response> {
+	let sender_user = body.sender_user();
+
+	let device_id = &body.body.device_id;
+	let existing_id = services
+		.users
+		.get_dehydrated_device_id(sender_user)
+		.await;
+
+	if existing_id
+		.as_ref()
+		.is_err_or(|existing_id| existing_id != device_id)
+	{
+		return Err!(Request(Forbidden("Not the dehydrated device_id.")));
+	}
+
+	let since: Option<u64> = body
+		.body
+		.next_batch
+		.as_deref()
+		.map(str::parse)
+		.transpose()?;
+
+	let mut next_batch: Option<u64> = None;
+	let events = services
+		.users
+		.get_to_device_events(sender_user, device_id, since, None)
+		.take(MAX_BATCH_EVENTS)
+		.inspect(|&(count, _)| {
+			next_batch.replace(count);
+		})
+		.map(at!(1))
+		.collect()
+		.await;
+
+	Ok(get_events::Response {
+		events,
+		next_batch: next_batch.as_ref().map(ToString::to_string),
+	})
+}

src/api/client/device.rs

@@ -7,7 +7,7 @@
 		self, delete_device, delete_devices, get_device, get_devices, update_device,
 	},
 };
-use tuwunel_core::{Err, Result, debug, err, utils};
+use tuwunel_core::{Err, Result, debug, err, utils, utils::string::to_small_string};
 
 use crate::{Ruma, client::DEVICE_ID_LENGTH, router::auth_uiaa};
 
@@ -61,18 +61,21 @@ pub(crate) async fn update_device_route(
 		.await
 	{
 		| Ok(mut device) => {
+			let notify = device.display_name != body.display_name;
 			device.display_name.clone_from(&body.display_name);
+
 			device
 				.last_seen_ip
-				.clone_from(&Some(client.to_string()));
+				.clone_from(&Some(to_small_string(client)));
+
 			device
 				.last_seen_ts
 				.clone_from(&Some(MilliSecondsSinceUnixEpoch::now()));
 
+			assert_eq!(device.device_id, body.device_id, "device_id mismatch");
 			services
 				.users
-				.update_device_metadata(sender_user, &body.device_id, &device)
-				.await?;
+				.put_device_metadata(sender_user, notify, &device);
 
 			Ok(update_device::v3::Response {})
 		},
@@ -80,6 +83,7 @@ pub(crate) async fn update_device_route(
 			let Some(appservice) = appservice else {
 				return Err!(Request(NotFound("Device not found.")));
 			};
+
 			if !appservice.registration.device_management {
 				return Err!(Request(NotFound("Device not found.")));
 			}
@@ -97,7 +101,7 @@ pub(crate) async fn update_device_route(
 				.create_device(
 					sender_user,
 					&device_id,
-					(&appservice.registration.as_token, None),
+					(Some(&appservice.registration.as_token), None),
 					None,
 					None,
 					Some(client.to_string()),

src/api/client/directory.rs

@@ -220,8 +220,8 @@ pub(crate) async fn get_public_rooms_filtered_helper(
 		server.filter(|server_name| !services.globals.server_is_ours(server_name))
 	{
 		let response = services
-			.sending
-			.send_federation_request(
+			.federation
+			.execute(
 				other_server,
 				federation::directory::get_public_rooms_filtered::v1::Request {
 					limit,

src/api/client/events.rs

@@ -0,0 +1,116 @@
+use std::iter::once;
+
+use axum::extract::State;
+use futures::StreamExt;
+use ruma::api::client::peeking::listen_to_new_events::v3::{Request, Response};
+use tokio::time::{Duration, Instant, timeout_at};
+use tuwunel_core::{
+	Err, Event, Result, at,
+	matrix::PduCount,
+	utils::{
+		BoolExt,
+		result::FlatOk,
+		stream::{IterStream, ReadyExt},
+	},
+};
+
+use crate::Ruma;
+
+const EVENT_LIMIT: usize = 50;
+
+/// GET `/_matrix/client/v3/events`
+pub(crate) async fn events_route(
+	State(services): State<crate::State>,
+	body: Ruma<Request>,
+) -> Result<Response> {
+	let (sender_user, sender_device) = body.sender();
+
+	let from = body
+		.body
+		.from
+		.as_deref()
+		.map(str::parse)
+		.flat_ok()
+		.unwrap_or_default();
+
+	let timeout = body
+		.body
+		.timeout
+		.as_ref()
+		.map(Duration::as_millis)
+		.map(TryInto::try_into)
+		.flat_ok()
+		.unwrap_or(services.config.client_sync_timeout_default)
+		.max(services.config.client_sync_timeout_min)
+		.min(services.config.client_sync_timeout_max);
+
+	let Some(room_id) = body.room_id.as_deref() else {
+		//TODO: upgrade ruma
+		return Err!(Request(InvalidParam("Missing RoomId parameter.")));
+	};
+
+	if !services
+		.state_accessor
+		.user_can_see_state_events(sender_user, room_id)
+		.await
+	{
+		return Err!(Request(Forbidden("No room preview available.")));
+	}
+
+	let stop_at = Instant::now()
+		.checked_add(Duration::from_millis(timeout))
+		.expect("configuration must limit maximum timeout");
+
+	loop {
+		let watchers = services
+			.sync
+			.watch(sender_user, sender_device, once(room_id).stream());
+
+		let next_batch = services.globals.wait_pending().await?;
+
+		let events = services
+			.timeline
+			.pdus(Some(sender_user), room_id, Some(PduCount::Normal(from)))
+			.ready_filter_map(Result::ok)
+			.ready_take_while(|(count, _)| PduCount::Normal(next_batch).ge(count))
+			.take(EVENT_LIMIT)
+			.collect::<Vec<_>>()
+			.await;
+
+		if !events.is_empty() {
+			return Ok(Response {
+				start: events
+					.first()
+					.map(at!(0))
+					.as_ref()
+					.map(ToString::to_string),
+
+				end: events
+					.last()
+					.map(at!(0))
+					.as_ref()
+					.map(ToString::to_string),
+
+				chunk: events
+					.into_iter()
+					.map(at!(1))
+					.map(Event::into_format)
+					.collect(),
+			});
+		}
+
+		if timeout_at(stop_at, watchers).await.is_err() || services.server.is_stopping() {
+			return Ok(Response {
+				chunk: Default::default(),
+				start: body.body.from,
+				end: services
+					.server
+					.is_stopping()
+					.is_false()
+					.then_some(next_batch)
+					.as_ref()
+					.map(ToString::to_string),
+			});
+		}
+	}
+}

src/api/client/keys.rs

@@ -6,14 +6,10 @@
 	CanonicalJsonObject, CanonicalJsonValue, OneTimeKeyAlgorithm, OwnedDeviceId, OwnedUserId,
 	UserId,
 	api::{
-		client::{
-			error::ErrorKind,
-			keys::{
-				claim_keys, get_key_changes, get_keys, upload_keys,
-				upload_signatures::{self},
-				upload_signing_keys,
-			},
-			uiaa::{AuthFlow, AuthType, UiaaInfo},
+		client::keys::{
+			claim_keys, get_key_changes, get_keys, upload_keys,
+			upload_signatures::{self},
+			upload_signing_keys,
 		},
 		federation,
 	},
@@ -21,11 +17,12 @@
 	serde::Raw,
 };
 use serde_json::json;
-use tuwunel_core::{Err, Error, Result, debug, debug_warn, err, result::NotFound, utils};
+use tuwunel_core::{
+	Err, Result, debug, debug_error, debug_warn, err, result::NotFound, utils::json,
+};
 use tuwunel_service::{Services, users::parse_master_key};
 
-use super::SESSION_ID_LENGTH;
-use crate::Ruma;
+use crate::{Ruma, router::auth_uiaa};
 
 /// # `POST /_matrix/client/r0/keys/upload`
 ///
@@ -40,30 +37,16 @@ pub(crate) async fn upload_keys_route(
 ) -> Result<upload_keys::v3::Response> {
 	let (sender_user, sender_device) = body.sender();
 
-	for (key_id, one_time_key) in body
+	let one_time_keys = body
 		.one_time_keys
 		.iter()
 		.take(services.config.one_time_key_limit)
-	{
-		if one_time_key
-			.deserialize()
-			.inspect_err(|e| {
-				debug_warn!(
-					?key_id,
-					?one_time_key,
-					"Invalid one time key JSON submitted by client, skipping: {e}"
-				);
-			})
-			.is_err()
-		{
-			continue;
-		}
+		.map(|(id, val)| (id.as_ref(), val));
 
-		services
-			.users
-			.add_one_time_key(sender_user, sender_device, key_id, one_time_key)
-			.await?;
-	}
+	services
+		.users
+		.add_one_time_keys(sender_user, sender_device, one_time_keys)
+		.await?;
 
 	if let Some(device_keys) = &body.device_keys {
 		let deser_device_keys = device_keys.deserialize().map_err(|e| {
@@ -88,8 +71,11 @@ pub(crate) async fn upload_keys_route(
 			.users
 			.get_device_keys(sender_user, sender_device)
 			.await
+			.and_then(|keys| keys.deserialize().map_err(Into::into))
 		{
-			if existing_keys.json().get() == device_keys.json().get() {
+			// NOTE: also serves as a workaround for a nheko bug which omits cross-signing
+			// NOTE: signatures when re-uploading the same DeviceKeys.
+			if existing_keys.keys == deser_device_keys.keys {
 				debug!(
 					?sender_user,
 					?sender_device,
@@ -162,23 +148,17 @@ pub(crate) async fn upload_signing_keys_route(
 	State(services): State<crate::State>,
 	body: Ruma<upload_signing_keys::v3::Request>,
 ) -> Result<upload_signing_keys::v3::Response> {
-	let (sender_user, sender_device) = body.sender();
-
-	// UIAA
-	let mut uiaainfo = UiaaInfo {
-		flows: vec![AuthFlow { stages: vec![AuthType::Password] }],
-		..Default::default()
-	};
-
+	// Access token is required for this endpoint regardless of conditional UIAA so
+	// we'll always have a sender_user.
 	match check_for_new_keys(
 		services,
-		sender_user,
+		body.sender_user(),
 		body.self_signing_key.as_ref(),
 		body.user_signing_key.as_ref(),
 		body.master_key.as_ref(),
 	)
 	.await
-	.inspect_err(|e| debug!(?e))
+	.inspect_err(|e| debug_error!(?e))
 	{
 		| Ok(exists) => {
 			if let Some(result) = exists {
@@ -186,45 +166,25 @@ pub(crate) async fn upload_signing_keys_route(
 				// (lost connection for example)
 				return Ok(result);
 			}
-			debug!(
-				"Skipping UIA in accordance with MSC3967, the user didn't have any existing keys"
-			);
+
 			// Some of the keys weren't found, so we let them upload
+			debug!("Skipping UIA in accordance with MSC3967, user had no existing keys");
 		},
 		| _ => {
-			match &body.auth {
-				| Some(auth) => {
-					let (worked, uiaainfo) = services
-						.uiaa
-						.try_auth(sender_user, sender_device, auth, &uiaainfo)
-						.await?;
-
-					if !worked {
-						return Err(Error::Uiaa(uiaainfo));
-					}
-					// Success!
-				},
-				| _ => match body.json_body.as_ref() {
-					| Some(json) => {
-						uiaainfo.session = Some(utils::random_string(SESSION_ID_LENGTH));
-						services
-							.uiaa
-							.create(sender_user, sender_device, &uiaainfo, json);
-
-						return Err(Error::Uiaa(uiaainfo));
-					},
-					| _ => {
-						return Err(Error::BadRequest(ErrorKind::NotJson, "Not json."));
-					},
-				},
-			}
+			let authed_user = auth_uiaa(&services, &body).await?;
+			assert_eq!(
+				body.sender_user(),
+				authed_user,
+				"Expected UIAA of {0} and not {authed_user}",
+				body.sender_user(),
+			);
 		},
 	}
 
 	services
 		.users
 		.add_cross_signing_keys(
-			sender_user,
+			body.sender_user(),
 			&body.master_key,
 			&body.self_signing_key,
 			&body.user_signing_key,
@@ -250,6 +210,7 @@ async fn check_for_new_keys(
 			.users
 			.get_master_key(None, user_id, &|_| true)
 			.await;
+
 		if result.is_not_found() {
 			empty = true;
 		} else {
@@ -262,6 +223,7 @@ async fn check_for_new_keys(
 			}
 		}
 	}
+
 	if let Some(user_signing_key) = user_signing_key {
 		let key = services.users.get_user_signing_key(user_id).await;
 		if key.is_not_found() && !empty {
@@ -269,6 +231,7 @@ async fn check_for_new_keys(
 				"Tried to update an existing user signing key, UIA required"
 			)));
 		}
+
 		if !key.is_not_found() {
 			let existing_signing_key = key?.deserialize()?;
 			if existing_signing_key != user_signing_key.deserialize()? {
@@ -278,17 +241,20 @@ async fn check_for_new_keys(
 			}
 		}
 	}
+
 	if let Some(self_signing_key) = self_signing_key {
 		let key = services
 			.users
 			.get_self_signing_key(None, user_id, &|_| true)
 			.await;
+
 		if key.is_not_found() && !empty {
-			debug!(?key);
+			debug_error!(?key);
 			return Err!(Request(Forbidden(
 				"Tried to add a new signing key independently from the master key"
 			)));
 		}
+
 		if !key.is_not_found() {
 			let existing_signing_key = key?.deserialize()?;
 			if existing_signing_key != self_signing_key.deserialize()? {
@@ -298,6 +264,7 @@ async fn check_for_new_keys(
 			}
 		}
 	}
+
 	if empty {
 		return Ok(None);
 	}
@@ -381,12 +348,12 @@ pub(crate) async fn get_key_changes_route(
 	let from = body
 		.from
 		.parse()
-		.map_err(|_| Error::BadRequest(ErrorKind::InvalidParam, "Invalid `from`."))?;
+		.map_err(|_| err!(Request(InvalidParam("Invalid `from`."))))?;
 
 	let to = body
 		.to
 		.parse()
-		.map_err(|_| Error::BadRequest(ErrorKind::InvalidParam, "Invalid `to`."))?;
+		.map_err(|_| err!(Request(InvalidParam("Invalid `to`."))))?;
 
 	device_list_updates.extend(
 		services
@@ -536,10 +503,7 @@ pub(crate) async fn get_keys_helper<F>(
 			let request =
 				federation::keys::get_keys::v1::Request { device_keys: device_keys_input_fed };
 
-			let response = services
-				.sending
-				.send_federation_request(server, request)
-				.await;
+			let response = services.federation.execute(server, request).await;
 
 			(server, response)
 		})
@@ -561,16 +525,16 @@ pub(crate) async fn get_keys_helper<F>(
 							.signatures
 							.append(&mut our_master_key.signatures);
 					}
-					let json = serde_json::to_value(master_key).expect("to_value always works");
-					let raw = serde_json::from_value(json).expect("Raw::from_value always works");
+
+					// Dont notify. A notification would trigger another key request resulting in
+					// an endless loop.
+					let notify = false;
+					let raw = Some(json::to_raw(master_key)?);
 					services
 						.users
-						.add_cross_signing_keys(
-							&user, &raw, &None, &None,
-							false, /* Dont notify. A notification would trigger another key
-							       * request resulting in an endless loop */
-						)
+						.add_cross_signing_keys(&user, &raw, &None, &None, notify)
 						.await?;
+
 					if let Some(raw) = raw {
 						master_keys.insert(user.clone(), raw);
 					}
@@ -607,7 +571,7 @@ fn add_unsigned_device_display_name(
 			.or_insert_with(|| CanonicalJsonObject::default().into())
 		{
 			let display_name = if include_display_names {
-				CanonicalJsonValue::String(display_name)
+				CanonicalJsonValue::String(display_name.to_string())
 			} else {
 				CanonicalJsonValue::String(metadata.device_id.into())
 			};
@@ -664,8 +628,8 @@ pub(crate) async fn claim_keys_helper(
 			(
 				server,
 				services
-					.sending
-					.send_federation_request(server, federation::keys::claim_keys::v1::Request {
+					.federation
+					.execute(server, federation::keys::claim_keys::v1::Request {
 						one_time_keys: one_time_keys_input_fed,
 					})
 					.await,

src/api/client/membership/join.rs

@@ -5,7 +5,7 @@
 	RoomId, RoomOrAliasId,
 	api::client::membership::{join_room_by_id, join_room_by_id_or_alias},
 };
-use tuwunel_core::Result;
+use tuwunel_core::{Result, warn};
 
 use super::banned_room_check;
 use crate::{Ruma, client::membership::get_join_params};
@@ -36,7 +36,8 @@ pub(crate) async fn join_room_by_id_route(
 
 	let state_lock = services.state.mutex.lock(&room_id).await;
 
-	services
+	let mut errors = 0_usize;
+	while let Err(e) = services
 		.membership
 		.join(
 			sender_user,
@@ -47,7 +48,17 @@ pub(crate) async fn join_room_by_id_route(
 			&state_lock,
 		)
 		.boxed()
-		.await?;
+		.await
+	{
+		errors = errors.saturating_add(1);
+		if errors >= services.config.max_join_attempts_per_join_request {
+			warn!(
+				"Several servers failed. Giving up for this request. Try again for different \
+				 server selection."
+			);
+			return Err(e);
+		}
+	}
 
 	drop(state_lock);
 
@@ -80,7 +91,8 @@ pub(crate) async fn join_room_by_id_or_alias_route(
 
 	let state_lock = services.state.mutex.lock(&room_id).await;
 
-	services
+	let mut errors = 0_usize;
+	while let Err(e) = services
 		.membership
 		.join(
 			sender_user,
@@ -91,7 +103,17 @@ pub(crate) async fn join_room_by_id_or_alias_route(
 			&state_lock,
 		)
 		.boxed()
-		.await?;
+		.await
+	{
+		errors = errors.saturating_add(1);
+		if errors >= services.config.max_join_attempts_per_join_request {
+			warn!(
+				"Several servers failed. Giving up for this request. Try again for different \
+				 server selection."
+			);
+			return Err(e);
+		}
+	}
 
 	drop(state_lock);
 

src/api/client/membership/knock.rs

@@ -22,6 +22,7 @@
 use tuwunel_core::{
 	Err, Result, debug, debug_info, debug_warn, err, extract_variant, info,
 	matrix::{
+		PduCount,
 		event::{Event, gen_event_id},
 		pdu::{PduBuilder, PduEvent},
 	},
@@ -209,11 +210,11 @@ async fn knock_room_helper_local(
 	})?;
 
 	knock_event_stub.insert(
-		"origin".to_owned(),
+		"origin".into(),
 		CanonicalJsonValue::String(services.globals.server_name().as_str().to_owned()),
 	);
 	knock_event_stub.insert(
-		"origin_server_ts".to_owned(),
+		"origin_server_ts".into(),
 		CanonicalJsonValue::Integer(
 			utils::millis_since_unix_epoch()
 				.try_into()
@@ -221,7 +222,7 @@ async fn knock_room_helper_local(
 		),
 	);
 	knock_event_stub.insert(
-		"content".to_owned(),
+		"content".into(),
 		to_canonical_value(RoomMemberEventContent {
 			displayname: services.users.displayname(sender_user).await.ok(),
 			avatar_url: services.users.avatar_url(sender_user).await.ok(),
@@ -243,7 +244,7 @@ async fn knock_room_helper_local(
 
 	// Add event_id
 	knock_event_stub
-		.insert("event_id".to_owned(), CanonicalJsonValue::String(event_id.clone().into()));
+		.insert("event_id".into(), CanonicalJsonValue::String(event_id.clone().into()));
 
 	// It has enough fields to be called a proper event now
 	let knock_event = knock_event_stub;
@@ -259,8 +260,8 @@ async fn knock_room_helper_local(
 	};
 
 	let send_knock_response = services
-		.sending
-		.send_federation_request(&remote_server, send_knock_request)
+		.federation
+		.execute(&remote_server, send_knock_request)
 		.await?;
 
 	info!("send_knock finished");
@@ -276,6 +277,7 @@ async fn knock_room_helper_local(
 		.map_err(|e| err!(BadServerResponse("Invalid knock event PDU: {e:?}")))?;
 
 	info!("Updating membership locally to knock state with provided stripped state events");
+	let count = services.globals.next_count();
 	services
 		.state_cache
 		.update_membership(
@@ -294,6 +296,7 @@ async fn knock_room_helper_local(
 			),
 			None,
 			false,
+			PduCount::Normal(*count),
 		)
 		.await?;
 
@@ -343,11 +346,11 @@ async fn knock_room_helper_remote(
 		})?;
 
 	knock_event_stub.insert(
-		"origin".to_owned(),
+		"origin".into(),
 		CanonicalJsonValue::String(services.globals.server_name().as_str().to_owned()),
 	);
 	knock_event_stub.insert(
-		"origin_server_ts".to_owned(),
+		"origin_server_ts".into(),
 		CanonicalJsonValue::Integer(
 			utils::millis_since_unix_epoch()
 				.try_into()
@@ -355,7 +358,7 @@ async fn knock_room_helper_remote(
 		),
 	);
 	knock_event_stub.insert(
-		"content".to_owned(),
+		"content".into(),
 		to_canonical_value(RoomMemberEventContent {
 			displayname: services.users.displayname(sender_user).await.ok(),
 			avatar_url: services.users.avatar_url(sender_user).await.ok(),
@@ -377,7 +380,7 @@ async fn knock_room_helper_remote(
 
 	// Add event_id
 	knock_event_stub
-		.insert("event_id".to_owned(), CanonicalJsonValue::String(event_id.clone().into()));
+		.insert("event_id".into(), CanonicalJsonValue::String(event_id.clone().into()));
 
 	// It has enough fields to be called a proper event now
 	let knock_event = knock_event_stub;
@@ -393,8 +396,8 @@ async fn knock_room_helper_remote(
 	};
 
 	let send_knock_response = services
-		.sending
-		.send_federation_request(&remote_server, send_knock_request)
+		.federation
+		.execute(&remote_server, send_knock_request)
 		.await?;
 
 	info!("send_knock finished");
@@ -490,6 +493,7 @@ async fn knock_room_helper_remote(
 		.await?;
 
 	info!("Updating membership locally to knock state with provided stripped state events");
+	let count = services.globals.next_count();
 	services
 		.state_cache
 		.update_membership(
@@ -508,6 +512,7 @@ async fn knock_room_helper_remote(
 			),
 			None,
 			false,
+			PduCount::Normal(*count),
 		)
 		.await?;
 
@@ -551,18 +556,15 @@ async fn make_knock_request(
 		info!("Asking {remote_server} for make_knock ({make_knock_counter})");
 
 		let make_knock_response = services
-			.sending
-			.send_federation_request(
-				remote_server,
-				federation::membership::prepare_knock_event::v1::Request {
-					room_id: room_id.to_owned(),
-					user_id: sender_user.to_owned(),
-					ver: services
-						.server
-						.supported_room_versions()
-						.collect(),
-				},
-			)
+			.federation
+			.execute(remote_server, federation::membership::prepare_knock_event::v1::Request {
+				room_id: room_id.to_owned(),
+				user_id: sender_user.to_owned(),
+				ver: services
+					.server
+					.supported_room_versions()
+					.collect(),
+			})
 			.await;
 
 		trace!("make_knock response: {make_knock_response:?}");

src/api/client/membership/members.rs

@@ -2,7 +2,7 @@
 use futures::{FutureExt, StreamExt, pin_mut};
 use ruma::{
 	api::client::membership::{
-		get_member_events::{self, v3::MembershipEventFilter},
+		get_member_events::{self},
 		joined_members::{self, v3::RoomMember},
 	},
 	events::{
@@ -94,9 +94,7 @@ pub(crate) async fn joined_members_route(
 			.ready_filter_map(Result::ok)
 			.ready_filter(|((ty, _), _)| *ty == StateEventType::RoomMember)
 			.map(at!(1))
-			.ready_filter_map(|pdu| {
-				membership_filter(pdu, Some(&MembershipEventFilter::Join), None)
-			})
+			.ready_filter_map(|pdu| membership_filter(pdu, Some(&MembershipState::Join), None))
 			.ready_filter_map(|pdu| {
 				let content = pdu.get_content::<RoomMemberEventContent>().ok()?;
 				let sender = pdu.sender().to_owned();
@@ -115,22 +113,22 @@ pub(crate) async fn joined_members_route(
 
 fn membership_filter<Pdu: Event>(
 	pdu: Pdu,
-	for_membership: Option<&MembershipEventFilter>,
-	not_membership: Option<&MembershipEventFilter>,
+	for_membership: Option<&MembershipState>,
+	not_membership: Option<&MembershipState>,
 ) -> Option<impl Event> {
 	let membership_state_filter = match for_membership {
-		| Some(MembershipEventFilter::Ban) => MembershipState::Ban,
-		| Some(MembershipEventFilter::Invite) => MembershipState::Invite,
-		| Some(MembershipEventFilter::Knock) => MembershipState::Knock,
-		| Some(MembershipEventFilter::Leave) => MembershipState::Leave,
+		| Some(MembershipState::Ban) => MembershipState::Ban,
+		| Some(MembershipState::Invite) => MembershipState::Invite,
+		| Some(MembershipState::Knock) => MembershipState::Knock,
+		| Some(MembershipState::Leave) => MembershipState::Leave,
 		| Some(_) | None => MembershipState::Join,
 	};
 
 	let not_membership_state_filter = match not_membership {
-		| Some(MembershipEventFilter::Ban) => MembershipState::Ban,
-		| Some(MembershipEventFilter::Invite) => MembershipState::Invite,
-		| Some(MembershipEventFilter::Join) => MembershipState::Join,
-		| Some(MembershipEventFilter::Knock) => MembershipState::Knock,
+		| Some(MembershipState::Ban) => MembershipState::Ban,
+		| Some(MembershipState::Invite) => MembershipState::Invite,
+		| Some(MembershipState::Join) => MembershipState::Join,
+		| Some(MembershipState::Knock) => MembershipState::Knock,
 		| Some(_) | None => MembershipState::Leave,
 	};
 

src/api/client/membership/mod.rs

@@ -151,7 +151,7 @@ async fn get_join_params(
 	via: &[OwnedServerName],
 ) -> Result<(OwnedRoomId, Vec<OwnedServerName>)> {
 	// servers tried first, additional_servers shuffled then tried after
-	let (room_id, mut servers, mut additional_servers) =
+	let (room_id, mut primary_servers, mut additional_servers) =
 		match OwnedRoomId::try_from(room_id_or_alias.to_owned()) {
 			// if room id, shuffle via + room_id server_name ...
 			| Ok(room_id) => {
@@ -165,10 +165,7 @@ async fn get_join_params(
 			},
 			// ... if room alias, resolve and don't shuffle ...
 			| Err(room_alias) => {
-				let (room_id, servers) = services
-					.alias
-					.resolve_alias(&room_alias, Some(via.to_vec()))
-					.await?;
+				let (room_id, servers) = services.alias.resolve_alias(&room_alias).await?;
 
 				(room_id, servers, Vec::new())
 			},
@@ -197,13 +194,23 @@ async fn get_join_params(
 			.map(|user| user.server_name().to_owned()),
 	);
 
+	primary_servers.sort_unstable();
+	primary_servers.dedup();
+	shuffle(&mut primary_servers);
+
 	// shuffle additionals, append to base servers
 	additional_servers.sort_unstable();
 	additional_servers.dedup();
 	shuffle(&mut additional_servers);
-	servers.sort_unstable();
-	servers.dedup();
-	servers.append(&mut additional_servers);
+
+	let mut servers: Vec<_> = room_id_or_alias
+		.server_name()
+		.filter(|_| room_id_or_alias.is_room_alias_id())
+		.map(ToOwned::to_owned)
+		.into_iter()
+		.chain(primary_servers.into_iter())
+		.chain(additional_servers.into_iter())
+		.collect();
 
 	// sort deprioritized servers last
 	servers.sort_by(|a, b| {
@@ -212,6 +219,7 @@ async fn get_join_params(
 			.config
 			.deprioritize_joins_through_servers
 			.is_match(a.host());
+
 		let b_matches = services
 			.server
 			.config

src/api/client/message.rs

@@ -1,5 +1,9 @@
 use axum::extract::State;
-use futures::{FutureExt, StreamExt, TryFutureExt, future::OptionFuture, pin_mut};
+use futures::{
+	FutureExt, StreamExt, TryFutureExt,
+	future::{Either, OptionFuture},
+	pin_mut,
+};
 use ruma::{
 	RoomId, UserId,
 	api::{
@@ -105,17 +109,18 @@ pub(crate) async fn get_message_events_route(
 	}
 
 	let it = match body.dir {
-		| Direction::Forward => services
-			.timeline
-			.pdus(Some(sender_user), room_id, Some(from))
-			.ignore_err()
-			.boxed(),
-
-		| Direction::Backward => services
-			.timeline
-			.pdus_rev(Some(sender_user), room_id, Some(from))
-			.ignore_err()
-			.boxed(),
+		| Direction::Forward => Either::Left(
+			services
+				.timeline
+				.pdus(Some(sender_user), room_id, Some(from))
+				.ignore_err(),
+		),
+		| Direction::Backward => Either::Right(
+			services
+				.timeline
+				.pdus_rev(Some(sender_user), room_id, Some(from))
+				.ignore_err(),
+		),
 	};
 
 	let events: Vec<_> = it

src/api/client/mod.rs

@@ -5,8 +5,10 @@
 pub(super) mod backup;
 pub(super) mod capabilities;
 pub(super) mod context;
+pub(super) mod dehydrated_device;
 pub(super) mod device;
 pub(super) mod directory;
+pub(super) mod events;
 pub(super) mod filter;
 pub(super) mod keys;
 pub(super) mod media;
@@ -49,8 +51,10 @@
 pub(super) use backup::*;
 pub(super) use capabilities::*;
 pub(super) use context::*;
+pub(super) use dehydrated_device::*;
 pub(super) use device::*;
 pub(super) use directory::*;
+pub(super) use events::*;
 pub(super) use filter::*;
 pub(super) use keys::*;
 pub(super) use media::*;

src/api/client/profile.rs

@@ -46,13 +46,11 @@ pub(crate) async fn set_displayname_route(
 		.update_displayname(&body.user_id, body.displayname.clone(), &all_joined_rooms)
 		.await;
 
-	if services.config.allow_local_presence {
-		// Presence update
-		services
-			.presence
-			.ping_presence(&body.user_id, &PresenceState::Online)
-			.await?;
-	}
+	// Presence update
+	services
+		.presence
+		.maybe_ping_presence(&body.user_id, body.sender_device.as_deref(), &PresenceState::Online)
+		.await?;
 
 	Ok(set_display_name::v3::Response {})
 }
@@ -70,8 +68,8 @@ pub(crate) async fn get_displayname_route(
 	if !services.globals.user_is_local(&body.user_id) {
 		// Create and update our local copy of the user
 		if let Ok(response) = services
-			.sending
-			.send_federation_request(
+			.federation
+			.execute(
 				body.user_id.server_name(),
 				federation::query::get_profile_information::v1::Request {
 					user_id: body.user_id.clone(),
@@ -148,14 +146,12 @@ pub(crate) async fn set_avatar_url_route(
 		)
 		.await;
 
-	if services.config.allow_local_presence {
-		// Presence update
-		services
-			.presence
-			.ping_presence(&body.user_id, &PresenceState::Online)
-			.await
-			.ok();
-	}
+	// Presence update
+	services
+		.presence
+		.maybe_ping_presence(&body.user_id, body.sender_device.as_deref(), &PresenceState::Online)
+		.await
+		.ok();
 
 	Ok(set_avatar_url::v3::Response {})
 }
@@ -173,8 +169,8 @@ pub(crate) async fn get_avatar_url_route(
 	if !services.globals.user_is_local(&body.user_id) {
 		// Create and update our local copy of the user
 		if let Ok(response) = services
-			.sending
-			.send_federation_request(
+			.federation
+			.execute(
 				body.user_id.server_name(),
 				federation::query::get_profile_information::v1::Request {
 					user_id: body.user_id.clone(),
@@ -235,8 +231,8 @@ pub(crate) async fn get_profile_route(
 	if !services.globals.user_is_local(&body.user_id) {
 		// Create and update our local copy of the user
 		if let Ok(response) = services
-			.sending
-			.send_federation_request(
+			.federation
+			.execute(
 				body.user_id.server_name(),
 				federation::query::get_profile_information::v1::Request {
 					user_id: body.user_id.clone(),

src/api/client/push.rs

@@ -1,10 +1,11 @@
 use axum::extract::State;
+use futures::StreamExt;
 use ruma::{
-	CanonicalJsonObject, CanonicalJsonValue,
+	CanonicalJsonObject, CanonicalJsonValue, MilliSecondsSinceUnixEpoch,
 	api::client::{
 		error::ErrorKind,
 		push::{
-			delete_pushrule, get_pushers, get_pushrule, get_pushrule_actions,
+			delete_pushrule, get_notifications, get_pushers, get_pushrule, get_pushrule_actions,
 			get_pushrule_enabled, get_pushrules_all, get_pushrules_global_scope, set_pusher,
 			set_pushrule, set_pushrule_actions, set_pushrule_enabled,
 		},
@@ -14,15 +15,117 @@
 		push_rules::{PushRulesEvent, PushRulesEventContent},
 	},
 	push::{
-		InsertPushRuleError, PredefinedContentRuleId, PredefinedOverrideRuleId,
+		Action, InsertPushRuleError, PredefinedContentRuleId, PredefinedOverrideRuleId,
 		RemovePushRuleError, Ruleset,
 	},
 };
-use tuwunel_core::{Err, Error, Result, err};
+use tuwunel_core::{
+	Err, Error, Result, at, err,
+	matrix::{Event, PduId},
+	utils::{
+		stream::{ReadyExt, WidebandExt},
+		string::to_small_string,
+	},
+};
 use tuwunel_service::Services;
 
 use crate::Ruma;
 
+/// # `GET /_matrix/client/r0/notifications/`
+///
+/// Paginate through the list of events the user has been, or would have been
+/// notified about.
+pub(crate) async fn get_notifications_route(
+	State(services): State<crate::State>,
+	body: Ruma<get_notifications::v3::Request>,
+) -> Result<get_notifications::v3::Response> {
+	use get_notifications::v3::Notification;
+
+	let sender_user = body.sender_user();
+
+	let from = body
+		.body
+		.from
+		.as_deref()
+		.map(str::parse)
+		.transpose()
+		.map_err(|e| err!(Request(InvalidParam("Invalid `from' parameter: {e}"))))?;
+
+	let limit: usize = body
+		.body
+		.limit
+		.map(TryInto::try_into)
+		.transpose()?
+		.unwrap_or(50)
+		.clamp(1, 100);
+
+	let only_highlight = body
+		.body
+		.only
+		.as_deref()
+		.is_some_and(|only| only.contains("highlight"));
+
+	let mut next_token: Option<u64> = None;
+	let notifications = services
+		.pusher
+		.get_notifications(sender_user, from)
+		.ready_filter(|(_, notify)| {
+			if only_highlight && !notify.actions.iter().any(Action::is_highlight) {
+				return false;
+			}
+
+			true
+		})
+		.wide_filter_map(async |(count, notify)| {
+			let pdu_id = PduId {
+				shortroomid: notify.sroomid,
+				count: count.into(),
+			};
+
+			let event = services
+				.timeline
+				.get_pdu_from_id(&pdu_id.into())
+				.await
+				.ok()
+				.filter(|event| !event.is_redacted())?;
+
+			let read = services
+				.pusher
+				.last_notification_read(sender_user, event.room_id())
+				.await
+				.is_ok_and(|last_read| last_read.ge(&count));
+
+			let ts = notify
+				.ts
+				.try_into()
+				.map(MilliSecondsSinceUnixEpoch)
+				.ok()?;
+
+			let notification = Notification {
+				room_id: event.room_id().into(),
+				event: event.into_format(),
+				ts,
+				read,
+				profile_tag: notify.tag,
+				actions: notify.actions,
+			};
+
+			Some((count, notification))
+		})
+		.take(limit)
+		.inspect(|(count, _)| {
+			next_token.replace(*count);
+		})
+		.map(at!(1))
+		.collect::<Vec<_>>()
+		.await;
+
+	Ok(get_notifications::v3::Response {
+		next_token: next_token.map(to_small_string),
+		notifications,
+	})
+}
+
 /// # `GET /_matrix/client/r0/pushrules/`
 ///
 /// Retrieves the push rules event for this user.
@@ -335,7 +438,7 @@ pub(crate) async fn set_pushrule_actions_route(
 	if account_data
 		.content
 		.global
-		.set_actions(body.kind.clone(), &body.rule_id, body.actions.clone())
+		.set_actions(body.kind.clone(), &body.rule_id, body.actions.clone().into())
 		.is_err()
 	{
 		return Err!(Request(NotFound("Push rule not found.")));

src/api/client/read_marker.rs

@@ -26,6 +26,12 @@ pub(crate) async fn set_read_marker_route(
 ) -> Result<set_read_marker::v3::Response> {
 	let sender_user = body.sender_user();
 
+	if body.private_read_receipt.is_some() || body.read_receipt.is_some() {
+		services
+			.pusher
+			.reset_notification_counts(sender_user, &body.room_id);
+	}
+
 	if let Some(event) = &body.fully_read {
 		let fully_read_event = ruma::events::fully_read::FullyReadEvent {
 			content: ruma::events::fully_read::FullyReadEventContent { event_id: event.clone() },
@@ -39,23 +45,29 @@ pub(crate) async fn set_read_marker_route(
 				RoomAccountDataEventType::FullyRead,
 				&serde_json::to_value(fully_read_event)?,
 			)
-			.await?;
+			.await
+			.ok();
 	}
 
-	if body.private_read_receipt.is_some() || body.read_receipt.is_some() {
+	if let Some(event) = &body.private_read_receipt {
+		let count = services
+			.timeline
+			.get_pdu_count(event)
+			.await
+			.map_err(|_| err!(Request(NotFound("Event not found."))))?;
+
+		let PduCount::Normal(count) = count else {
+			return Err!(Request(InvalidParam(
+				"Event is a backfilled PDU and cannot be marked as read."
+			)));
+		};
+
 		services
-			.user
-			.reset_notification_counts(sender_user, &body.room_id);
+			.read_receipt
+			.private_read_set(&body.room_id, sender_user, count);
 	}
 
 	if let Some(event) = &body.read_receipt {
-		if services.config.allow_local_presence {
-			services
-				.presence
-				.ping_presence(sender_user, &ruma::presence::PresenceState::Online)
-				.await?;
-		}
-
 		let receipt_content = BTreeMap::from_iter([(
 			event.to_owned(),
 			BTreeMap::from_iter([(
@@ -78,24 +90,16 @@ pub(crate) async fn set_read_marker_route(
 				},
 			)
 			.await;
-	}
-
-	if let Some(event) = &body.private_read_receipt {
-		let count = services
-			.timeline
-			.get_pdu_count(event)
-			.await
-			.map_err(|_| err!(Request(NotFound("Event not found."))))?;
-
-		let PduCount::Normal(count) = count else {
-			return Err!(Request(InvalidParam(
-				"Event is a backfilled PDU and cannot be marked as read."
-			)));
-		};
 
 		services
-			.read_receipt
-			.private_read_set(&body.room_id, sender_user, count);
+			.presence
+			.maybe_ping_presence(
+				sender_user,
+				body.sender_device.as_deref(),
+				&ruma::presence::PresenceState::Online,
+			)
+			.await
+			.ok();
 	}
 
 	Ok(set_read_marker::v3::Response {})
@@ -115,7 +119,7 @@ pub(crate) async fn create_receipt_route(
 		create_receipt::v3::ReceiptType::Read | create_receipt::v3::ReceiptType::ReadPrivate
 	) {
 		services
-			.user
+			.pusher
 			.reset_notification_counts(sender_user, &body.room_id);
 	}
 
@@ -137,13 +141,6 @@ pub(crate) async fn create_receipt_route(
 				.await?;
 		},
 		| create_receipt::v3::ReceiptType::Read => {
-			if services.config.allow_local_presence {
-				services
-					.presence
-					.ping_presence(sender_user, &ruma::presence::PresenceState::Online)
-					.await?;
-			}
-
 			let receipt_content = BTreeMap::from_iter([(
 				body.event_id.clone(),
 				BTreeMap::from_iter([(
@@ -169,6 +166,16 @@ pub(crate) async fn create_receipt_route(
 					},
 				)
 				.await;
+
+			services
+				.presence
+				.maybe_ping_presence(
+					sender_user,
+					body.sender_device.as_deref(),
+					&ruma::presence::PresenceState::Online,
+				)
+				.await
+				.ok();
 		},
 		| create_receipt::v3::ReceiptType::ReadPrivate => {
 			let count = services

src/api/client/register.rs

@@ -16,7 +16,9 @@
 	events::GlobalAccountDataEventType,
 	push,
 };
-use tuwunel_core::{Err, Error, Result, debug_info, error, info, is_equal_to, utils, warn};
+use tuwunel_core::{
+	Err, Error, Result, debug_info, debug_warn, error, info, is_equal_to, utils, warn,
+};
 use tuwunel_service::users::device::generate_refresh_token;
 
 use super::{DEVICE_ID_LENGTH, SESSION_ID_LENGTH};
@@ -58,8 +60,8 @@ pub(crate) async fn get_register_available_route(
 		});
 
 	if services
-		.globals
-		.forbidden_usernames()
+		.config
+		.forbidden_usernames
 		.is_match(&body.username)
 	{
 		return Err!(Request(Forbidden("Username is forbidden")));
@@ -181,31 +183,33 @@ pub(crate) async fn register_route(
 		return Err!(Request(Forbidden("Registration has been disabled.")));
 	}
 
-	if is_guest
-		&& (!services.config.allow_guest_registration
-			|| (services.config.allow_registration
-				&& services.globals.registration_token.is_some()))
-	{
-		info!(
+	if is_guest && !services.config.allow_guest_registration {
+		let display_name = body
+			.initial_device_display_name
+			.as_deref()
+			.unwrap_or("");
+
+		debug_warn!(
 			"Guest registration disabled / registration enabled with token configured, \
-			 rejecting guest registration attempt, initial device name: \"{}\"",
-			body.initial_device_display_name
-				.as_deref()
-				.unwrap_or("")
+			 rejecting guest registration attempt, initial device name: \"{display_name}\""
 		);
+
 		return Err!(Request(GuestAccessForbidden("Guest registration is disabled.")));
 	}
 
 	// forbid guests from registering if there is not a real admin user yet. give
 	// generic user error.
 	if is_guest && services.users.count().await < 2 {
+		let display_name = body
+			.initial_device_display_name
+			.as_deref()
+			.unwrap_or("");
+
 		warn!(
 			"Guest account attempted to register before a real admin user has been registered, \
-			 rejecting registration. Guest's initial device name: \"{}\"",
-			body.initial_device_display_name
-				.as_deref()
-				.unwrap_or("")
+			 rejecting registration. Guest's initial device name: \"{display_name}\""
 		);
+
 		return Err!(Request(Forbidden("Registration is temporarily disabled.")));
 	}
 
@@ -228,8 +232,8 @@ pub(crate) async fn register_route(
 					});
 
 			if services
-				.globals
-				.forbidden_usernames()
+				.config
+				.forbidden_usernames
 				.is_match(username)
 				&& !emergency_mode_enabled
 			{
@@ -309,7 +313,7 @@ pub(crate) async fn register_route(
 
 	// UIAA
 	let mut uiaainfo;
-	let skip_auth = if services.globals.registration_token.is_some() {
+	let skip_auth = if services.globals.registration_token.is_some() && !is_guest {
 		// Registration token required
 		uiaainfo = UiaaInfo {
 			flows: vec![AuthFlow {
@@ -320,6 +324,7 @@ pub(crate) async fn register_route(
 			session: None,
 			auth_error: None,
 		};
+
 		body.appservice_info.is_some()
 	} else {
 		// No registration token necessary, but clients must still go through the flow
@@ -330,6 +335,7 @@ pub(crate) async fn register_route(
 			session: None,
 			auth_error: None,
 		};
+
 		body.appservice_info.is_some() || is_guest
 	};
 
@@ -384,8 +390,8 @@ pub(crate) async fn register_route(
 	// If `new_user_displayname_suffix` is set, registration will push whatever
 	// content is set to the user's display name with a space before it
 	if !services
-		.globals
-		.new_user_displayname_suffix()
+		.config
+		.new_user_displayname_suffix
 		.is_empty()
 		&& body.appservice_info.is_none()
 	{
@@ -446,7 +452,7 @@ pub(crate) async fn register_route(
 		.create_device(
 			&user_id,
 			&device_id,
-			(&access_token, expires_in),
+			(Some(&access_token), expires_in),
 			refresh_token.as_deref(),
 			body.initial_device_display_name.clone(),
 			Some(client.to_string()),
@@ -535,7 +541,7 @@ pub(crate) async fn register_route(
 		&& (services.config.allow_guests_auto_join_rooms || !is_guest)
 	{
 		for room in &services.server.config.auto_join_rooms {
-			let Ok(room_id) = services.alias.resolve(room).await else {
+			let Ok(room_id) = services.alias.maybe_resolve(room).await else {
 				error!(
 					"Failed to resolve room alias to room ID when attempting to auto join \
 					 {room}, skipping"

src/api/client/relations.rs

@@ -1,5 +1,11 @@
+use std::iter::once;
+
 use axum::extract::State;
-use futures::StreamExt;
+use futures::{
+	FutureExt, StreamExt, TryFutureExt,
+	future::try_join3,
+	stream::{select_all, unfold},
+};
 use ruma::{
 	EventId, RoomId, UInt, UserId,
 	api::{
@@ -12,12 +18,16 @@
 	events::{TimelineEventType, relation::RelationType},
 };
 use tuwunel_core::{
-	Result, at,
+	Err, Error, Result, at, err,
 	matrix::{
 		event::{Event, RelationTypeEqual},
-		pdu::PduCount,
+		pdu::{PduCount, PduId},
+	},
+	utils::{
+		BoolExt,
+		result::FlatOk,
+		stream::{ReadyExt, WidebandExt},
 	},
-	utils::{IterStream, ReadyExt, result::FlatOk, stream::WidebandExt},
 };
 use tuwunel_service::Services;
 
@@ -99,6 +109,13 @@ pub(crate) async fn get_relating_events_route(
 }
 
 #[allow(clippy::too_many_arguments)]
+#[tracing::instrument(
+	name = "relations",
+	level = "debug",
+	skip_all,
+	fields(room_id, target, from, to, dir, limit, recurse),
+	ret(level = "trace")
+)]
 async fn paginate_relations_with_filter(
 	services: &Services,
 	sender_user: &UserId,
@@ -112,78 +129,115 @@ async fn paginate_relations_with_filter(
 	recurse: bool,
 	dir: Direction,
 ) -> Result<get_relating_events::v1::Response> {
-	let start: PduCount = from
-		.map(str::parse)
-		.transpose()?
-		.unwrap_or_else(|| match dir {
-			| Direction::Forward => PduCount::min(),
-			| Direction::Backward => PduCount::max(),
-		});
+	let from: Option<PduCount> = from.map(str::parse).transpose()?;
 
 	let to: Option<PduCount> = to.map(str::parse).flat_ok();
 
-	// Use limit or else 30, with maximum 100
+	// Spec (v1.10) recommends depth of at least 3
+	let max_depth: usize = if recurse { 3 } else { 0 };
+
 	let limit: usize = limit
 		.map(TryInto::try_into)
 		.flat_ok()
 		.unwrap_or(30)
 		.min(100);
 
-	// Spec (v1.10) recommends depth of at least 3
-	let depth: u8 = if recurse { 3 } else { 1 };
+	let target = services
+		.timeline
+		.get_pdu_id(target)
+		.map_ok(PduId::from)
+		.map_ok(Ok::<_, Error>);
 
-	let events: Vec<_> = services
-		.pdu_metadata
-		.get_relations(sender_user, room_id, target, start, limit, depth, dir)
-		.await
-		.into_iter()
-		.filter(|(_, pdu)| {
-			filter_event_type
-				.as_ref()
-				.is_none_or(|kind| kind == pdu.kind())
-		})
-		.filter(|(_, pdu)| {
-			filter_rel_type
-				.as_ref()
-				.is_none_or(|rel_type| rel_type.relation_type_equal(pdu))
-		})
-		.stream()
-		.ready_take_while(|(count, _)| Some(*count) != to)
-		.wide_filter_map(|item| visibility_filter(services, sender_user, item))
-		.take(limit)
-		.collect()
-		.await;
+	let visible = services
+		.state_accessor
+		.user_can_see_state_events(sender_user, room_id)
+		.map(|visible| {
+			visible.ok_or_else(|| err!(Request(Forbidden("You cannot view this room."))))
+		});
 
-	let next_batch = match dir {
-		| Direction::Forward => events.last(),
-		| Direction::Backward => events.first(),
+	let shortroomid = services.short.get_shortroomid(room_id);
+
+	let (shortroomid, target, ()) = try_join3(shortroomid, target, visible).await?;
+
+	let Ok(target) = target else {
+		return Ok(get_relating_events::v1::Response::new(Vec::new()));
+	};
+
+	if shortroomid != target.shortroomid {
+		return Err!(Request(NotFound("Event not found in room.")));
 	}
-	.map(at!(0))
-	.as_ref()
-	.map(ToString::to_string);
+
+	if let PduCount::Backfilled(_) = target.count {
+		return Ok(get_relating_events::v1::Response::new(Vec::new()));
+	}
+
+	let fetch = |depth: usize, count: PduCount| {
+		services
+			.pdu_metadata
+			.get_relations(shortroomid, count, from, dir, Some(sender_user))
+			.map(move |(count, pdu)| (depth, count, pdu))
+			.ready_filter(|(_, count, _)| matches!(count, PduCount::Normal(_)))
+			.boxed()
+	};
+
+	let events = unfold(select_all(once(fetch(0, target.count))), async |mut relations| {
+		let (depth, count, pdu) = relations.next().await?;
+
+		if depth < max_depth {
+			relations.push(fetch(depth.saturating_add(1), count));
+		}
+
+		Some(((depth, count, pdu), relations))
+	})
+	.ready_take_while(|&(_, count, _)| Some(count) != to)
+	.ready_filter(|(_, _, pdu)| {
+		filter_event_type
+			.as_ref()
+			.is_none_or(|kind| kind == pdu.kind())
+	})
+	.ready_filter(|(_, _, pdu)| {
+		filter_rel_type
+			.as_ref()
+			.is_none_or(|rel_type| rel_type.relation_type_equal(pdu))
+	})
+	.wide_filter_map(async |(depth, count, pdu)| {
+		services
+			.state_accessor
+			.user_can_see_event(sender_user, pdu.room_id(), pdu.event_id())
+			.await
+			.then_some((depth, count, pdu))
+	})
+	.take(limit)
+	.collect::<Vec<_>>()
+	.await;
 
 	Ok(get_relating_events::v1::Response {
-		next_batch,
-		prev_batch: from.map(Into::into),
-		recursion_depth: recurse.then_some(depth.into()),
+		recursion_depth: max_depth
+			.gt(&0)
+			.then(|| events.iter().map(at!(0)))
+			.into_iter()
+			.flatten()
+			.max()
+			.map(TryInto::try_into)
+			.transpose()?,
+
+		next_batch: events
+			.last()
+			.map(at!(1))
+			.as_ref()
+			.map(ToString::to_string),
+
+		prev_batch: events
+			.first()
+			.map(at!(1))
+			.or(from)
+			.as_ref()
+			.map(ToString::to_string),
+
 		chunk: events
 			.into_iter()
-			.map(at!(1))
+			.map(at!(2))
 			.map(Event::into_format)
 			.collect(),
 	})
 }
-
-async fn visibility_filter<Pdu: Event>(
-	services: &Services,
-	sender_user: &UserId,
-	item: (PduCount, Pdu),
-) -> Option<(PduCount, Pdu)> {
-	let (_, pdu) = &item;
-
-	services
-		.state_accessor
-		.user_can_see_event(sender_user, pdu.room_id(), pdu.event_id())
-		.await
-		.then_some(item)
-}

src/api/client/room/create.rs

@@ -3,7 +3,8 @@
 use axum::extract::State;
 use futures::{FutureExt, future::OptionFuture};
 use ruma::{
-	CanonicalJsonObject, Int, OwnedRoomAliasId, OwnedRoomId, OwnedUserId, RoomId, RoomVersionId,
+	CanonicalJsonObject, EventEncryptionAlgorithm, Int, OwnedRoomAliasId, OwnedRoomId,
+	OwnedUserId, RoomId, RoomVersionId,
 	api::client::room::{
 		self, create_room,
 		create_room::v3::{CreationContent, RoomPreset},
@@ -13,6 +14,7 @@
 		room::{
 			canonical_alias::RoomCanonicalAliasEventContent,
 			create::RoomCreateEventContent,
+			encryption::RoomEncryptionEventContent,
 			guest_access::{GuestAccess, RoomGuestAccessEventContent},
 			history_visibility::{HistoryVisibility, RoomHistoryVisibilityEventContent},
 			join_rules::{JoinRule, RoomJoinRulesEventContent},
@@ -262,6 +264,7 @@ pub(crate) async fn create_room_route(
 		.await?;
 
 	// 6. Events listed in initial_state
+	let mut is_encrypted = false;
 	for event in &body.initial_state {
 		let mut pdu_builder = event
 			.deserialize_as_unchecked::<PduBuilder>()
@@ -292,6 +295,10 @@ pub(crate) async fn create_room_route(
 			continue;
 		}
 
+		if pdu_builder.event_type == TimelineEventType::RoomEncryption {
+			is_encrypted = true;
+		}
+
 		services
 			.timeline
 			.build_and_append_pdu(pdu_builder, sender_user, &room_id, &state_lock)
@@ -299,6 +306,33 @@ pub(crate) async fn create_room_route(
 			.await?;
 	}
 
+	if services.config.allow_encryption && !is_encrypted {
+		use RoomPreset::*;
+
+		let config = services
+			.config
+			.encryption_enabled_by_default_for_room_type
+			.as_deref()
+			.unwrap_or("off");
+
+		let invite = matches!(config, "invite");
+		let always = matches!(config, "all" | "invite");
+		if always || (invite && matches!(preset, PrivateChat | TrustedPrivateChat)) {
+			let algorithm = EventEncryptionAlgorithm::MegolmV1AesSha2;
+			let content = RoomEncryptionEventContent::new(algorithm);
+			services
+				.timeline
+				.build_and_append_pdu(
+					PduBuilder::state(String::new(), &content),
+					sender_user,
+					&room_id,
+					&state_lock,
+				)
+				.boxed()
+				.await?;
+		}
+	}
+
 	// 7. Events implied by name and topic
 	if let Some(name) = &body.name {
 		services
@@ -678,8 +712,8 @@ async fn room_alias_check(
 
 	// check if room alias is forbidden
 	if services
-		.globals
-		.forbidden_alias_names()
+		.config
+		.forbidden_alias_names
 		.is_match(room_alias_name)
 	{
 		return Err!(Request(Unknown("Room alias name is forbidden.")));
@@ -725,8 +759,8 @@ async fn room_alias_check(
 async fn custom_room_id_check(services: &Services, custom_room_id: &str) -> Result<OwnedRoomId> {
 	// apply forbidden room alias checks to custom room IDs too
 	if services
-		.globals
-		.forbidden_alias_names()
+		.config
+		.forbidden_alias_names
 		.is_match(custom_room_id)
 	{
 		return Err!(Request(Unknown("Custom room ID is forbidden.")));
@@ -798,7 +832,7 @@ async fn can_create_room_check(
 	services: &Services,
 	body: &Ruma<create_room::v3::Request>,
 ) -> Result {
-	if !services.globals.allow_room_creation()
+	if !services.config.allow_room_creation
 		&& body.appservice_info.is_none()
 		&& !services.users.is_admin(body.sender_user()).await
 	{

src/api/client/room/initial_sync.rs

@@ -1,15 +1,20 @@
 use axum::extract::State;
-use futures::{FutureExt, TryStreamExt, future::try_join4};
-use ruma::api::client::room::initial_sync::v3::{PaginationChunk, Request, Response};
+use futures::{FutureExt, StreamExt, TryFutureExt, TryStreamExt, future::try_join5};
+use ruma::{
+	api::client::room::initial_sync::v3::{PaginationChunk, Request, Response},
+	events::AnyRawAccountDataEvent,
+};
 use tuwunel_core::{
-	Err, Event, Result, at,
-	utils::{BoolExt, stream::TryTools},
+	Err, Event, Result, at, extract_variant,
+	matrix::PduCount,
+	utils::stream::{ReadyExt, TryTools},
 };
 
 use crate::Ruma;
 
-const LIMIT_MAX: usize = 100;
+const LIMIT_MAX: usize = 50;
 
+/// GET `/_matrix/client/v3/rooms/{roomId}/initialSync`
 pub(crate) async fn room_initial_sync_route(
 	State(services): State<crate::State>,
 	body: Ruma<Request>,
@@ -24,13 +29,15 @@ pub(crate) async fn room_initial_sync_route(
 		return Err!(Request(Forbidden("No room preview available.")));
 	}
 
+	let next_batch = services.globals.current_count();
+
+	let visibility = services.directory.visibility(room_id).map(Ok);
+
 	let membership = services
 		.state_cache
 		.user_membership(body.sender_user(), room_id)
 		.map(Ok);
 
-	let visibility = services.directory.visibility(room_id).map(Ok);
-
 	let state = services
 		.state_accessor
 		.room_state_full_pdus(room_id)
@@ -40,42 +47,52 @@ pub(crate) async fn room_initial_sync_route(
 	let limit = LIMIT_MAX;
 	let events = services
 		.timeline
-		.pdus_rev(None, room_id, None)
+		.pdus_rev(None, room_id, Some(PduCount::Normal(next_batch).saturating_add(1)))
 		.try_take(limit)
-		.try_collect::<Vec<_>>();
+		.try_collect()
+		.map_ok(|mut vec: Vec<_>| {
+			vec.reverse();
+			vec
+		});
 
-	let (membership, visibility, state, events) =
-		try_join4(membership, visibility, state, events)
+	let account_data = services
+		.account_data
+		.changes_since(Some(room_id), body.sender_user(), 0, Some(next_batch))
+		.ready_filter_map(|e| extract_variant!(e, AnyRawAccountDataEvent::Room))
+		.collect::<Vec<_>>()
+		.map(Ok);
+
+	let (membership, visibility, state, events, account_data) =
+		try_join5(membership, visibility, state, events, account_data)
 			.boxed()
 			.await?;
 
-	let messages = PaginationChunk {
-		start: events
-			.last()
-			.map(at!(0))
-			.as_ref()
-			.map(ToString::to_string),
-
-		end: events
-			.first()
-			.map(at!(0))
-			.as_ref()
-			.map(ToString::to_string)
-			.unwrap_or_default(),
-
-		chunk: events
-			.into_iter()
-			.map(at!(1))
-			.map(Event::into_format)
-			.collect(),
-	};
-
 	Ok(Response {
 		room_id: room_id.to_owned(),
-		account_data: None,
-		state: state.into(),
-		messages: messages.chunk.is_empty().or_some(messages),
-		visibility: visibility.into(),
 		membership,
+		visibility: visibility.into(),
+		account_data: Some(account_data),
+		state: state.into(),
+		messages: PaginationChunk {
+			start: events
+				.first()
+				.map(at!(0))
+				.as_ref()
+				.map(ToString::to_string),
+
+			end: events
+				.last()
+				.map(at!(0))
+				.as_ref()
+				.map(ToString::to_string)
+				.unwrap_or_default(),
+
+			chunk: events
+				.into_iter()
+				.map(at!(1))
+				.map(Event::into_format)
+				.collect(),
+		}
+		.into(),
 	})
 }

src/api/client/room/summary.rs

@@ -55,7 +55,7 @@ pub(crate) async fn get_room_summary(
 ) -> Result<get_summary::v1::Response> {
 	let (room_id, servers) = services
 		.alias
-		.resolve_with_servers(&body.room_id_or_alias, Some(body.via.clone()))
+		.maybe_resolve_with_servers(&body.room_id_or_alias, Some(body.via.clone()))
 		.await?;
 
 	if services.metadata.is_banned(&room_id).await {
@@ -232,8 +232,8 @@ async fn remote_room_summary_hierarchy_response(
 		.iter()
 		.map(|server| {
 			services
-				.sending
-				.send_federation_request(server, request.clone())
+				.federation
+				.execute(server, request.clone())
 		})
 		.collect();
 

src/api/client/room/upgrade.rs

@@ -1,42 +1,63 @@
 use std::cmp::max;
 
 use axum::extract::State;
-use futures::StreamExt;
+use futures::{FutureExt, StreamExt, TryFutureExt, TryStreamExt};
 use ruma::{
-	CanonicalJsonObject, RoomId, RoomVersionId,
-	api::client::room::upgrade_room,
+	CanonicalJsonObject, OwnedEventId, OwnedRoomId, OwnedUserId, RoomId, RoomVersionId, UserId,
+	api::client::room::upgrade_room::v3,
 	events::{
 		StateEventType, TimelineEventType,
 		room::{
+			create::PreviousRoom,
 			member::{MembershipState, RoomMemberEventContent},
 			power_levels::RoomPowerLevelsEventContent,
 			tombstone::RoomTombstoneEventContent,
 		},
 	},
 	int,
-	room_version_rules::RoomIdFormatVersion,
+	room_version_rules::{RoomIdFormatVersion, RoomVersionRules},
+};
+use serde_json::{
+	Value as JsonValue, json,
+	value::{to_raw_value, to_value},
 };
-use serde_json::{json, value::to_raw_value};
 use tuwunel_core::{
-	Err, Result, err,
+	Err, Result, debug_info, err, error, implement, info, is_equal_to, is_less_than,
 	matrix::{Event, StateKey, pdu::PduBuilder, room_version},
+	utils::{
+		future::TryExtExt,
+		stream::{IterStream, ReadyExt, WidebandExt},
+	},
 };
+use tuwunel_service::{Services, rooms::timeline::RoomMutexGuard};
 
 use crate::Ruma;
 
-/// Recommended transferable state events list from the spec
-const TRANSFERABLE_STATE_EVENTS: &[StateEventType; 9] = &[
-	StateEventType::RoomAvatar,
+//TODO: Upgrade Ruma
+const RECOMMENDED_TRANSFERABLE_STATE_EVENT_TYPES: &[StateEventType; 9] = &[
+	StateEventType::RoomServerAcl,
 	StateEventType::RoomEncryption,
+	StateEventType::RoomName,
+	StateEventType::RoomAvatar,
+	StateEventType::RoomTopic,
 	StateEventType::RoomGuestAccess,
 	StateEventType::RoomHistoryVisibility,
 	StateEventType::RoomJoinRules,
-	StateEventType::RoomName,
 	StateEventType::RoomPowerLevels,
-	StateEventType::RoomServerAcl,
-	StateEventType::RoomTopic,
 ];
 
+#[derive(Debug)]
+struct RoomUpgradeContext<'a> {
+	services: &'a Services,
+	sender_user: &'a UserId,
+	old_room_id: &'a RoomId,
+	old_state_lock: &'a RoomMutexGuard,
+	new_room_id: &'a RoomId,
+	new_state_lock: &'a RoomMutexGuard,
+	new_version_rules: &'a RoomVersionRules,
+	additional_creators: &'a [OwnedUserId],
+}
+
 /// # `POST /_matrix/client/r0/rooms/{roomId}/upgrade`
 ///
 /// Upgrades the room.
@@ -47,117 +68,227 @@
 /// - Transfers some state events
 /// - Moves local aliases
 /// - Modifies old room power levels to prevent users from speaking
+#[tracing::instrument(level = "debug")]
 pub(crate) async fn upgrade_room_route(
 	State(services): State<crate::State>,
-	body: Ruma<upgrade_room::v3::Request>,
-) -> Result<upgrade_room::v3::Response> {
-	debug_assert!(
-		TRANSFERABLE_STATE_EVENTS.is_sorted(),
-		"TRANSFERABLE_STATE_EVENTS is not sorted"
-	);
-
+	body: Ruma<v3::Request>,
+) -> Result<v3::Response> {
 	let sender_user = body.sender_user();
+	let new_version = &body.new_version;
+	let version_rules = room_version::rules(new_version)?;
 
 	if !services
 		.server
-		.supported_room_version(&body.new_version)
+		.supported_room_version(new_version)
 	{
 		return Err!(Request(UnsupportedRoomVersion(
 			"This server does not support that room version.",
 		)));
 	}
 
-	if matches!(body.new_version, RoomVersionId::V12) {
-		return Err!(Request(UnsupportedRoomVersion(
-			"Upgrading to version 12 is still under development.",
-		)));
+	let old_room_id = &body.room_id;
+	let old_state_lock = services.state.mutex.lock(old_room_id).await;
+
+	if !services
+		.state_accessor
+		.user_can_tombstone(old_room_id, sender_user, &old_state_lock)
+		.await
+	{
+		return Err!(Request(Forbidden("You are not permitted to upgrade the room.")));
 	}
 
-	let room_version_rules = room_version::rules(&body.new_version)?;
-	let room_id_format = &room_version_rules.room_id_format;
-	assert!(*room_id_format == RoomIdFormatVersion::V1, "TODO");
-
-	// Create a replacement room
-	let replacement_room = RoomId::new_v1(services.globals.server_name());
-
-	let _short_id = services
-		.short
-		.get_or_create_shortroomid(&replacement_room)
-		.await;
-
-	let state_lock = services.state.mutex.lock(&body.room_id).await;
-
-	// Send a m.room.tombstone event to the old room to indicate that it is not
-	// intended to be used any further Fail if the sender does not have the required
-	// permissions
-	let tombstone_event_id = services
+	let latest_event = services
 		.timeline
-		.build_and_append_pdu(
-			PduBuilder::state(StateKey::new(), &RoomTombstoneEventContent {
-				body: "This room has been replaced".to_owned(),
-				replacement_room: replacement_room.clone(),
-			}),
-			sender_user,
-			&body.room_id,
-			&state_lock,
-		)
-		.await?;
+		.latest_pdu_in_room(old_room_id)
+		.await
+		.ok();
 
-	// Change lock to replacement room
-	drop(state_lock);
-	let state_lock = services.state.mutex.lock(&replacement_room).await;
+	let predecessor = PreviousRoom {
+		room_id: old_room_id.to_owned(),
+		event_id: latest_event
+			.as_ref()
+			.map(Event::event_id)
+			.map(ToOwned::to_owned),
+	};
 
+	debug_info!(
+		%sender_user,
+		%old_room_id,
+		last_event = ?predecessor.event_id,
+		?new_version,
+		"Attempting upgrade of room..."
+	);
+
+	let id_format = version_rules.room_id_format;
+	let (replacement_room, state_lock) = match id_format {
+		| RoomIdFormatVersion::V2 =>
+			upgrade_room_create(
+				&services,
+				sender_user,
+				old_room_id,
+				new_version,
+				&version_rules,
+				predecessor,
+				body.additional_creators.clone(),
+			)
+			.await,
+
+		| RoomIdFormatVersion::V1 =>
+			upgrade_room_create_legacy(
+				&services,
+				sender_user,
+				old_room_id,
+				new_version,
+				&version_rules,
+				predecessor,
+			)
+			.await,
+	}
+	.inspect_err(|e| error!(?body, "Upgrade m.room.create event failed: {e}"))?;
+
+	let context = RoomUpgradeContext {
+		services: &services,
+		sender_user,
+		old_room_id: &body.room_id,
+		old_state_lock: &old_state_lock,
+		new_room_id: &replacement_room,
+		new_state_lock: &state_lock,
+		new_version_rules: &version_rules,
+		additional_creators: &body.additional_creators,
+	};
+
+	if let Err(e) = context.transfer_room().await {
+		error!(?e, ?context, "Room upgrade failed. Cleaning up incomplete room...");
+
+		if let Err(e) = services
+			.delete
+			.delete_room(&replacement_room, false, state_lock)
+			.await
+		{
+			error!("Additional errors while deleting incomplete room: {e}");
+		}
+
+		return Err(e);
+	}
+
+	info!(
+		old_room_id = %context.old_room_id,
+		new_room_id = %context.new_room_id,
+		upgraded_by = %sender_user,
+		"Room upgraded",
+	);
+
+	Ok(v3::Response { replacement_room })
+}
+
+#[tracing::instrument(level = "info")]
+async fn upgrade_room_create(
+	services: &Services,
+	sender_user: &UserId,
+	old_room_id: &RoomId,
+	new_version: &RoomVersionId,
+	version_rules: &RoomVersionRules,
+	predecessor: PreviousRoom,
+	mut additional_creators: Vec<OwnedUserId>,
+) -> Result<(OwnedRoomId, RoomMutexGuard)> {
 	// Get the old room creation event
-	let mut create_event_content: CanonicalJsonObject = services
+	let mut content: CanonicalJsonObject = services
 		.state_accessor
-		.room_state_get_content(&body.room_id, &StateEventType::RoomCreate, "")
+		.room_state_get_content(old_room_id, &StateEventType::RoomCreate, "")
 		.await
 		.map_err(|_| err!(Database("Found room without m.room.create event.")))?;
 
-	// Use the m.room.tombstone event as the predecessor
-	let predecessor = Some(ruma::events::room::create::PreviousRoom::new(
-		body.room_id.clone(),
-		Some(tombstone_event_id),
-	));
+	content.remove("creator");
+	content.insert("predecessor".into(), json!(predecessor).try_into()?);
+	content.insert("room_version".into(), json!(new_version).try_into()?);
 
-	// Send a m.room.create event containing a predecessor field and the applicable
-	// room_version
+	if version_rules
+		.authorization
+		.additional_room_creators
 	{
-		use RoomVersionId::*;
-		match body.new_version {
-			| V1 | V2 | V3 | V4 | V5 | V6 | V7 | V8 | V9 | V10 => {
-				create_event_content.insert(
-					"creator".into(),
-					json!(&sender_user).try_into().map_err(|e| {
-						err!(Request(BadJson(error!("Error forming creation event: {e}"))))
-					})?,
-				);
-			},
-			| _ => {
-				// "creator" key no longer exists in V11+ rooms
-				create_event_content.remove("creator");
-			},
+		additional_creators.sort();
+		additional_creators.dedup();
+		content.remove("additional_creators");
+		if !additional_creators.is_empty() {
+			content.insert("additional_creators".into(), json!(additional_creators).try_into()?);
 		}
 	}
 
-	create_event_content.insert(
-		"room_version".into(),
-		json!(&body.new_version)
-			.try_into()
-			.map_err(|_| err!(Request(BadJson("Error forming creation event"))))?,
-	);
-	create_event_content.insert(
-		"predecessor".into(),
-		json!(predecessor)
-			.try_into()
-			.map_err(|_| err!(Request(BadJson("Error forming creation event"))))?,
-	);
+	// Validate creation event content
+	let raw_content = to_raw_value(&content)?;
+	if let Err(e) = serde_json::from_str::<CanonicalJsonObject>(raw_content.get()) {
+		return Err!(Request(BadJson("Error forming creation event: {e}")));
+	}
+
+	let room_id = ruma::room_id!("!thiswillbereplaced").to_owned();
+	let state_lock = services.state.mutex.lock(&room_id).await;
+	let create_event_id = services
+		.timeline
+		.build_and_append_pdu(
+			PduBuilder {
+				event_type: TimelineEventType::RoomCreate,
+				content: to_raw_value(&content)?,
+				state_key: Some(StateKey::new()),
+				..Default::default()
+			},
+			sender_user,
+			&room_id,
+			&state_lock,
+		)
+		.boxed()
+		.await?;
+
+	drop(state_lock);
+
+	// The real room_id is now the event_id.
+	let room_id = OwnedRoomId::from_parts('!', create_event_id.localpart(), None)?;
+	let state_lock = services.state.mutex.lock(&room_id).await;
+
+	Ok((room_id, state_lock))
+}
+
+#[tracing::instrument(level = "info")]
+async fn upgrade_room_create_legacy(
+	services: &Services,
+	sender_user: &UserId,
+	old_room_id: &RoomId,
+	new_version: &RoomVersionId,
+	version_rules: &RoomVersionRules,
+	predecessor: PreviousRoom,
+) -> Result<(OwnedRoomId, RoomMutexGuard)> {
+	// Create a replacement room
+	let new_room_id = RoomId::new_v1(services.globals.server_name());
+	let state_lock = services.state.mutex.lock(&new_room_id).await;
+	let _short_id = services
+		.short
+		.get_or_create_shortroomid(&new_room_id)
+		.await;
+
+	// Get the old room creation event
+	let mut content: CanonicalJsonObject = services
+		.state_accessor
+		.room_state_get_content(old_room_id, &StateEventType::RoomCreate, "")
+		.await
+		.map_err(|_| err!(Database("Found room without m.room.create event.")))?;
+
+	// Send a m.room.create event containing a predecessor field and the applicable
+	// room_version. "creator" key no longer exists in V11+ rooms.
+	{
+		use RoomVersionId::*;
+		match new_version {
+			| V1 | V2 | V3 | V4 | V5 | V6 | V7 | V8 | V9 | V10 =>
+				content.insert("creator".into(), json!(&sender_user).try_into()?),
+			| _ => content.remove("creator"),
+		}
+	};
+
+	content.insert("predecessor".into(), json!(predecessor).try_into()?);
+	content.insert("room_version".into(), json!(new_version).try_into()?);
 
 	// Validate creation event content
-	if serde_json::from_str::<CanonicalJsonObject>(to_raw_value(&create_event_content)?.get())
-		.is_err()
-	{
-		return Err!(Request(BadJson("Error forming creation event")));
+	let raw_content = to_raw_value(&content)?;
+	if let Err(e) = serde_json::from_str::<CanonicalJsonObject>(raw_content.get()) {
+		return Err!(Request(BadJson("Error forming creation event: {e}")));
 	}
 
 	services
@@ -165,125 +296,229 @@ pub(crate) async fn upgrade_room_route(
 		.build_and_append_pdu(
 			PduBuilder {
 				event_type: TimelineEventType::RoomCreate,
-				content: to_raw_value(&create_event_content)?,
-				unsigned: None,
+				content: to_raw_value(&content)?,
 				state_key: Some(StateKey::new()),
-				redacts: None,
-				timestamp: None,
+				..Default::default()
 			},
 			sender_user,
-			&replacement_room,
+			&new_room_id,
 			&state_lock,
 		)
 		.await?;
 
-	// Join the new room
-	services
+	Ok((new_room_id, state_lock))
+}
+
+#[implement(RoomUpgradeContext, params = "<'_>")]
+#[tracing::instrument(level = "debug")]
+async fn transfer_room(&self) -> Result {
+	self.move_joined_member().await?;
+
+	self.move_state_events().await?;
+
+	self.move_local_aliases().await?;
+
+	self.tombstone_old_room().await?;
+
+	// After commitment to the tombstone above no more errors can propagate.
+	self.lockdown_old_room()
+		.await
+		.inspect_err(|e| error!(?self, "Failed to lockdown old room: {e}"))
+		.ok();
+
+	Ok(())
+}
+
+// Join the new room
+#[implement(RoomUpgradeContext, params = "<'_>")]
+#[tracing::instrument(level = "debug")]
+async fn move_joined_member(&self) -> Result<OwnedEventId> {
+	let old_content: RoomMemberEventContent = self
+		.services
+		.state_accessor
+		.room_state_get_content(
+			self.old_room_id,
+			&StateEventType::RoomMember,
+			self.sender_user.as_str(),
+		)
+		.inspect_err(|e| error!(?self, "Missing room member event: {e}"))
+		.await?;
+
+	self.services
 		.timeline
 		.build_and_append_pdu(
-			PduBuilder {
-				event_type: TimelineEventType::RoomMember,
-				content: to_raw_value(&RoomMemberEventContent {
-					membership: MembershipState::Join,
-					displayname: services.users.displayname(sender_user).await.ok(),
-					avatar_url: services.users.avatar_url(sender_user).await.ok(),
-					is_direct: None,
-					third_party_invite: None,
-					blurhash: services.users.blurhash(sender_user).await.ok(),
-					reason: None,
-					join_authorized_via_users_server: None,
-				})?,
-				unsigned: None,
-				state_key: Some(sender_user.as_str().into()),
-				redacts: None,
-				timestamp: None,
-			},
-			sender_user,
-			&replacement_room,
-			&state_lock,
+			PduBuilder::state(self.sender_user.as_str(), &RoomMemberEventContent {
+				membership: MembershipState::Join,
+				..old_content
+			}),
+			self.sender_user,
+			self.new_room_id,
+			self.new_state_lock,
 		)
-		.await?;
+		.await
+}
 
-	// Replicate transferable state events to the new room
-	for event_type in TRANSFERABLE_STATE_EVENTS {
-		let event_content = match services
-			.state_accessor
-			.room_state_get(&body.room_id, event_type, "")
-			.await
+// Replicate transferable state events to the new room
+#[implement(RoomUpgradeContext, params = "<'_>")]
+#[tracing::instrument(level = "debug")]
+async fn move_state_events(&self) -> Result {
+	RECOMMENDED_TRANSFERABLE_STATE_EVENT_TYPES
+		.iter()
+		.rev()
+		.stream()
+		.wide_filter_map(|event_type| {
+			self.services
+				.state_accessor
+				.room_state_get(self.old_room_id, event_type, "")
+				.ok()
+		})
+		.map(Ok)
+		.try_for_each(async |event| {
+			self.services
+				.timeline
+				.build_and_append_pdu(
+					self.rebuild_state_event(&event)?,
+					self.sender_user,
+					self.new_room_id,
+					self.new_state_lock,
+				)
+				.inspect_err(|e| {
+					error!(?event, ?self, "Failed to transfer state on upgrade: {e}");
+				})
+				.map_ok(|_| ())
+				.await
+		})
+		.await
+}
+
+#[implement(RoomUpgradeContext, params = "<'_>")]
+#[tracing::instrument(level = "debug")]
+fn rebuild_state_event<Pdu: Event>(&self, event: &Pdu) -> Result<PduBuilder> {
+	let content = match event.kind() {
+		| TimelineEventType::RoomPowerLevels
+			if self
+				.new_version_rules
+				.authorization
+				.explicitly_privilege_room_creators =>
 		{
-			| Ok(v) => v.content().to_owned(),
-			| Err(_) => continue, // Skipping missing events.
-		};
+			let mut content = event.get_content_as_value();
 
-		services
-			.timeline
-			.build_and_append_pdu(
-				PduBuilder {
-					event_type: event_type.to_string().into(),
-					content: event_content,
-					state_key: Some(StateKey::new()),
-					..Default::default()
-				},
-				sender_user,
-				&replacement_room,
-				&state_lock,
-			)
-			.await?;
-	}
+			if let Some(users) = content
+				.get_mut("users")
+				.and_then(JsonValue::as_object_mut)
+			{
+				users.retain(|user_id, _pl| {
+					!self
+						.additional_creators
+						.iter()
+						.map(AsRef::as_ref)
+						.map(UserId::as_str)
+						.any(is_equal_to!(user_id.as_str()))
+						&& self.sender_user.as_str() != user_id.as_str()
+				});
+			}
 
-	// Moves any local aliases to the new room
-	let mut local_aliases = services
+			if content["events"]["m.room.tombstone"]
+				.as_i64()
+				.is_none_or(is_less_than!(150))
+			{
+				content["events"]["m.room.tombstone"] = to_value(150)?;
+			}
+
+			to_raw_value(&content)?
+		},
+		| _ => to_raw_value(event.content())?,
+	};
+
+	Ok(PduBuilder {
+		content,
+		event_type: event.kind().clone(),
+		state_key: event.state_key().map(Into::into),
+		..Default::default()
+	})
+}
+
+// Moves any local aliases to the new room
+#[implement(RoomUpgradeContext, params = "<'_>")]
+#[tracing::instrument(level = "debug")]
+async fn move_local_aliases(&self) -> Result {
+	self.services
 		.alias
-		.local_aliases_for_room(&body.room_id)
-		.boxed();
+		.local_aliases_for_room(self.old_room_id)
+		.filter_map(|alias| {
+			self.services
+				.alias
+				.remove_alias(alias, self.sender_user)
+				.inspect_err(move |e| error!(?alias, ?self, "Failed to remove alias: {e}"))
+				.map_ok(move |()| alias)
+				.ok()
+		})
+		.ready_for_each(|alias| {
+			self.services
+				.alias
+				.set_alias(alias, self.new_room_id, self.sender_user)
+				.inspect_err(|e| error!(?self, "Failed to add alias: {e}"))
+				.ok();
+		})
+		.map(Ok)
+		.await
+}
 
-	while let Some(alias) = local_aliases.next().await {
-		services
-			.alias
-			.remove_alias(alias, sender_user)
-			.await?;
-
-		services
-			.alias
-			.set_alias(alias, &replacement_room, sender_user)?;
-	}
+// Send a m.room.tombstone event to the old room to indicate that it is not
+// intended to be used any further Fail if the sender does not have the required
+// permissions.
+#[implement(RoomUpgradeContext, params = "<'_>")]
+#[tracing::instrument(level = "debug")]
+async fn tombstone_old_room(&self) -> Result<OwnedEventId> {
+	self.services
+		.timeline
+		.build_and_append_pdu(
+			PduBuilder::state(StateKey::new(), &RoomTombstoneEventContent {
+				body: "This room has been upgraded.".to_owned(),
+				replacement_room: self.new_room_id.to_owned(),
+			}),
+			self.sender_user,
+			self.old_room_id,
+			self.old_state_lock,
+		)
+		.await
+}
 
+// Modify the power levels in the old room to prevent sending of events and
+// inviting new users. Though a Result is returned, the callsite above treats it
+// as infallible because the tombstone represents the commitment.
+#[implement(RoomUpgradeContext, params = "<'_>")]
+#[tracing::instrument(level = "debug")]
+async fn lockdown_old_room(&self) -> Result<OwnedEventId> {
 	// Get the old room power levels
-	let power_levels_event_content: RoomPowerLevelsEventContent = services
+	let old_content: RoomPowerLevelsEventContent = self
+		.services
 		.state_accessor
-		.room_state_get_content(&body.room_id, &StateEventType::RoomPowerLevels, "")
+		.room_state_get_content(self.old_room_id, &StateEventType::RoomPowerLevels, "")
 		.await
 		.map_err(|_| err!(Database("Found room without m.room.power_levels event.")))?;
 
-	// Setting events_default and invite to the greater of 50 and users_default + 1
-	let new_level = max(
-		int!(50),
-		power_levels_event_content
-			.users_default
-			.checked_add(int!(1))
-			.ok_or_else(|| {
-				err!(Request(BadJson("users_default power levels event content is not valid")))
-			})?,
-	);
+	let old_users_default = old_content
+		.users_default
+		.checked_add(int!(1))
+		.ok_or_else(|| {
+			err!(Request(BadJson("users_default power levels event content is not valid")))
+		})?;
 
-	// Modify the power levels in the old room to prevent sending of events and
-	// inviting new users
-	services
+	// Setting events_default and invite to the greater of 50 and users_default + 1
+	let new_level = max(int!(50), old_users_default);
+
+	self.services
 		.timeline
 		.build_and_append_pdu(
 			PduBuilder::state(StateKey::new(), &RoomPowerLevelsEventContent {
 				events_default: new_level,
 				invite: new_level,
-				..power_levels_event_content
+				..old_content
 			}),
-			sender_user,
-			&body.room_id,
-			&state_lock,
+			self.sender_user,
+			self.old_room_id,
+			self.old_state_lock,
 		)
-		.await?;
-
-	drop(state_lock);
-
-	// Return the replacement room id
-	Ok(upgrade_room::v3::Response { replacement_room })
+		.await
 }

src/api/client/session/mod.rs

@@ -116,7 +116,7 @@ pub(crate) async fn login_route(
 			.create_device(
 				&user_id,
 				&device_id,
-				(&access_token, expires_in),
+				(Some(&access_token), expires_in),
 				refresh_token.as_deref(),
 				body.initial_device_display_name.clone(),
 				Some(client.to_string()),

src/api/client/space.rs

@@ -44,7 +44,7 @@ pub(crate) async fn get_hierarchy_route(
 		.as_ref()
 		.and_then(|s| PaginationToken::from_str(s).ok());
 
-	// Should prevent unexpeded behaviour in (bad) clients
+	// Should prevent unexpected behaviour in (bad) clients
 	if let Some(ref token) = key {
 		if token.suggested_only != body.suggested_only || token.max_depth != max_depth {
 			return Err!(Request(InvalidParam(

src/api/client/state.rs

@@ -195,6 +195,7 @@ async fn send_state_event_for_key_helper(
 			room_id,
 			&state_lock,
 		)
+		.boxed()
 		.await?;
 
 	Ok(event_id)
@@ -328,7 +329,7 @@ async fn allowed_to_send_state_event(
 					for alias in aliases {
 						let (alias_room_id, _servers) = services
 							.alias
-							.resolve_alias(&alias, None)
+							.resolve_alias(&alias)
 							.await
 							.map_err(|e| {
 								err!(Request(BadAlias("Failed resolving alias \"{alias}\": {e}")))

src/api/client/sync/mod.rs

@@ -1,13 +1,8 @@
 mod v3;
 mod v5;
 
-use futures::{StreamExt, pin_mut};
-use ruma::{
-	RoomId, UserId,
-	events::TimelineEventType::{
-		self, Beacon, CallInvite, PollStart, RoomEncrypted, RoomMessage, Sticker,
-	},
-};
+use futures::{FutureExt, StreamExt, pin_mut};
+use ruma::{RoomId, UserId};
 use tuwunel_core::{
 	Error, PduCount, Result,
 	matrix::pdu::PduEvent,
@@ -17,9 +12,6 @@
 
 pub(crate) use self::{v3::sync_events_route, v5::sync_events_v5_route};
 
-pub(crate) const DEFAULT_BUMP_TYPES: &[TimelineEventType; 6] =
-	&[CallInvite, PollStart, Beacon, RoomEncrypted, RoomMessage, Sticker];
-
 async fn load_timeline(
 	services: &Services,
 	sender_user: &UserId,
@@ -50,10 +42,12 @@ async fn load_timeline(
 		.by_ref()
 		.take(limit)
 		.collect()
+		.map(|mut pdus: Vec<_>| {
+			pdus.reverse();
+			pdus
+		})
 		.await;
 
-	let timeline_pdus: Vec<_> = timeline_pdus.into_iter().rev().collect();
-
 	// They /sync response doesn't always return all messages, so we say the output
 	// is limited unless there are events in non_timeline_pdus
 	let limited = non_timeline_pdus.next().await.is_some();

src/api/client/sync/v5/extensions.rs

@@ -0,0 +1,150 @@
+mod account_data;
+mod e2ee;
+mod receipts;
+mod to_device;
+mod typing;
+
+use std::fmt::Debug;
+
+use futures::{
+	FutureExt,
+	future::{OptionFuture, join5},
+};
+use ruma::{
+	RoomId,
+	api::client::sync::sync_events::v5::{ListId, request::ExtensionRoomConfig, response},
+};
+use tuwunel_core::{Result, apply, at, extract_variant, utils::BoolExt};
+use tuwunel_service::sync::Connection;
+
+use super::{SyncInfo, Window, share_encrypted_room};
+
+#[tracing::instrument(
+	name = "extensions",
+	level = "debug",
+	skip_all,
+	fields(
+		next_batch = conn.next_batch,
+		window = window.len(),
+		rooms = conn.rooms.len(),
+		subs = conn.subscriptions.len(),
+	)
+)]
+pub(super) async fn handle(
+	sync_info: SyncInfo<'_>,
+	conn: &Connection,
+	window: &Window,
+) -> Result<response::Extensions> {
+	let SyncInfo { .. } = sync_info;
+
+	let account_data: OptionFuture<_> = conn
+		.extensions
+		.account_data
+		.enabled
+		.unwrap_or(false)
+		.then(|| account_data::collect(sync_info, conn, window))
+		.into();
+
+	let receipts: OptionFuture<_> = conn
+		.extensions
+		.receipts
+		.enabled
+		.unwrap_or(false)
+		.then(|| receipts::collect(sync_info, conn, window))
+		.into();
+
+	let typing: OptionFuture<_> = conn
+		.extensions
+		.typing
+		.enabled
+		.unwrap_or(false)
+		.then(|| typing::collect(sync_info, conn, window))
+		.into();
+
+	let to_device: OptionFuture<_> = conn
+		.extensions
+		.to_device
+		.enabled
+		.unwrap_or(false)
+		.then(|| to_device::collect(sync_info, conn))
+		.into();
+
+	let e2ee: OptionFuture<_> = conn
+		.extensions
+		.e2ee
+		.enabled
+		.unwrap_or(false)
+		.then(|| e2ee::collect(sync_info, conn))
+		.into();
+
+	let (account_data, receipts, typing, to_device, e2ee) =
+		join5(account_data, receipts, typing, to_device, e2ee)
+			.map(apply!(5, |t: Option<_>| t.unwrap_or(Ok(Default::default()))))
+			.await;
+
+	Ok(response::Extensions {
+		account_data: account_data?,
+		receipts: receipts?,
+		typing: typing?,
+		to_device: to_device?,
+		e2ee: e2ee?,
+	})
+}
+
+#[tracing::instrument(
+	name = "selector",
+	level = "trace",
+	skip_all,
+	fields(?implicit, ?explicit),
+)]
+fn selector<'a, ListIter, SubsIter>(
+	SyncInfo { .. }: SyncInfo<'a>,
+	conn: &'a Connection,
+	window: &'a Window,
+	implicit: Option<ListIter>,
+	explicit: Option<SubsIter>,
+) -> impl Iterator<Item = &'a RoomId> + Send + Sync + 'a
+where
+	ListIter: Iterator<Item = &'a ListId> + Clone + Debug + Send + Sync + 'a,
+	SubsIter: Iterator<Item = &'a ExtensionRoomConfig> + Clone + Debug + Send + Sync + 'a,
+{
+	let has_all_subscribed = explicit
+		.clone()
+		.into_iter()
+		.flatten()
+		.any(|erc| matches!(erc, ExtensionRoomConfig::AllSubscribed));
+
+	let all_subscribed = has_all_subscribed
+		.then(|| conn.subscriptions.keys())
+		.into_iter()
+		.flatten()
+		.map(AsRef::as_ref);
+
+	let rooms_explicit = has_all_subscribed
+		.is_false()
+		.then(move || {
+			explicit
+				.into_iter()
+				.flatten()
+				.filter_map(|erc| extract_variant!(erc, ExtensionRoomConfig::Room))
+				.map(AsRef::as_ref)
+		})
+		.into_iter()
+		.flatten();
+
+	let rooms_selected = window
+		.iter()
+		.filter(move |(_, room)| {
+			implicit.as_ref().is_none_or(|lists| {
+				lists
+					.clone()
+					.any(|list| room.lists.contains(list))
+			})
+		})
+		.map(at!(0))
+		.map(AsRef::as_ref);
+
+	all_subscribed
+		.chain(rooms_explicit)
+		.chain(rooms_selected)
+}

src/api/client/sync/v5/extensions/account_data.rs

@@ -0,0 +1,60 @@
+use futures::{StreamExt, future::join};
+use ruma::{api::client::sync::sync_events::v5::response, events::AnyRawAccountDataEvent};
+use tuwunel_core::{
+	Result, extract_variant,
+	utils::{IterStream, ReadyExt, stream::BroadbandExt},
+};
+use tuwunel_service::sync::Room;
+
+use super::{Connection, SyncInfo, Window, selector};
+
+#[tracing::instrument(name = "account_data", level = "trace", skip_all)]
+pub(super) async fn collect(
+	sync_info: SyncInfo<'_>,
+	conn: &Connection,
+	window: &Window,
+) -> Result<response::AccountData> {
+	let SyncInfo { services, sender_user, .. } = sync_info;
+
+	let implicit = conn
+		.extensions
+		.account_data
+		.lists
+		.as_deref()
+		.map(<[_]>::iter);
+
+	let explicit = conn
+		.extensions
+		.account_data
+		.rooms
+		.as_deref()
+		.map(<[_]>::iter);
+
+	let rooms = selector(sync_info, conn, window, implicit, explicit)
+		.stream()
+		.broad_filter_map(async |room_id| {
+			let &Room { roomsince, .. } = conn.rooms.get(room_id)?;
+			let changes: Vec<_> = services
+				.account_data
+				.changes_since(Some(room_id), sender_user, roomsince, Some(conn.next_batch))
+				.ready_filter_map(|e| extract_variant!(e, AnyRawAccountDataEvent::Room))
+				.collect()
+				.await;
+
+			changes
+				.is_empty()
+				.eq(&false)
+				.then(move || (room_id.to_owned(), changes))
+		})
+		.collect();
+
+	let global = services
+		.account_data
+		.changes_since(None, sender_user, conn.globalsince, Some(conn.next_batch))
+		.ready_filter_map(|e| extract_variant!(e, AnyRawAccountDataEvent::Global))
+		.collect();
+
+	let (global, rooms) = join(global, rooms).await;
+
+	Ok(response::AccountData { global, rooms })
+}

src/api/client/sync/v5/extensions/e2ee.rs

@@ -0,0 +1,219 @@
+use std::collections::HashSet;
+
+use futures::{
+	FutureExt, StreamExt, TryFutureExt,
+	future::{OptionFuture, join, join3},
+	stream::once,
+};
+use ruma::{
+	OwnedUserId, RoomId,
+	api::client::sync::sync_events::{DeviceLists, v5::response},
+	events::{
+		StateEventType, TimelineEventType,
+		room::member::{MembershipState, RoomMemberEventContent},
+	},
+};
+use tuwunel_core::{
+	Result, error,
+	matrix::{Event, pdu::PduCount},
+	pair_of,
+	utils::{
+		BoolExt, IterStream, ReadyExt, TryFutureExtExt, future::OptionStream,
+		stream::BroadbandExt,
+	},
+};
+use tuwunel_service::sync::Connection;
+
+use super::{SyncInfo, share_encrypted_room};
+
+#[tracing::instrument(name = "e2ee", level = "trace", skip_all)]
+pub(super) async fn collect(
+	sync_info: SyncInfo<'_>,
+	conn: &Connection,
+) -> Result<response::E2EE> {
+	let SyncInfo { services, sender_user, sender_device, .. } = sync_info;
+
+	let keys_changed = services
+		.users
+		.keys_changed(sender_user, conn.globalsince, Some(conn.next_batch))
+		.map(ToOwned::to_owned)
+		.collect::<HashSet<_>>()
+		.map(|changed| (changed, HashSet::new()));
+
+	let (changed, left) = (HashSet::new(), HashSet::new());
+	let (changed, left) = services
+		.state_cache
+		.rooms_joined(sender_user)
+		.map(ToOwned::to_owned)
+		.broad_filter_map(async |room_id| collect_room(sync_info, conn, &room_id).await.ok())
+		.chain(once(keys_changed))
+		.ready_fold((changed, left), |(mut changed, mut left), room| {
+			changed.extend(room.0);
+			left.extend(room.1);
+			(changed, left)
+		})
+		.await;
+
+	let left = left
+		.into_iter()
+		.stream()
+		.filter_map(async |user_id| {
+			share_encrypted_room(services, sender_user, &user_id, None)
+				.await
+				.is_false()
+				.then_some(user_id)
+		})
+		.collect();
+
+	let device_one_time_keys_count = services
+		.users
+		.last_one_time_keys_update(sender_user)
+		.then(|since| -> OptionFuture<_> {
+			since
+				.gt(&conn.globalsince)
+				.then(|| {
+					services
+						.users
+						.count_one_time_keys(sender_user, sender_device)
+				})
+				.into()
+		})
+		.map(Option::unwrap_or_default);
+
+	let (left, device_one_time_keys_count) = join(left, device_one_time_keys_count)
+		.boxed()
+		.await;
+
+	Ok(response::E2EE {
+		device_one_time_keys_count,
+		device_unused_fallback_key_types: None,
+		device_lists: DeviceLists {
+			changed: changed.into_iter().collect(),
+			left,
+		},
+	})
+}
+
+#[tracing::instrument(level = "trace", skip_all, fields(room_id), ret)]
+async fn collect_room(
+	SyncInfo { services, sender_user, .. }: SyncInfo<'_>,
+	conn: &Connection,
+	room_id: &RoomId,
+) -> Result<pair_of!(HashSet<OwnedUserId>)> {
+	let current_shortstatehash = services
+		.state
+		.get_room_shortstatehash(room_id)
+		.inspect_err(|e| error!("Room {room_id} has no state: {e}"));
+
+	let room_keys_changed = services
+		.users
+		.room_keys_changed(room_id, conn.globalsince, Some(conn.next_batch))
+		.map(|(user_id, _)| user_id)
+		.map(ToOwned::to_owned)
+		.collect::<HashSet<_>>();
+
+	let (current_shortstatehash, device_list_changed) =
+		join(current_shortstatehash, room_keys_changed)
+			.boxed()
+			.await;
+
+	let lists = (device_list_changed, HashSet::new());
+	let Ok(current_shortstatehash) = current_shortstatehash else {
+		return Ok(lists);
+	};
+
+	if current_shortstatehash <= conn.globalsince {
+		return Ok(lists);
+	}
+
+	let Ok(since_shortstatehash) = services
+		.timeline
+		.prev_shortstatehash(room_id, PduCount::Normal(conn.globalsince).saturating_add(1))
+		.await
+	else {
+		return Ok(lists);
+	};
+
+	if since_shortstatehash == current_shortstatehash {
+		return Ok(lists);
+	}
+
+	let encrypted_room = services
+		.state_accessor
+		.state_get(current_shortstatehash, &StateEventType::RoomEncryption, "")
+		.is_ok();
+
+	let since_encryption = services
+		.state_accessor
+		.state_get(since_shortstatehash, &StateEventType::RoomEncryption, "")
+		.is_ok();
+
+	let sender_joined_count = services
+		.state_cache
+		.get_joined_count(room_id, sender_user);
+
+	let (encrypted_room, since_encryption, sender_joined_count) =
+		join3(encrypted_room, since_encryption, sender_joined_count).await;
+
+	if !encrypted_room {
+		return Ok(lists);
+	}
+
+	let encrypted_since_last_sync = !since_encryption;
+	let joined_since_last_sync = sender_joined_count.is_ok_and(|count| count > conn.globalsince);
+	let joined_members_burst: OptionFuture<_> = (joined_since_last_sync
+		|| encrypted_since_last_sync)
+		.then(|| {
+			services
+				.state_cache
+				.room_members(room_id)
+				.ready_filter(|&user_id| user_id != sender_user)
+				.map(ToOwned::to_owned)
+				.map(|user_id| (MembershipState::Join, user_id))
+				.boxed()
+				.into_future()
+		})
+		.into();
+
+	services
+		.state_accessor
+		.state_added((since_shortstatehash, current_shortstatehash))
+		.broad_filter_map(async |(_shortstatekey, shorteventid)| {
+			services
+				.timeline
+				.get_pdu_from_shorteventid(shorteventid)
+				.ok()
+				.await
+		})
+		.ready_filter(|event| *event.kind() == TimelineEventType::RoomMember)
+		.ready_filter(|event| {
+			event
+				.state_key()
+				.is_some_and(|state_key| state_key != sender_user)
+		})
+		.ready_filter_map(|event| {
+			let content: RoomMemberEventContent = event.get_content().ok()?;
+			let user_id: OwnedUserId = event.state_key()?.parse().ok()?;
+
+			Some((content.membership, user_id))
+		})
+		.chain(joined_members_burst.stream())
+		.fold(lists, async |(mut changed, mut left), (membership, user_id)| {
+			use MembershipState::*;
+
+			let should_add = async |user_id| {
+				!share_encrypted_room(services, sender_user, user_id, Some(room_id)).await
+			};
+
+			match membership {
+				| Join if should_add(&user_id).await => changed.insert(user_id),
+				| Leave => left.insert(user_id),
+				| _ => false,
+			};
+
+			(changed, left)
+		})
+		.map(Ok)
+		.boxed()
+		.await
+}

src/api/client/sync/v5/extensions/receipts.rs

@@ -0,0 +1,93 @@
+use futures::{FutureExt, StreamExt};
+use ruma::{
+	OwnedRoomId, RoomId,
+	api::client::sync::sync_events::v5::response,
+	events::{AnySyncEphemeralRoomEvent, receipt::SyncReceiptEvent},
+	serde::Raw,
+};
+use tuwunel_core::{
+	Result,
+	utils::{BoolExt, IterStream, stream::BroadbandExt},
+};
+use tuwunel_service::{rooms::read_receipt::pack_receipts, sync::Room};
+
+use super::{Connection, SyncInfo, Window, selector};
+
+#[tracing::instrument(name = "receipts", level = "trace", skip_all)]
+pub(super) async fn collect(
+	sync_info: SyncInfo<'_>,
+	conn: &Connection,
+	window: &Window,
+) -> Result<response::Receipts> {
+	let SyncInfo { .. } = sync_info;
+
+	let implicit = conn
+		.extensions
+		.receipts
+		.lists
+		.as_deref()
+		.map(<[_]>::iter);
+
+	let explicit = conn
+		.extensions
+		.receipts
+		.rooms
+		.as_deref()
+		.map(<[_]>::iter);
+
+	let rooms = selector(sync_info, conn, window, implicit, explicit)
+		.stream()
+		.broad_filter_map(|room_id| collect_room(sync_info, conn, window, room_id))
+		.collect()
+		.await;
+
+	Ok(response::Receipts { rooms })
+}
+
+#[tracing::instrument(level = "trace", skip_all, fields(room_id), ret)]
+async fn collect_room(
+	SyncInfo { services, sender_user, .. }: SyncInfo<'_>,
+	conn: &Connection,
+	_window: &Window,
+	room_id: &RoomId,
+) -> Option<(OwnedRoomId, Raw<SyncReceiptEvent>)> {
+	let &Room { roomsince, .. } = conn.rooms.get(room_id)?;
+	let private_receipt = services
+		.read_receipt
+		.last_privateread_update(sender_user, room_id)
+		.then(async |last_private_update| {
+			if last_private_update <= roomsince || last_private_update > conn.next_batch {
+				return None;
+			}
+
+			services
+				.read_receipt
+				.private_read_get(room_id, sender_user)
+				.map(Some)
+				.await
+		})
+		.map(Option::into_iter)
+		.map(Iterator::flatten)
+		.map(IterStream::stream)
+		.flatten_stream();
+
+	let receipts: Vec<Raw<AnySyncEphemeralRoomEvent>> = services
+		.read_receipt
+		.readreceipts_since(room_id, roomsince, Some(conn.next_batch))
+		.filter_map(async |(read_user, _ts, v)| {
+			services
+				.users
+				.user_is_ignored(read_user, sender_user)
+				.await
+				.or_some(v)
+		})
+		.chain(private_receipt)
+		.collect()
+		.boxed()
+		.await;
+
+	receipts
+		.is_empty()
+		.is_false()
+		.then(|| (room_id.to_owned(), pack_receipts(receipts.into_iter())))
+}

src/api/client/sync/v5/extensions/to_device.rs

@@ -0,0 +1,33 @@
+use futures::StreamExt;
+use ruma::api::client::sync::sync_events::v5::response;
+use tuwunel_core::{self, Result, at};
+
+use super::{Connection, SyncInfo};
+
+#[tracing::instrument(name = "to_device", level = "trace", skip_all, ret)]
+pub(super) async fn collect(
+	SyncInfo { services, sender_user, sender_device, .. }: SyncInfo<'_>,
+	conn: &Connection,
+) -> Result<Option<response::ToDevice>> {
+	services
+		.users
+		.remove_to_device_events(sender_user, sender_device, conn.globalsince)
+		.await;
+
+	let events: Vec<_> = services
+		.users
+		.get_to_device_events(sender_user, sender_device, None, Some(conn.next_batch))
+		.map(at!(1))
+		.collect()
+		.await;
+
+	let to_device = events
+		.is_empty()
+		.eq(&false)
+		.then(|| response::ToDevice {
+			next_batch: conn.next_batch.to_string().into(),
+			events,
+		});
+
+	Ok(to_device)
+}

src/api/client/sync/v5/extensions/typing.rs

@@ -0,0 +1,63 @@
+use std::collections::BTreeMap;
+
+use futures::{FutureExt, StreamExt, TryFutureExt};
+use ruma::{
+	api::client::sync::sync_events::v5::response,
+	events::typing::{SyncTypingEvent, TypingEventContent},
+	serde::Raw,
+};
+use tuwunel_core::{
+	Result, debug_error,
+	utils::{IterStream, ReadyExt},
+};
+
+use super::{Connection, SyncInfo, Window, selector};
+
+#[tracing::instrument(name = "typing", level = "trace", skip_all, ret)]
+pub(super) async fn collect(
+	sync_info: SyncInfo<'_>,
+	conn: &Connection,
+	window: &Window,
+) -> Result<response::Typing> {
+	use response::Typing;
+
+	let SyncInfo { services, sender_user, .. } = sync_info;
+
+	let implicit = conn
+		.extensions
+		.typing
+		.lists
+		.as_deref()
+		.map(<[_]>::iter);
+
+	let explicit = conn
+		.extensions
+		.typing
+		.rooms
+		.as_deref()
+		.map(<[_]>::iter);
+
+	selector(sync_info, conn, window, implicit, explicit)
+		.stream()
+		.filter_map(async |room_id| {
+			services
+				.typing
+				.typing_users_for_user(room_id, sender_user)
+				.inspect_err(|e| debug_error!(%room_id, "Failed to get typing events: {e}"))
+				.await
+				.ok()
+				.filter(|users| !users.is_empty())
+				.map(|users| (room_id, users))
+		})
+		.ready_filter_map(|(room_id, users)| {
+			let content = TypingEventContent::new(users);
+			let event = SyncTypingEvent { content };
+			let event = Raw::new(&event);
+
+			Some((room_id.to_owned(), event.ok()?))
+		})
+		.collect::<BTreeMap<_, _>>()
+		.map(|rooms| Typing { rooms })
+		.map(Ok)
+		.await
+}

src/api/client/sync/v5/filter.rs

@@ -0,0 +1,153 @@
+use futures::{StreamExt, future::OptionFuture, pin_mut};
+use ruma::{
+	RoomId, api::client::sync::sync_events::v5::request::ListFilters, directory::RoomTypeFilter,
+	events::room::member::MembershipState,
+};
+use tuwunel_core::{
+	is_equal_to, is_true,
+	utils::{
+		BoolExt, FutureBoolExt, IterStream, ReadyExt,
+		future::{self, OptionExt, ReadyBoolExt},
+	},
+};
+
+use super::SyncInfo;
+
+#[tracing::instrument(name = "filter", level = "trace", skip_all)]
+pub(super) async fn filter_room(
+	SyncInfo { services, sender_user, .. }: SyncInfo<'_>,
+	filter: &ListFilters,
+	room_id: &RoomId,
+	membership: Option<&MembershipState>,
+) -> bool {
+	let match_invite: OptionFuture<_> = filter
+		.is_invite
+		.map(async |is_invite| match (membership, is_invite) {
+			| (Some(MembershipState::Invite), true) => true,
+			| (Some(MembershipState::Invite), false) => false,
+			| (Some(_), true) => false,
+			| (Some(_), false) => true,
+			| _ =>
+				services
+					.state_cache
+					.is_invited(sender_user, room_id)
+					.await == is_invite,
+		})
+		.into();
+
+	let match_direct: OptionFuture<_> = filter
+		.is_dm
+		.map(async |is_dm| {
+			services
+				.account_data
+				.is_direct(sender_user, room_id)
+				.await == is_dm
+		})
+		.into();
+
+	let match_direct_member: OptionFuture<_> = filter
+		.is_dm
+		.map(async |is_dm| {
+			services
+				.state_accessor
+				.is_direct(room_id, sender_user)
+				.await == is_dm
+		})
+		.into();
+
+	let match_encrypted: OptionFuture<_> = filter
+		.is_encrypted
+		.map(async |is_encrypted| {
+			services
+				.state_accessor
+				.is_encrypted_room(room_id)
+				.await == is_encrypted
+		})
+		.into();
+
+	let match_space_child: OptionFuture<_> = filter
+		.spaces
+		.is_empty()
+		.is_false()
+		.then(async || {
+			filter
+				.spaces
+				.iter()
+				.stream()
+				.flat_map(|room_id| services.spaces.get_space_children(room_id))
+				.ready_any(is_equal_to!(room_id))
+				.await
+		})
+		.into();
+
+	let fetch_tags = !filter.tags.is_empty() || !filter.not_tags.is_empty();
+	let match_room_tag: OptionFuture<_> = fetch_tags
+		.then(async || {
+			if let Some(tags) = services
+				.account_data
+				.get_room_tags(sender_user, room_id)
+				.await
+				.ok()
+				.filter(|tags| !tags.is_empty())
+			{
+				tags.keys().any(|tag| {
+					(filter.not_tags.is_empty() || !filter.not_tags.contains(tag))
+						|| (!filter.tags.is_empty() && filter.tags.contains(tag))
+				})
+			} else {
+				filter.tags.is_empty()
+			}
+		})
+		.into();
+
+	let fetch_room_type = !filter.room_types.is_empty() || !filter.not_room_types.is_empty();
+	let match_room_type: OptionFuture<_> = fetch_room_type
+		.then(async || {
+			let room_type = services
+				.state_accessor
+				.get_room_type(room_id)
+				.await
+				.ok();
+
+			let room_type = RoomTypeFilter::from(room_type);
+			(filter.not_room_types.is_empty() || !filter.not_room_types.contains(&room_type))
+				&& (filter.room_types.is_empty() || filter.room_types.contains(&room_type))
+		})
+		.into();
+
+	future::and7(
+		match_invite.is_none_or(is_true!()),
+		match_encrypted.is_none_or(is_true!()),
+		match_direct.is_none_or(is_true!()),
+		match_direct_member.is_none_or(is_true!()),
+		match_space_child.is_none_or(is_true!()),
+		match_room_type.is_none_or(is_true!()),
+		match_room_tag.is_none_or(is_true!()),
+	)
+	.await
+}
+
+#[tracing::instrument(name = "filter_meta", level = "trace", skip_all)]
+pub(super) async fn filter_room_meta(
+	SyncInfo { services, sender_user, .. }: SyncInfo<'_>,
+	room_id: &RoomId,
+) -> bool {
+	let not_exists = services.metadata.exists(room_id).is_false();
+
+	let is_disabled = services.metadata.is_disabled(room_id);
+
+	let is_banned = services.metadata.is_banned(room_id);
+
+	let not_visible = services
+		.state_accessor
+		.user_can_see_state_events(sender_user, room_id)
+		.is_false();
+
+	pin_mut!(not_visible, not_exists, is_disabled, is_banned);
+	not_visible
+		.or(not_exists)
+		.or(is_disabled)
+		.or(is_banned)
+		.is_false()
+		.await
+}

src/api/client/sync/v5/rooms.rs

@@ -0,0 +1,455 @@
+use std::{
+	cmp::Ordering,
+	collections::{BTreeMap, HashSet},
+};
+
+use futures::{
+	FutureExt, StreamExt, TryFutureExt, TryStreamExt,
+	future::{OptionFuture, join, join3, join4},
+};
+use ruma::{
+	JsOption, MxcUri, OwnedMxcUri, OwnedRoomId, RoomId, UserId,
+	api::client::sync::sync_events::{
+		UnreadNotificationsCount,
+		v5::{DisplayName, response, response::Heroes},
+	},
+	events::{
+		StateEventType,
+		TimelineEventType::{
+			self, Beacon, CallInvite, PollStart, RoomEncrypted, RoomMessage, Sticker,
+		},
+		room::member::MembershipState,
+	},
+};
+use tuwunel_core::{
+	Result, at, err, error, is_equal_to,
+	matrix::{Event, StateKey, pdu::PduCount},
+	ref_at,
+	utils::{
+		BoolExt, IterStream, ReadyExt, TryFutureExtExt, math::usize_from_ruma, result::FlatOk,
+		stream::BroadbandExt,
+	},
+};
+use tuwunel_service::{Services, sync::Room};
+
+use super::{super::load_timeline, Connection, SyncInfo, Window, WindowRoom};
+use crate::client::ignored_filter;
+
+static DEFAULT_BUMP_TYPES: [TimelineEventType; 6] =
+	[CallInvite, PollStart, Beacon, RoomEncrypted, RoomMessage, Sticker];
+
+#[tracing::instrument(
+    name = "rooms",
+    level = "debug",
+    skip_all,
+    fields(
+        next_batch = conn.next_batch,
+        window = window.len(),
+    )
+)]
+pub(super) async fn handle(
+	sync_info: SyncInfo<'_>,
+	conn: &Connection,
+	window: &Window,
+) -> Result<BTreeMap<OwnedRoomId, response::Room>> {
+	window
+		.iter()
+		.stream()
+		.broad_filter_map(async |(room_id, room)| {
+			handle_room(sync_info, conn, room)
+				.map_ok(move |room| (room_id.clone(), room))
+				.inspect_err(|e| error!(?room_id, "sync handler: {e:?}"))
+				.await
+				.ok()
+		})
+		.collect()
+		.map(Ok)
+		.await
+}
+
+#[tracing::instrument(
+	name = "room",
+	level = "debug",
+	skip_all,
+	fields(room_id, roomsince)
+)]
+#[allow(clippy::too_many_arguments)]
+async fn handle_room(
+	SyncInfo { services, sender_user, .. }: SyncInfo<'_>,
+	conn: &Connection,
+	WindowRoom {
+		lists, membership, room_id, last_count, ..
+	}: &WindowRoom,
+) -> Result<response::Room> {
+	debug_assert!(
+		DEFAULT_BUMP_TYPES.is_sorted(),
+		"DEFAULT_BUMP_TYPES must be sorted for binary search"
+	);
+
+	let &Room { roomsince, .. } = conn
+		.rooms
+		.get(room_id)
+		.ok_or_else(|| err!("Missing connection state for {room_id}"))?;
+
+	debug_assert!(
+		*last_count > roomsince || *last_count == 0 || roomsince == 0,
+		"Stale room shouldn't be in the window"
+	);
+
+	if *membership == Some(MembershipState::Leave) {
+		return Ok(response::Room {
+			initial: roomsince.eq(&0).then_some(true),
+			lists: lists.clone(),
+			membership: membership.clone(),
+			prev_batch: Some(conn.next_batch.to_string().into()),
+			limited: true,
+			required_state: vec![
+				services
+					.state_accessor
+					.room_state_get(room_id, &StateEventType::RoomMember, sender_user.as_str())
+					.map_ok(Event::into_format)
+					.await?,
+			],
+
+			..Default::default()
+		});
+	}
+
+	let is_invite = *membership == Some(MembershipState::Invite);
+	let default_details = (0_usize, HashSet::new());
+	let (timeline_limit, required_state) = lists
+		.iter()
+		.filter_map(|list_id| conn.lists.get(list_id))
+		.map(|list| &list.room_details)
+		.chain(conn.subscriptions.get(room_id).into_iter())
+		.fold(default_details, |(mut timeline_limit, mut required_state), config| {
+			let limit = usize_from_ruma(config.timeline_limit);
+
+			timeline_limit = timeline_limit.max(limit);
+			required_state.extend(config.required_state.clone());
+
+			(timeline_limit, required_state)
+		});
+
+	let timeline: OptionFuture<_> = is_invite
+		.is_false()
+		.then(|| {
+			load_timeline(
+				services,
+				sender_user,
+				room_id,
+				PduCount::Normal(roomsince),
+				Some(PduCount::from(conn.next_batch)),
+				timeline_limit,
+			)
+		})
+		.into();
+
+	let (timeline_pdus, limited, _lastcount) = timeline
+		.await
+		.flat_ok()
+		.unwrap_or_else(|| (Vec::new(), true, PduCount::default()));
+
+	let required_state = required_state
+		.into_iter()
+		.filter(|_| !timeline_pdus.is_empty())
+		.collect::<Vec<_>>();
+
+	let prev_batch = timeline_pdus
+		.first()
+		.map(at!(0))
+		.map(PduCount::into_unsigned)
+		.as_ref()
+		.map(ToString::to_string);
+
+	let bump_stamp = timeline_pdus
+		.iter()
+		.filter(|(_, pdu)| {
+			if *pdu.event_type() == TimelineEventType::RoomMember {
+				return pdu
+					.state_key()
+					.is_some_and(is_equal_to!(sender_user.as_str()));
+			}
+
+			DEFAULT_BUMP_TYPES
+				.binary_search(pdu.event_type())
+				.is_ok()
+		})
+		.filter(|(_, pdu)| !pdu.is_redacted())
+		.map(at!(0))
+		.map(PduCount::into_signed)
+		.max()
+		.map(TryInto::try_into)
+		.flat_ok();
+
+	let num_live: OptionFuture<_> = roomsince
+		.ne(&0)
+		.and_is(limited || timeline_pdus.len() >= timeline_limit)
+		.then(|| {
+			services
+				.timeline
+				.pdus(None, room_id, Some(roomsince.into()))
+				.count()
+				.map(TryInto::try_into)
+				.map(Result::ok)
+		})
+		.into();
+
+	let lazy = required_state
+		.iter()
+		.any(is_equal_to!(&(StateEventType::RoomMember, "$LAZY".into())));
+
+	let mut timeline_senders: Vec<_> = timeline_pdus
+		.iter()
+		.filter(|_| lazy)
+		.map(ref_at!(1))
+		.map(Event::sender)
+		.collect();
+
+	timeline_senders.sort();
+	timeline_senders.dedup();
+	let timeline_senders = timeline_senders
+		.iter()
+		.map(|sender| (StateEventType::RoomMember, StateKey::from_str(sender.as_str())))
+		.stream();
+
+	let wildcard_state = required_state
+		.iter()
+		.filter(|(_, state_key)| state_key == "*")
+		.map(|(event_type, _)| {
+			services
+				.state_accessor
+				.room_state_keys(room_id, event_type)
+				.map_ok(|state_key| (event_type.clone(), state_key))
+				.ready_filter_map(Result::ok)
+		})
+		.stream()
+		.flatten();
+
+	let required_state = required_state
+		.iter()
+		.cloned()
+		.stream()
+		.chain(wildcard_state)
+		.chain(timeline_senders)
+		.broad_filter_map(async |state| {
+			let state_key: StateKey = match state.1.as_str() {
+				| "$LAZY" | "*" => return None,
+				| "$ME" => sender_user.as_str().into(),
+				| _ => state.1.clone(),
+			};
+
+			services
+				.state_accessor
+				.room_state_get(room_id, &state.0, &state_key)
+				.map_ok(Event::into_format)
+				.ok()
+				.await
+		})
+		.collect();
+
+	// TODO: figure out a timestamp we can use for remote invites
+	let invite_state: OptionFuture<_> = is_invite
+		.then(|| {
+			services
+				.state_cache
+				.invite_state(sender_user, room_id)
+				.ok()
+		})
+		.into();
+
+	let room_name = services
+		.state_accessor
+		.get_name(room_id)
+		.map_ok(Into::into)
+		.map(Result::ok);
+
+	let room_avatar = services
+		.state_accessor
+		.get_avatar(room_id)
+		.map_ok(|content| content.url)
+		.ok()
+		.map(Option::flatten);
+
+	let highlight_count = services
+		.pusher
+		.highlight_count(sender_user, room_id)
+		.map(TryInto::try_into)
+		.map(Result::ok);
+
+	let notification_count = services
+		.pusher
+		.notification_count(sender_user, room_id)
+		.map(TryInto::try_into)
+		.map(Result::ok);
+
+	let joined_count = services
+		.state_cache
+		.room_joined_count(room_id)
+		.map_ok(TryInto::try_into)
+		.map_ok(Result::ok)
+		.map(FlatOk::flat_ok);
+
+	let invited_count = services
+		.state_cache
+		.room_invited_count(room_id)
+		.map_ok(TryInto::try_into)
+		.map_ok(Result::ok)
+		.map(FlatOk::flat_ok);
+
+	let is_dm = services
+		.state_accessor
+		.is_direct(room_id, sender_user)
+		.map(|is_dm| is_dm.then_some(is_dm));
+
+	let last_read_count = services
+		.pusher
+		.last_notification_read(sender_user, room_id);
+
+	let timeline = timeline_pdus
+		.iter()
+		.stream()
+		.filter_map(|item| ignored_filter(services, item.clone(), sender_user))
+		.map(at!(1))
+		.map(Event::into_format)
+		.collect();
+
+	let meta = join3(room_name, room_avatar, is_dm);
+	let events = join4(timeline, num_live, required_state, invite_state);
+	let member_counts = join(joined_count, invited_count);
+	let notification_counts = join3(highlight_count, notification_count, last_read_count);
+	let (
+		(room_name, room_avatar, is_dm),
+		(timeline, num_live, required_state, invite_state),
+		(joined_count, invited_count),
+		(highlight_count, notification_count, _last_notification_read),
+	) = join4(meta, events, member_counts, notification_counts)
+		.boxed()
+		.await;
+
+	let heroes: OptionFuture<_> = services
+		.config
+		.calculate_heroes
+		.then(|| {
+			calculate_heroes(
+				services,
+				sender_user,
+				room_id,
+				room_name.as_ref(),
+				room_avatar.as_deref(),
+			)
+		})
+		.into();
+
+	let (heroes, heroes_name, heroes_avatar) = heroes.await.unwrap_or_default();
+
+	Ok(response::Room {
+		initial: roomsince.eq(&0).then_some(true),
+		lists: lists.clone(),
+		membership: membership.clone(),
+		name: room_name.or(heroes_name),
+		avatar: JsOption::from_option(room_avatar.or(heroes_avatar)),
+		is_dm,
+		heroes,
+		required_state,
+		invite_state: invite_state.flatten(),
+		prev_batch: prev_batch.as_deref().map(Into::into),
+		num_live: num_live.flatten(),
+		limited,
+		timeline,
+		bump_stamp,
+		joined_count,
+		invited_count,
+		unread_notifications: UnreadNotificationsCount { highlight_count, notification_count },
+	})
+}
+
+#[tracing::instrument(name = "heroes", level = "trace", skip_all)]
+#[allow(clippy::type_complexity)]
+async fn calculate_heroes(
+	services: &Services,
+	sender_user: &UserId,
+	room_id: &RoomId,
+	room_name: Option<&DisplayName>,
+	room_avatar: Option<&MxcUri>,
+) -> (Option<Heroes>, Option<DisplayName>, Option<OwnedMxcUri>) {
+	const MAX_HEROES: usize = 5;
+
+	let heroes: Heroes = services
+		.state_cache
+		.room_members(room_id)
+		.ready_filter(|&member| member != sender_user)
+		.ready_filter_map(|member| room_name.is_none().then_some(member))
+		.map(ToOwned::to_owned)
+		.broadn_filter_map(MAX_HEROES, async |user_id| {
+			let content = services
+				.state_accessor
+				.get_member(room_id, &user_id)
+				.await
+				.ok()?;
+
+			let name: OptionFuture<_> = content
+				.displayname
+				.is_none()
+				.then(|| services.users.displayname(&user_id).ok())
+				.into();
+
+			let avatar: OptionFuture<_> = content
+				.avatar_url
+				.is_none()
+				.then(|| services.users.avatar_url(&user_id).ok())
+				.into();
+
+			let (name, avatar) = join(name, avatar).await;
+			let hero = response::Hero {
+				user_id,
+				avatar: avatar.unwrap_or(content.avatar_url),
+				name: name
+					.unwrap_or(content.displayname)
+					.map(Into::into),
+			};
+
+			Some(hero)
+		})
+		.take(MAX_HEROES)
+		.collect()
+		.await;
+
+	let hero_name = match heroes.len().cmp(&(1_usize)) {
+		| Ordering::Less => None,
+		| Ordering::Equal => Some(
+			heroes[0]
+				.name
+				.clone()
+				.unwrap_or_else(|| heroes[0].user_id.as_str().into()),
+		),
+		| Ordering::Greater => {
+			let firsts = heroes[1..]
+				.iter()
+				.map(|h| {
+					h.name
+						.clone()
+						.unwrap_or_else(|| h.user_id.as_str().into())
+				})
+				.collect::<Vec<_>>()
+				.join(", ");
+
+			let last = heroes[0]
+				.name
+				.clone()
+				.unwrap_or_else(|| heroes[0].user_id.as_str().into());
+
+			Some(format!("{firsts} and {last}")).map(Into::into)
+		},
+	};
+
+	let heroes_avatar = (room_avatar.is_none() && room_name.is_none())
+		.then(|| {
+			heroes
+				.first()
+				.and_then(|hero| hero.avatar.clone())
+		})
+		.flatten();
+
+	(Some(heroes), hero_name, heroes_avatar)
+}

src/api/client/sync/v5/selector.rs

@@ -0,0 +1,264 @@
+use std::cmp::Ordering;
+
+use futures::{
+	FutureExt, StreamExt, TryFutureExt,
+	future::{OptionFuture, join5},
+};
+use ruma::{OwnedRoomId, UInt, events::room::member::MembershipState, uint};
+use tuwunel_core::{
+	apply, is_true,
+	matrix::PduCount,
+	trace,
+	utils::{
+		BoolExt, TryFutureExtExt,
+		math::usize_from_ruma,
+		stream::{BroadbandExt, IterStream},
+	},
+};
+use tuwunel_service::sync::Connection;
+
+use super::{
+	ListIds, ResponseLists, SyncInfo, Window, WindowRoom,
+	filter::{filter_room, filter_room_meta},
+};
+
+#[tracing::instrument(level = "debug", skip_all)]
+pub(super) async fn selector(
+	conn: &mut Connection,
+	sync_info: SyncInfo<'_>,
+) -> (Window, ResponseLists) {
+	use MembershipState::*;
+
+	let SyncInfo { services, sender_user, .. } = sync_info;
+
+	let mut rooms = services
+		.state_cache
+		.user_memberships(sender_user, Some(&[Join, Invite, Knock]))
+		.map(|(membership, room_id)| (room_id.to_owned(), Some(membership)))
+		.broad_filter_map(|(room_id, membership)| matcher(sync_info, conn, room_id, membership))
+		.collect::<Vec<_>>()
+		.await;
+
+	rooms.sort_by(room_sort);
+	rooms
+		.iter_mut()
+		.enumerate()
+		.for_each(|(i, room)| {
+			room.ranked = i;
+
+			conn.rooms
+				.entry(room.room_id.clone())
+				.or_default();
+		});
+
+	trace!(?rooms);
+	let lists = response_lists(rooms.iter());
+
+	trace!(?lists);
+	let window = window(sync_info, conn, rooms.iter(), &lists).await;
+
+	trace!(?window);
+	(window, lists)
+}
+
+#[tracing::instrument(
+	name = "matcher",
+	level = "trace",
+	skip_all,
+	fields(?room_id, ?membership)
+)]
+async fn matcher(
+	sync_info: SyncInfo<'_>,
+	conn: &Connection,
+	room_id: OwnedRoomId,
+	membership: Option<MembershipState>,
+) -> Option<WindowRoom> {
+	let SyncInfo { services, sender_user, .. } = sync_info;
+
+	let (matched, lists) = conn
+		.lists
+		.iter()
+		.stream()
+		.filter_map(async |(id, list)| {
+			let filter: OptionFuture<_> = list
+				.filters
+				.clone()
+				.map(async |filters| {
+					filter_room(sync_info, &filters, &room_id, membership.as_ref()).await
+				})
+				.into();
+
+			filter
+				.await
+				.is_none_or(is_true!())
+				.then(|| id.clone())
+		})
+		.collect::<ListIds>()
+		.map(|lists| (lists.is_empty().is_false(), lists))
+		.await;
+
+	let last_notification: OptionFuture<_> = matched
+		.then(|| {
+			services
+				.pusher
+				.last_notification_read(sender_user, &room_id)
+				.unwrap_or_default()
+		})
+		.into();
+
+	let last_privateread: OptionFuture<_> = matched
+		.then(|| {
+			services
+				.read_receipt
+				.last_privateread_update(sender_user, &room_id)
+		})
+		.into();
+
+	let last_receipt: OptionFuture<_> = matched
+		.then(|| {
+			services
+				.read_receipt
+				.last_receipt_count(&room_id, sender_user.into(), None)
+				.unwrap_or_default()
+		})
+		.into();
+
+	let last_account: OptionFuture<_> = matched
+		.then(|| {
+			services
+				.account_data
+				.last_count(Some(room_id.as_ref()), sender_user, Some(conn.next_batch))
+				.unwrap_or_default()
+		})
+		.into();
+
+	let last_timeline: OptionFuture<_> = matched
+		.then(|| {
+			services
+				.timeline
+				.last_timeline_count(None, &room_id, Some(conn.next_batch.into()))
+				.map_ok(PduCount::into_unsigned)
+				.unwrap_or_default()
+		})
+		.into();
+
+	let (last_timeline, last_notification, last_account, last_receipt, last_privateread) =
+		join5(last_timeline, last_notification, last_account, last_receipt, last_privateread)
+			.await;
+
+	Some(WindowRoom {
+		room_id: room_id.clone(),
+		membership,
+		lists,
+		ranked: 0,
+		last_count: [
+			last_timeline,
+			last_notification,
+			last_account,
+			last_receipt,
+			last_privateread,
+		]
+		.into_iter()
+		.map(Option::unwrap_or_default)
+		.filter(|count| conn.next_batch.ge(count))
+		.max()
+		.unwrap_or_default(),
+	})
+}
+
+#[tracing::instrument(
+	level = "debug",
+	skip_all,
+	fields(rooms = rooms.clone().count())
+)]
+async fn window<'a, Rooms>(
+	sync_info: SyncInfo<'_>,
+	conn: &Connection,
+	rooms: Rooms,
+	lists: &ResponseLists,
+) -> Window
+where
+	Rooms: Iterator<Item = &'a WindowRoom> + Clone + Send + Sync,
+{
+	static FULL_RANGE: (UInt, UInt) = (UInt::MIN, UInt::MAX);
+
+	let SyncInfo { services, sender_user, .. } = sync_info;
+
+	let selections = lists
+		.keys()
+		.cloned()
+		.filter_map(|id| conn.lists.get(&id).map(|list| (id, list)))
+		.flat_map(|(id, list)| {
+			let full_range = list
+				.ranges
+				.is_empty()
+				.then_some(&FULL_RANGE)
+				.into_iter();
+
+			list.ranges
+				.iter()
+				.chain(full_range)
+				.map(apply!(2, usize_from_ruma))
+				.map(move |range| (id.clone(), range))
+		})
+		.flat_map(|(id, (start, end))| {
+			rooms
+				.clone()
+				.filter(move |&room| room.lists.contains(&id))
+				.filter(|&room| {
+					conn.rooms
+						.get(&room.room_id)
+						.is_some_and(|conn_room| {
+							conn_room.roomsince == 0 || room.last_count > conn_room.roomsince
+						})
+				})
+				.enumerate()
+				.skip_while(move |&(i, _)| i < start)
+				.take(end.saturating_add(1).saturating_sub(start))
+				.map(|(_, room)| (room.room_id.clone(), room.clone()))
+		})
+		.stream();
+
+	let subscriptions = conn
+		.subscriptions
+		.iter()
+		.stream()
+		.broad_filter_map(async |(room_id, _)| {
+			filter_room_meta(sync_info, room_id)
+				.await
+				.into_option()?;
+
+			Some(WindowRoom {
+				room_id: room_id.clone(),
+				lists: Default::default(),
+				ranked: usize::MAX,
+				last_count: 0,
+				membership: services
+					.state_cache
+					.user_membership(sender_user, room_id)
+					.await,
+			})
+		})
+		.map(|room| (room.room_id.clone(), room));
+
+	subscriptions.chain(selections).collect().await
+}
+
+fn response_lists<'a, Rooms>(rooms: Rooms) -> ResponseLists
+where
+	Rooms: Iterator<Item = &'a WindowRoom>,
+{
+	rooms
+		.flat_map(|room| room.lists.iter())
+		.fold(ResponseLists::default(), |mut lists, id| {
+			let list = lists.entry(id.clone()).or_default();
+			list.count = list
+				.count
+				.checked_add(uint!(1))
+				.expect("list count must not overflow JsInt");
+
+			lists
+		})
+}
+
+fn room_sort(a: &WindowRoom, b: &WindowRoom) -> Ordering { b.last_count.cmp(&a.last_count) }

src/api/client/typing.rs

@@ -28,7 +28,7 @@ pub(crate) async fn create_typing_event_route(
 
 	match body.state {
 		| Typing::Yes(duration) => {
-			let duration = utils::clamp(
+			let duration = Ord::clamp(
 				duration
 					.as_millis()
 					.try_into()
@@ -64,12 +64,14 @@ pub(crate) async fn create_typing_event_route(
 	}
 
 	// ping presence
-	if services.config.allow_local_presence {
-		services
-			.presence
-			.ping_presence(&body.user_id, &ruma::presence::PresenceState::Online)
-			.await?;
-	}
+	services
+		.presence
+		.maybe_ping_presence(
+			&body.user_id,
+			body.sender_device.as_deref(),
+			&ruma::presence::PresenceState::Online,
+		)
+		.await?;
 
 	Ok(create_typing_event::v3::Response {})
 }

src/api/client/unstable.rs

@@ -73,13 +73,11 @@ pub(crate) async fn delete_timezone_key_route(
 
 	services.users.set_timezone(&body.user_id, None);
 
-	if services.config.allow_local_presence {
-		// Presence update
-		services
-			.presence
-			.ping_presence(&body.user_id, &PresenceState::Online)
-			.await?;
-	}
+	// Presence update
+	services
+		.presence
+		.maybe_ping_presence(&body.user_id, body.sender_device.as_deref(), &PresenceState::Online)
+		.await?;
 
 	Ok(delete_timezone_key::unstable::Response {})
 }
@@ -103,13 +101,11 @@ pub(crate) async fn set_timezone_key_route(
 		.users
 		.set_timezone(&body.user_id, body.tz.clone());
 
-	if services.config.allow_local_presence {
-		// Presence update
-		services
-			.presence
-			.ping_presence(&body.user_id, &PresenceState::Online)
-			.await?;
-	}
+	// Presence update
+	services
+		.presence
+		.maybe_ping_presence(&body.user_id, body.sender_device.as_deref(), &PresenceState::Online)
+		.await?;
 
 	Ok(set_timezone_key::unstable::Response {})
 }
@@ -171,13 +167,11 @@ pub(crate) async fn set_profile_field_route(
 		);
 	}
 
-	if services.config.allow_local_presence {
-		// Presence update
-		services
-			.presence
-			.ping_presence(&body.user_id, &PresenceState::Online)
-			.await?;
-	}
+	// Presence update
+	services
+		.presence
+		.maybe_ping_presence(&body.user_id, body.sender_device.as_deref(), &PresenceState::Online)
+		.await?;
 
 	Ok(set_profile_field::v3::Response {})
 }
@@ -227,13 +221,11 @@ pub(crate) async fn delete_profile_field_route(
 			.set_profile_key(&body.user_id, body.field.as_str(), None);
 	}
 
-	if services.config.allow_local_presence {
-		// Presence update
-		services
-			.presence
-			.ping_presence(&body.user_id, &PresenceState::Online)
-			.await?;
-	}
+	// Presence update
+	services
+		.presence
+		.maybe_ping_presence(&body.user_id, body.sender_device.as_deref(), &PresenceState::Online)
+		.await?;
 
 	Ok(delete_profile_field::v3::Response {})
 }
@@ -251,8 +243,8 @@ pub(crate) async fn get_timezone_key_route(
 	if !services.globals.user_is_local(&body.user_id) {
 		// Create and update our local copy of the user
 		if let Ok(response) = services
-			.sending
-			.send_federation_request(
+			.federation
+			.execute(
 				body.user_id.server_name(),
 				federation::query::get_profile_information::v1::Request {
 					user_id: body.user_id.clone(),
@@ -312,8 +304,8 @@ pub(crate) async fn get_profile_field_route(
 	if !services.globals.user_is_local(&body.user_id) {
 		// Create and update our local copy of the user
 		if let Ok(response) = services
-			.sending
-			.send_federation_request(
+			.federation
+			.execute(
 				body.user_id.server_name(),
 				federation::query::get_profile_information::v1::Request {
 					user_id: body.user_id.clone(),

src/api/client/unversioned.rs

@@ -37,22 +37,25 @@ pub(crate) async fn get_supported_versions_route(
 			"v1.3".to_owned(),
 			"v1.4".to_owned(),
 			"v1.5".to_owned(),
-			"v1.11".to_owned(),
+			"v1.10".to_owned(), // relations recursion
+			"v1.11".to_owned(), // authenticated media
 		],
 		unstable_features: BTreeMap::from_iter([
 			("org.matrix.e2e_cross_signing".to_owned(), true),
 			("org.matrix.msc2285.stable".to_owned(), true), /* private read receipts (https://github.com/matrix-org/matrix-spec-proposals/pull/2285) */
+			("fi.mau.msc2659.stable".to_owned(), true), /* appservice ping https://github.com/matrix-org/matrix-spec-proposals/pull/2659) */
 			("uk.half-shot.msc2666.query_mutual_rooms".to_owned(), true), /* query mutual rooms (https://github.com/matrix-org/matrix-spec-proposals/pull/2666) */
 			("org.matrix.msc2836".to_owned(), true), /* threading/threads (https://github.com/matrix-org/matrix-spec-proposals/pull/2836) */
 			("org.matrix.msc2946".to_owned(), true), /* spaces/hierarchy summaries (https://github.com/matrix-org/matrix-spec-proposals/pull/2946) */
 			("org.matrix.msc3026.busy_presence".to_owned(), true), /* busy presence status (https://github.com/matrix-org/matrix-spec-proposals/pull/3026) */
-			("org.matrix.msc3827".to_owned(), true), /* filtering of /publicRooms by room type (https://github.com/matrix-org/matrix-spec-proposals/pull/3827) */
-			("org.matrix.msc3952_intentional_mentions".to_owned(), true), /* intentional mentions (https://github.com/matrix-org/matrix-spec-proposals/pull/3952) */
 			("org.matrix.msc3575".to_owned(), true), /* sliding sync (https://github.com/matrix-org/matrix-spec-proposals/pull/3575/files#r1588877046) */
+			("org.matrix.msc3814".to_owned(), true), /* dehydrated devices */
+			("org.matrix.msc3827".to_owned(), true), /* filtering of /publicRooms by room type (https://github.com/matrix-org/matrix-spec-proposals/pull/3827) */
 			("org.matrix.msc3916.stable".to_owned(), true), /* authenticated media (https://github.com/matrix-org/matrix-spec-proposals/pull/3916) */
-			("org.matrix.msc4180".to_owned(), true), /* stable flag for 3916 (https://github.com/matrix-org/matrix-spec-proposals/pull/4180) */
+			("org.matrix.msc3952_intentional_mentions".to_owned(), true), /* intentional mentions (https://github.com/matrix-org/matrix-spec-proposals/pull/3952) */
 			("uk.tcpip.msc4133".to_owned(), true), /* Extending User Profile API with Key:Value Pairs (https://github.com/matrix-org/matrix-spec-proposals/pull/4133) */
 			("us.cloke.msc4175".to_owned(), true), /* Profile field for user time zone (https://github.com/matrix-org/matrix-spec-proposals/pull/4175) */
+			("org.matrix.msc4180".to_owned(), true), /* stable flag for 3916 (https://github.com/matrix-org/matrix-spec-proposals/pull/4180) */
 			("org.matrix.simplified_msc3575".to_owned(), true), /* Simplified Sliding sync (https://github.com/matrix-org/matrix-spec-proposals/pull/4186) */
 		]),
 	};