diff --git a/.ci/scripts/schema_diff.py b/.ci/scripts/schema_diff.py new file mode 100755 index 0000000000..5dcc248f80 --- /dev/null +++ b/.ci/scripts/schema_diff.py @@ -0,0 +1,233 @@ +#!/usr/bin/env python3 +# +# Get a diff showing the change in the database schema. +# +# Usage: +# # Compare against develop (default): +# PGUSER=postgres PGPASSWORD=postgres scripts-dev/schema_diff.py +# +# # Compare against a specific branch/commit: +# PGUSER=postgres PGPASSWORD=postgres scripts-dev/schema_diff.py --base origin/release-v1.100 + +import argparse +import os +import subprocess +import sys +import tempfile +from pathlib import Path + +REPO_ROOT = Path(__file__).resolve().parent.parent.parent +SCHEMA_DIR = "synapse/storage/schema" +MAKE_FULL_SCHEMA_SCRIPT = REPO_ROOT / "scripts-dev" / "make_full_schema.sh" + + +def run_make_full_schema(output_dir: Path) -> None: + """Run make_full_schema.sh, piping the password via stdin.""" + pg_user = os.environ.get("PGUSER", "") + pg_password = os.environ.get("PGPASSWORD", "") + if not pg_user: + print("ERROR: PGUSER environment variable not set.", file=sys.stderr) + sys.exit(1) + if not pg_password: + print("ERROR: PGPASSWORD environment variable not set.", file=sys.stderr) + sys.exit(1) + + cmd: list[str] = [ + # Use faketime here for schema deltas that are wall-clock sensitive under SQLite + # We must only use faketime at this level because freezing the clock + # seems to cause `poetry install` to hang when recompiling our Rust module + "faketime", + "-f", + "2001-05-25 12:42:42", + "poetry", + "run", + str(MAKE_FULL_SCHEMA_SCRIPT), + "-p", + pg_user, + "-o", + str(output_dir), + "-c", + "-n", + "9999", + ] + + print(f"Running: {' '.join(cmd)}", file=sys.stderr) + + proc = subprocess.Popen( + cmd, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + cwd=REPO_ROOT, + text=True, + ) + stdout, _ = proc.communicate(input=pg_password + "\n") + # Forward script output to stderr so stdout stays clean for markdown + if stdout: + print(stdout, file=sys.stderr, end="") + if proc.returncode != 0: + print( + f"ERROR: make_full_schema.sh failed with exit code {proc.returncode}", + file=sys.stderr, + ) + sys.exit(proc.returncode) + + +def diff_schemas( + before_dir: Path, after_dir: Path, before_ref: str, after_ref: str +) -> str: + """Diff SQLite and Postgres full schemas, return a Markdown report.""" + parts: list[str] = [ + "## Schema Diff", + "", + "Please check that this looks as expected!", + "", + ] + + for db in ["common", "main", "state"]: + for engine in ["sqlite", "postgres"]: + filename = f"full.sql.{engine}" + + before_file = before_dir / db / "full_schemas" / "9999" / filename + + after_file = after_dir / db / "full_schemas" / "9999" / filename + + if not before_file.exists(): + raise RuntimeError(f"No before file found for {db = }, {engine = }") + if not after_file.exists(): + raise RuntimeError(f"No after file found for {db = }, {engine = }") + + result = subprocess.run( + ["diff", "-U", "10", str(before_file), str(after_file)], + capture_output=True, + text=True, + ) + + if result.returncode == 0: + parts.append(f"### {db} ({engine})\n\nUnchanged\n") + else: + parts.append(f"### {db} ({engine})\n\n```diff\n{result.stdout}\n```\n") + + return "\n".join(parts) + + +def main() -> None: + parser = argparse.ArgumentParser(description="Show database schema changes") + parser.add_argument( + "--base", + default="develop", + help="Base commit/branch to compare against (default: develop)", + ) + args = parser.parse_args() + + # Create temp output directory with before/after subdirectories + with tempfile.TemporaryDirectory(prefix="schema_diff_") as tmpdir: + after_dir = Path(tmpdir) / "after" + before_dir = Path(tmpdir) / "before" + after_dir.mkdir() + before_dir.mkdir() + + print("\n--- Running make_full_schema.sh (after) ---", file=sys.stderr) + run_make_full_schema(after_dir) + + # Checkout base and run make_full_schema.sh + print( + f"\n--- Checking out {args.base} and running make_full_schema.sh (before) ---", + file=sys.stderr, + ) + + # Save current ref so we can return to it without detaching. + # (Not useful in CI, but is useful for local development.) + # If we are on a named branch, use the branch name; otherwise use the SHA. + head_ref = subprocess.run( + ["git", "rev-parse", "--abbrev-ref", "HEAD"], + capture_output=True, + text=True, + cwd=REPO_ROOT, + check=True, + ).stdout.strip() + + before_sha = subprocess.run( + ["git", "rev-parse", args.base], + capture_output=True, + text=True, + cwd=REPO_ROOT, + check=True, + ).stdout.strip() + after_sha = subprocess.run( + ["git", "rev-parse", "HEAD"], + capture_output=True, + text=True, + cwd=REPO_ROOT, + check=True, + ).stdout.strip() + + # Check if working tree is dirty before potentially stashing + status = subprocess.run( + [ + "git", + "status", + # Machine-readable output for easy parsing + "--porcelain", + ], + capture_output=True, + text=True, + cwd=REPO_ROOT, + check=True, + ).stdout.strip() + + did_stash = False + if status: + print("Stashing local changes before checkout...", file=sys.stderr) + subprocess.run( + [ + "git", + "stash", + "push", + "--include-untracked", + "-m", + "schema_diff temporary stash", + ], + cwd=REPO_ROOT, + check=True, + ) + did_stash = True + + try: + subprocess.run(["git", "checkout", args.base], cwd=REPO_ROOT, check=True) + + # Refresh dependencies + print("Installing dependencies for base commit...", file=sys.stderr) + subprocess.run( + ["poetry", "install", "--extras", "postgres"], + cwd=REPO_ROOT, + check=True, + # Poetry install is noisy, so pipe its stdout to stderr + stdout=sys.stderr, + ) + + run_make_full_schema(before_dir) + finally: + print("Returning to HEAD...", file=sys.stderr) + subprocess.run( + [ + "git", + "checkout", + head_ref, + ], + cwd=REPO_ROOT, + check=True, + ) + if did_stash: + subprocess.run(["git", "stash", "pop"], cwd=REPO_ROOT, check=True) + print("✓ Restored stashed changes.", file=sys.stderr) + + # Diff + print("\n--- Diffing schemas ---", file=sys.stderr) + markdown = diff_schemas(before_dir, after_dir, before_sha, after_sha) + + print(markdown) + + +if __name__ == "__main__": + main() diff --git a/.github/workflows/complement_tests.yml b/.github/workflows/complement_tests.yml index a891802ac8..220ea57e1a 100644 --- a/.github/workflows/complement_tests.yml +++ b/.github/workflows/complement_tests.yml @@ -39,7 +39,7 @@ jobs: steps: - name: Checkout synapse codebase - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: path: synapse @@ -50,10 +50,10 @@ jobs: shell: bash - name: Install Rust - uses: dtolnay/rust-toolchain@e97e2d8cc328f1b50210efc529dca0028893a2d9 # master + uses: dtolnay/rust-toolchain@6c977a6ca4077a0ceb28ffbe03f59d46e9ac8772 # master with: toolchain: ${{ env.RUST_VERSION }} - - uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1 + - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 # We use `poetry` in `complement.sh` - uses: matrix-org/setup-python-poetry@5bbf6603c5c930615ec8a29f1b5d7d258d905aa4 # v2.0.0 @@ -65,7 +65,7 @@ jobs: - name: Prepare Complement's Prerequisites run: synapse/.ci/scripts/setup_complement_prerequisites.sh - - uses: actions/setup-go@4a3601121dd01d1626a1e23e37211e3254c1c06c # v6.4.0 + - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 with: cache-dependency-path: complement/go.sum go-version-file: complement/go.mod diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 39ddb61918..90cfd9becf 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -28,10 +28,10 @@ jobs: steps: - name: Set up Docker Buildx id: buildx - uses: docker/setup-buildx-action@d7f5e7f509e45cec5c76c4d5afdd7de93d0b3df5 # v4.1.0 + uses: docker/setup-buildx-action@37fe631027851001ddb9b187196cc803df7f5f0e # v4.3.0 - name: Checkout repository - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - name: Extract version from pyproject.toml # Note: explicitly requesting bash will mean bash is invoked with `-eo pipefail`, see @@ -41,20 +41,20 @@ jobs: echo "SYNAPSE_VERSION=$(grep "^version" pyproject.toml | sed -E 's/version\s*=\s*["]([^"]*)["]/\1/')" >> $GITHUB_ENV - name: Log in to DockerHub - uses: docker/login-action@650006c6eb7dba73a995cc03b0b2d7f5ca915bee # v4.2.0 + uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4.6.0 with: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} - name: Log in to GHCR - uses: docker/login-action@650006c6eb7dba73a995cc03b0b2d7f5ca915bee # v4.2.0 + uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4.6.0 with: registry: ghcr.io username: ${{ github.repository_owner }} password: ${{ secrets.GITHUB_TOKEN }} - name: Tailscale - uses: tailscale/github-action@306e68a486fd2350f2bfc3b19fcd143891a4a2d8 # v4.1.2 + uses: tailscale/github-action@780049a30b6ff5c378a9e7b389d15ece7a204888 # v4.1.3 with: oauth-client-id: ${{ secrets.TS_OAUTH_CLIENT_ID }} audience: ${{ secrets.TS_AUDIENCE }} @@ -79,7 +79,7 @@ jobs: services/backend-repositories/secret/data/oci.element.io password | OCI_PASSWORD ; - name: Login to Element OCI Registry - uses: docker/login-action@650006c6eb7dba73a995cc03b0b2d7f5ca915bee # v4.2.0 + uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4.6.0 with: registry: oci-push.vpn.infra.element.io username: ${{ steps.import-secrets.outputs.OCI_USERNAME }} @@ -87,7 +87,7 @@ jobs: - name: Build and push by digest id: build - uses: docker/build-push-action@f9f3042f7e2789586610d6e8b85c8f03e5195baf # v7.2.0 + uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7.3.0 with: push: true labels: | @@ -136,14 +136,14 @@ jobs: merge-multiple: true - name: Log in to DockerHub - uses: docker/login-action@650006c6eb7dba73a995cc03b0b2d7f5ca915bee # v4.2.0 + uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4.6.0 if: ${{ startsWith(matrix.repository, 'docker.io') }} with: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} - name: Log in to GHCR - uses: docker/login-action@650006c6eb7dba73a995cc03b0b2d7f5ca915bee # v4.2.0 + uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4.6.0 if: ${{ startsWith(matrix.repository, 'ghcr.io') }} with: registry: ghcr.io @@ -151,7 +151,7 @@ jobs: password: ${{ secrets.GITHUB_TOKEN }} - name: Tailscale - uses: tailscale/github-action@306e68a486fd2350f2bfc3b19fcd143891a4a2d8 # v4.1.2 + uses: tailscale/github-action@780049a30b6ff5c378a9e7b389d15ece7a204888 # v4.1.3 with: oauth-client-id: ${{ secrets.TS_OAUTH_CLIENT_ID }} audience: ${{ secrets.TS_AUDIENCE }} @@ -176,20 +176,20 @@ jobs: services/backend-repositories/secret/data/oci.element.io password | OCI_PASSWORD ; - name: Login to Element OCI Registry - uses: docker/login-action@650006c6eb7dba73a995cc03b0b2d7f5ca915bee # v4.2.0 + uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4.6.0 with: registry: oci-push.vpn.infra.element.io username: ${{ steps.import-secrets.outputs.OCI_USERNAME }} password: ${{ steps.import-secrets.outputs.OCI_PASSWORD }} - name: Set up Docker Buildx - uses: docker/setup-buildx-action@d7f5e7f509e45cec5c76c4d5afdd7de93d0b3df5 # v4.1.0 + uses: docker/setup-buildx-action@37fe631027851001ddb9b187196cc803df7f5f0e # v4.3.0 - name: Install Cosign uses: sigstore/cosign-installer@6f9f17788090df1f26f669e9d70d6ae9567deba6 # v4.1.2 - name: Calculate docker image tag - uses: docker/metadata-action@80c7e94dd9b9319bd5eb7a0e0fe9291e23a2a2e9 # v6.1.0 + uses: docker/metadata-action@dc802804100637a589fabce1cb79ff13a1411302 # v6.2.0 with: images: ${{ matrix.repository }} flavor: | diff --git a/.github/workflows/docs-pr.yaml b/.github/workflows/docs-pr.yaml index e1a5b7be89..6e5428dee0 100644 --- a/.github/workflows/docs-pr.yaml +++ b/.github/workflows/docs-pr.yaml @@ -13,7 +13,7 @@ jobs: name: GitHub Pages runs-on: ubuntu-latest steps: - - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: # Fetch all history so that the schema_versions script works. fetch-depth: 0 @@ -24,7 +24,7 @@ jobs: mdbook-version: '0.5.2' - name: Setup python - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 with: python-version: "3.x" @@ -50,7 +50,7 @@ jobs: name: Check links in documentation runs-on: ubuntu-latest steps: - - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - name: Setup mdbook uses: peaceiris/actions-mdbook@ee69d230fe19748b7abf22df32acaa93833fad08 # v2.0.0 diff --git a/.github/workflows/docs.yaml b/.github/workflows/docs.yaml index 7236bf99d9..0d1138f988 100644 --- a/.github/workflows/docs.yaml +++ b/.github/workflows/docs.yaml @@ -50,7 +50,7 @@ jobs: needs: - pre steps: - - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: # Fetch all history so that the schema_versions script works. fetch-depth: 0 @@ -64,7 +64,7 @@ jobs: run: echo 'window.SYNAPSE_VERSION = "${{ needs.pre.outputs.branch-version }}";' > ./docs/website_files/version.js - name: Setup python - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 with: python-version: "3.x" diff --git a/.github/workflows/fix_lint.yaml b/.github/workflows/fix_lint.yaml index e0817698f4..31ac11bfcd 100644 --- a/.github/workflows/fix_lint.yaml +++ b/.github/workflows/fix_lint.yaml @@ -20,14 +20,14 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - name: Install Rust - uses: dtolnay/rust-toolchain@e97e2d8cc328f1b50210efc529dca0028893a2d9 # master + uses: dtolnay/rust-toolchain@6c977a6ca4077a0ceb28ffbe03f59d46e9ac8772 # master with: toolchain: ${{ env.RUST_VERSION }} components: clippy, rustfmt - - uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1 + - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 - name: Setup Poetry uses: matrix-org/setup-python-poetry@5bbf6603c5c930615ec8a29f1b5d7d258d905aa4 # v2.0.0 @@ -49,6 +49,6 @@ jobs: - run: cargo fmt continue-on-error: true - - uses: stefanzweifel/git-auto-commit-action@04702edda442b2e678b25b537cec683a1493fcb9 # v7.1.0 + - uses: stefanzweifel/git-auto-commit-action@4a55954c782fc1ea30b9056cd3e7a2b40ca8887d # v7.2.0 with: commit_message: "Attempt to fix linting" diff --git a/.github/workflows/latest_deps.yml b/.github/workflows/latest_deps.yml index 815593ffcd..fe324e3dbb 100644 --- a/.github/workflows/latest_deps.yml +++ b/.github/workflows/latest_deps.yml @@ -42,12 +42,12 @@ jobs: if: needs.check_repo.outputs.should_run_workflow == 'true' runs-on: ubuntu-latest steps: - - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - name: Install Rust - uses: dtolnay/rust-toolchain@e97e2d8cc328f1b50210efc529dca0028893a2d9 # master + uses: dtolnay/rust-toolchain@6c977a6ca4077a0ceb28ffbe03f59d46e9ac8772 # master with: toolchain: ${{ env.RUST_VERSION }} - - uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1 + - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 # The dev dependencies aren't exposed in the wheel metadata (at least with current # poetry-core versions), so we install with poetry. @@ -77,13 +77,13 @@ jobs: postgres-version: "14" steps: - - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - name: Install Rust - uses: dtolnay/rust-toolchain@e97e2d8cc328f1b50210efc529dca0028893a2d9 # master + uses: dtolnay/rust-toolchain@6c977a6ca4077a0ceb28ffbe03f59d46e9ac8772 # master with: toolchain: ${{ env.RUST_VERSION }} - - uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1 + - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 - run: sudo apt-get -qq install xmlsec1 - name: Set up PostgreSQL ${{ matrix.postgres-version }} @@ -93,7 +93,7 @@ jobs: -e POSTGRES_PASSWORD=postgres \ -e POSTGRES_INITDB_ARGS="--lc-collate C --lc-ctype C --encoding UTF8" \ postgres:${{ matrix.postgres-version }} - - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 + - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 with: python-version: "3.x" - run: pip install .[all,test] @@ -151,13 +151,13 @@ jobs: BLACKLIST: ${{ matrix.workers && 'synapse-blacklist-with-workers' }} steps: - - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - name: Install Rust - uses: dtolnay/rust-toolchain@e97e2d8cc328f1b50210efc529dca0028893a2d9 # master + uses: dtolnay/rust-toolchain@6c977a6ca4077a0ceb28ffbe03f59d46e9ac8772 # master with: toolchain: ${{ env.RUST_VERSION }} - - uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1 + - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 - name: Ensure sytest runs `pip install` # Delete the lockfile so sytest will `pip install` rather than `poetry install` @@ -201,7 +201,7 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - uses: JasonEtco/create-an-issue@1b14a70e4d8dc185e5cc76d3bec9eab20257b2c5 # v2.9.2 env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/poetry_lockfile.yaml b/.github/workflows/poetry_lockfile.yaml index 06545bd18a..6d87b87acf 100644 --- a/.github/workflows/poetry_lockfile.yaml +++ b/.github/workflows/poetry_lockfile.yaml @@ -16,8 +16,8 @@ jobs: name: "Check locked dependencies have sdists" runs-on: ubuntu-latest steps: - - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 - - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 with: python-version: '3.x' - run: pip install tomli diff --git a/.github/workflows/push_complement_image.yml b/.github/workflows/push_complement_image.yml index 6f4c966cdc..6fc7be4329 100644 --- a/.github/workflows/push_complement_image.yml +++ b/.github/workflows/push_complement_image.yml @@ -33,17 +33,17 @@ jobs: packages: write steps: - name: Checkout specific branch (debug build) - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 if: github.event_name == 'workflow_dispatch' with: ref: ${{ inputs.branch }} - name: Checkout clean copy of develop (scheduled build) - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 if: github.event_name == 'schedule' with: ref: develop - name: Checkout clean copy of master (on-push) - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 if: github.event_name == 'push' with: ref: master @@ -52,14 +52,14 @@ jobs: with: poetry-version: "2.4.1" - name: Login to registry - uses: docker/login-action@650006c6eb7dba73a995cc03b0b2d7f5ca915bee # v4.2.0 + uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4.6.0 with: registry: ghcr.io username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - name: Work out labels for complement image id: meta - uses: docker/metadata-action@80c7e94dd9b9319bd5eb7a0e0fe9291e23a2a2e9 # v6.1.0 + uses: docker/metadata-action@dc802804100637a589fabce1cb79ff13a1411302 # v6.2.0 with: images: ghcr.io/${{ github.repository }}/complement-synapse tags: | diff --git a/.github/workflows/release-artifacts.yml b/.github/workflows/release-artifacts.yml index c6b9f60baf..75ceb0f533 100644 --- a/.github/workflows/release-artifacts.yml +++ b/.github/workflows/release-artifacts.yml @@ -27,8 +27,8 @@ jobs: name: "Calculate list of debian distros" runs-on: ubuntu-latest steps: - - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 - - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 with: python-version: "3.x" - id: set-distros @@ -61,16 +61,16 @@ jobs: steps: - name: Checkout - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: path: src - name: Set up Docker Buildx id: buildx - uses: docker/setup-buildx-action@d7f5e7f509e45cec5c76c4d5afdd7de93d0b3df5 # v4.1.0 + uses: docker/setup-buildx-action@37fe631027851001ddb9b187196cc803df7f5f0e # v4.3.0 - name: Set up docker layer caching - uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 + uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 with: path: /tmp/.buildx-cache key: ${{ runner.os }}-buildx-${{ github.sha }} @@ -78,7 +78,7 @@ jobs: ${{ runner.os }}-buildx- - name: Set up python - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 with: python-version: "3.x" @@ -129,9 +129,9 @@ jobs: os: "ubuntu-24.04-arm" steps: - - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 + - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 with: # setup-python@v4 doesn't impose a default python version. Need to use 3.x # here, because `python` on osx points to Python 2.7. @@ -167,8 +167,8 @@ jobs: if: ${{ !startsWith(github.ref, 'refs/pull/') }} steps: - - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 - - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 with: python-version: "3.10" diff --git a/.github/workflows/schema.yaml b/.github/workflows/schema.yaml index e36114d354..d9105be981 100644 --- a/.github/workflows/schema.yaml +++ b/.github/workflows/schema.yaml @@ -14,8 +14,8 @@ jobs: name: Ensure Synapse config schema is valid runs-on: ubuntu-latest steps: - - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 - - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 with: python-version: "3.x" - name: Install check-jsonschema @@ -40,8 +40,8 @@ jobs: name: Ensure generated documentation is up-to-date runs-on: ubuntu-latest steps: - - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 - - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 with: python-version: "3.x" - name: Install PyYAML diff --git a/.github/workflows/schema_diff.yml b/.github/workflows/schema_diff.yml new file mode 100644 index 0000000000..f0d148b071 --- /dev/null +++ b/.github/workflows/schema_diff.yml @@ -0,0 +1,103 @@ +name: Schema Diff + +on: + pull_request: + paths: + - synapse/storage/schema/*/delta/** + - synapse/storage/schema/*/full_schemas/** + - .github/workflows/schema_diff.yml + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +permissions: + contents: read + +jobs: + # Posts a GitHub PR comment that shows what the effective change to the schema is. + # Provides an excuse to run the `make_full_schema.sh` script in CI (so we keep it working) + # and can act as a review aid for schema changes, letting you easily see the diff of the + # end result, even when background updates or complex schema deltas are present. + show-schema-diff: + name: Show schema diff + runs-on: ubuntu-latest + + permissions: + pull-requests: write + + steps: + - name: Start postgres with a faked clock + background: true + id: postgres + # Use faketime here for schema deltas that are wall-clock sensitive under Postgres + # For SQLite, faketime is used when invoking `make_full_schema.sh` within the script + run: | + # Build a docker image with faketime + mkdir /tmp/postgres-faketime + cat > /tmp/postgres-faketime/Dockerfile <<'EOF' + FROM postgres:14-alpine + RUN apk add --no-cache libfaketime + + # It seems like it could be harmful to fake the monotonic timer + # as it might prevent deadlock detection, etc. + # But not sure, just doing out of precaution. + ENV FAKETIME_DONT_FAKE_MONOTONIC=1 + ENTRYPOINT ["faketime", "-f", "2001-05-25 12:42:42", "docker-entrypoint.sh"] + CMD ["postgres"] + EOF + docker build -t localhost/postgres-faketime /tmp/postgres-faketime + + # Run it in the background + docker run -d --name postgres -p 5432:5432 \ + -e POSTGRES_PASSWORD=postgres \ + -e POSTGRES_INITDB_ARGS="--lc-collate C --lc-ctype C --encoding UTF8" \ + --health-cmd pg_isready --health-interval 10s \ + --health-timeout 5s --health-retries 5 \ + localhost/postgres-faketime + + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + fetch-depth: 0 + + - name: Install PostgreSQL client and faketime + run: sudo apt-get -qq install postgresql-client faketime + + - uses: matrix-org/setup-python-poetry@5bbf6603c5c930615ec8a29f1b5d7d258d905aa4 # v2.0.0 + with: + poetry-version: "2.4.1" + extras: "postgres" + python-version: "3.x" + + - name: Wait for Postgres to be up + run: | + until [ "$(docker inspect -f '{{.State.Health.Status}}' postgres)" = healthy ]; do sleep 2; done + + - name: Generate schema diff + id: schema_diff + env: + PGHOST: localhost + PGUSER: postgres + PGPASSWORD: postgres + run: | + poetry run python .ci/scripts/schema_diff.py \ + --base origin/develop \ + > "${{ runner.temp }}/schema_diff.md" + + - name: Stop postgres + cancel: postgres + + # If the generation step failed, write an error message so the sticky + # comment step still has a file to read. + - name: Ensure output file exists on failure + if: always() && steps.schema_diff.outcome == 'failure' + run: | + echo "⚠️ Schema diff generation failed. See job logs for details." \ + > "${{ runner.temp }}/schema_diff.md" + + - name: Post sticky PR comment + uses: marocchino/sticky-pull-request-comment@3d7b8546315c63df45a03981d50a43ec19237f80 # v3 + if: always() + with: + header: schema-diff + path: ${{ runner.temp }}/schema_diff.md diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 45fa2b8cad..90b7f9021a 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -37,7 +37,7 @@ jobs: linting_readme: ${{ !startsWith(github.ref, 'refs/pull/') || steps.filter.outputs.linting_readme }} golangci: ${{ !startsWith(github.ref, 'refs/pull/') || steps.filter.outputs.golangci }} steps: - - uses: dorny/paths-filter@fbd0ab8f3e69293af611ebaee6363fc25e6d187d # v4.0.1 + - uses: dorny/paths-filter@ceb8a2b8f2d89434be7ff52d3de7ec3738c5cc9d # v4.0.3 id: filter # We only check on PRs if: startsWith(github.ref, 'refs/pull/') @@ -106,12 +106,12 @@ jobs: if: ${{ needs.changes.outputs.linting == 'true' }} steps: - - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - name: Install Rust - uses: dtolnay/rust-toolchain@e97e2d8cc328f1b50210efc529dca0028893a2d9 # master + uses: dtolnay/rust-toolchain@6c977a6ca4077a0ceb28ffbe03f59d46e9ac8772 # master with: toolchain: ${{ env.RUST_VERSION }} - - uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1 + - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 - uses: matrix-org/setup-python-poetry@5bbf6603c5c930615ec8a29f1b5d7d258d905aa4 # v2.0.0 with: python-version: "3.x" @@ -126,8 +126,8 @@ jobs: if: ${{ needs.changes.outputs.linting == 'true' }} steps: - - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 - - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 with: python-version: "3.x" - run: "pip install 'click==8.1.1' 'GitPython>=3.1.20' 'sqlglot>=28.0.0'" @@ -136,8 +136,8 @@ jobs: check-lockfile: runs-on: ubuntu-latest steps: - - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 - - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 with: python-version: "3.x" - run: .ci/scripts/check_lockfile.py @@ -149,7 +149,7 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - name: Setup Poetry uses: matrix-org/setup-python-poetry@5bbf6603c5c930615ec8a29f1b5d7d258d905aa4 # v2.0.0 @@ -171,13 +171,13 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - name: Install Rust - uses: dtolnay/rust-toolchain@e97e2d8cc328f1b50210efc529dca0028893a2d9 # master + uses: dtolnay/rust-toolchain@6c977a6ca4077a0ceb28ffbe03f59d46e9ac8772 # master with: toolchain: ${{ env.RUST_VERSION }} - - uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1 + - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 - name: Setup Poetry uses: matrix-org/setup-python-poetry@5bbf6603c5c930615ec8a29f1b5d7d258d905aa4 # v2.0.0 @@ -194,7 +194,7 @@ jobs: # Cribbed from # https://github.com/AustinScola/mypy-cache-github-action/blob/85ea4f2972abed39b33bd02c36e341b28ca59213/src/restore.ts#L10-L17 - name: Restore/persist mypy's cache - uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 + uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 with: path: | .mypy_cache @@ -207,7 +207,7 @@ jobs: lint-crlf: runs-on: ubuntu-latest steps: - - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - name: Check line endings run: scripts-dev/check_line_terminators.sh @@ -216,11 +216,11 @@ jobs: if: ${{ github.event_name == 'pull_request' && (github.base_ref == 'develop' || contains(github.base_ref, 'release-')) && github.event.pull_request.user.login != 'dependabot[bot]' }} runs-on: ubuntu-latest steps: - - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: ref: ${{ github.event.pull_request.head.sha }} fetch-depth: 0 - - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 + - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 with: python-version: "3.x" - run: "pip install 'towncrier>=18.6.0rc1'" @@ -234,14 +234,14 @@ jobs: if: ${{ needs.changes.outputs.rust == 'true' }} steps: - - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - name: Install Rust - uses: dtolnay/rust-toolchain@e97e2d8cc328f1b50210efc529dca0028893a2d9 # master + uses: dtolnay/rust-toolchain@6c977a6ca4077a0ceb28ffbe03f59d46e9ac8772 # master with: components: clippy toolchain: ${{ env.RUST_VERSION }} - - uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1 + - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 - run: cargo clippy -- -D warnings @@ -253,14 +253,14 @@ jobs: if: ${{ needs.changes.outputs.rust == 'true' }} steps: - - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - name: Install Rust - uses: dtolnay/rust-toolchain@e97e2d8cc328f1b50210efc529dca0028893a2d9 # master + uses: dtolnay/rust-toolchain@6c977a6ca4077a0ceb28ffbe03f59d46e9ac8772 # master with: toolchain: ${{ env.RUST_NIGHTLY_VERSION }} components: clippy - - uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1 + - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 - run: cargo clippy --all-features -- -D warnings @@ -271,13 +271,13 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - name: Install Rust - uses: dtolnay/rust-toolchain@e97e2d8cc328f1b50210efc529dca0028893a2d9 # master + uses: dtolnay/rust-toolchain@6c977a6ca4077a0ceb28ffbe03f59d46e9ac8772 # master with: toolchain: ${{ env.RUST_VERSION }} - - uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1 + - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 - name: Setup Poetry uses: matrix-org/setup-python-poetry@5bbf6603c5c930615ec8a29f1b5d7d258d905aa4 # v2.0.0 @@ -307,16 +307,16 @@ jobs: if: ${{ needs.changes.outputs.rust == 'true' }} steps: - - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - name: Install Rust - uses: dtolnay/rust-toolchain@e97e2d8cc328f1b50210efc529dca0028893a2d9 # master + uses: dtolnay/rust-toolchain@6c977a6ca4077a0ceb28ffbe03f59d46e9ac8772 # master with: # We use nightly so that we can use some unstable options that we use in # `.rustfmt.toml`. toolchain: ${{ env.RUST_NIGHTLY_VERSION }} components: rustfmt - - uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1 + - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 - run: cargo fmt --check @@ -326,9 +326,9 @@ jobs: if: ${{ needs.changes.outputs.golangci == 'true' }} steps: - - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - - uses: actions/setup-go@4a3601121dd01d1626a1e23e37211e3254c1c06c # v6.4.0 + - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 with: cache-dependency-path: complement/go.sum go-version-file: complement/go.mod @@ -344,8 +344,8 @@ jobs: needs: changes if: ${{ needs.changes.outputs.linting_readme == 'true' }} steps: - - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 - - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 with: python-version: "3.x" - run: "pip install rstcheck" @@ -393,8 +393,8 @@ jobs: needs: linting-done runs-on: ubuntu-latest steps: - - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 - - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 with: python-version: "3.x" - id: get-matrix @@ -414,7 +414,7 @@ jobs: job: ${{ fromJson(needs.calculate-test-jobs.outputs.trial_test_matrix) }} steps: - - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - run: sudo apt-get -qq install xmlsec1 - name: Set up PostgreSQL ${{ matrix.job.postgres-version }} if: ${{ matrix.job.postgres-version }} @@ -429,10 +429,10 @@ jobs: postgres:${{ matrix.job.postgres-version }} - name: Install Rust - uses: dtolnay/rust-toolchain@e97e2d8cc328f1b50210efc529dca0028893a2d9 # master + uses: dtolnay/rust-toolchain@6c977a6ca4077a0ceb28ffbe03f59d46e9ac8772 # master with: toolchain: ${{ env.RUST_VERSION }} - - uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1 + - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 - uses: matrix-org/setup-python-poetry@5bbf6603c5c930615ec8a29f1b5d7d258d905aa4 # v2.0.0 with: @@ -470,13 +470,13 @@ jobs: - changes runs-on: ubuntu-22.04 steps: - - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - name: Install Rust - uses: dtolnay/rust-toolchain@e97e2d8cc328f1b50210efc529dca0028893a2d9 # master + uses: dtolnay/rust-toolchain@6c977a6ca4077a0ceb28ffbe03f59d46e9ac8772 # master with: toolchain: ${{ env.RUST_VERSION }} - - uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1 + - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 # There aren't wheels for some of the older deps, so we need to install # their build dependencies @@ -485,7 +485,7 @@ jobs: sudo apt-get -qq install build-essential libffi-dev python3-dev \ libxml2-dev libxslt-dev xmlsec1 zlib1g-dev libjpeg-dev libwebp-dev - - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 + - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 with: python-version: "3.10" @@ -533,7 +533,7 @@ jobs: extras: ["all"] steps: - - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 # Install libs necessary for PyPy to build binary wheels for dependencies - run: sudo apt-get -qq install xmlsec1 libxml2-dev libxslt-dev - uses: matrix-org/setup-python-poetry@5bbf6603c5c930615ec8a29f1b5d7d258d905aa4 # v2.0.0 @@ -583,15 +583,15 @@ jobs: job: ${{ fromJson(needs.calculate-test-jobs.outputs.sytest_test_matrix) }} steps: - - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - name: Prepare test blacklist run: cat sytest-blacklist .ci/worker-blacklist > synapse-blacklist-with-workers - name: Install Rust - uses: dtolnay/rust-toolchain@e97e2d8cc328f1b50210efc529dca0028893a2d9 # master + uses: dtolnay/rust-toolchain@6c977a6ca4077a0ceb28ffbe03f59d46e9ac8772 # master with: toolchain: ${{ env.RUST_VERSION }} - - uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1 + - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 - name: Run SyTest run: /bootstrap.sh synapse @@ -630,7 +630,7 @@ jobs: --health-retries 5 steps: - - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - run: sudo apt-get -qq install xmlsec1 postgresql-client - uses: matrix-org/setup-python-poetry@5bbf6603c5c930615ec8a29f1b5d7d258d905aa4 # v2.0.0 with: @@ -673,7 +673,7 @@ jobs: --health-retries 5 steps: - - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - name: Add PostgreSQL apt repository # We need a version of pg_dump that can handle the version of # PostgreSQL being tested against. The Ubuntu package repository lags @@ -726,13 +726,13 @@ jobs: - changes steps: - - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - name: Install Rust - uses: dtolnay/rust-toolchain@e97e2d8cc328f1b50210efc529dca0028893a2d9 # master + uses: dtolnay/rust-toolchain@6c977a6ca4077a0ceb28ffbe03f59d46e9ac8772 # master with: toolchain: ${{ env.RUST_VERSION }} - - uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1 + - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 - run: cargo test @@ -747,13 +747,13 @@ jobs: - changes steps: - - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - name: Install Rust - uses: dtolnay/rust-toolchain@e97e2d8cc328f1b50210efc529dca0028893a2d9 # master + uses: dtolnay/rust-toolchain@6c977a6ca4077a0ceb28ffbe03f59d46e9ac8772 # master with: toolchain: ${{ env.RUST_NIGHTLY_VERSION }} - - uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1 + - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 - run: cargo bench --no-run diff --git a/.github/workflows/triage_labelled.yml b/.github/workflows/triage_labelled.yml index 85d7be7b34..f852cfcf11 100644 --- a/.github/workflows/triage_labelled.yml +++ b/.github/workflows/triage_labelled.yml @@ -22,7 +22,7 @@ jobs: # This field is case-sensitive. TARGET_STATUS: Needs info steps: - - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: # Only clone the script file we care about, instead of the whole repo. sparse-checkout: .ci/scripts/triage_labelled_issue.sh diff --git a/.github/workflows/twisted_trunk.yml b/.github/workflows/twisted_trunk.yml index 1b906f7f44..9836f743c7 100644 --- a/.github/workflows/twisted_trunk.yml +++ b/.github/workflows/twisted_trunk.yml @@ -42,13 +42,13 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - name: Install Rust - uses: dtolnay/rust-toolchain@e97e2d8cc328f1b50210efc529dca0028893a2d9 # master + uses: dtolnay/rust-toolchain@6c977a6ca4077a0ceb28ffbe03f59d46e9ac8772 # master with: toolchain: ${{ env.RUST_VERSION }} - - uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1 + - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 - uses: matrix-org/setup-python-poetry@5bbf6603c5c930615ec8a29f1b5d7d258d905aa4 # v2.0.0 with: @@ -69,14 +69,14 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - run: sudo apt-get -qq install xmlsec1 - name: Install Rust - uses: dtolnay/rust-toolchain@e97e2d8cc328f1b50210efc529dca0028893a2d9 # master + uses: dtolnay/rust-toolchain@6c977a6ca4077a0ceb28ffbe03f59d46e9ac8772 # master with: toolchain: ${{ env.RUST_VERSION }} - - uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1 + - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 - uses: matrix-org/setup-python-poetry@5bbf6603c5c930615ec8a29f1b5d7d258d905aa4 # v2.0.0 with: @@ -115,13 +115,13 @@ jobs: - ${{ github.workspace }}:/src steps: - - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - name: Install Rust - uses: dtolnay/rust-toolchain@e97e2d8cc328f1b50210efc529dca0028893a2d9 # master + uses: dtolnay/rust-toolchain@6c977a6ca4077a0ceb28ffbe03f59d46e9ac8772 # master with: toolchain: ${{ env.RUST_VERSION }} - - uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1 + - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 - name: Patch dependencies # Note: The poetry commands want to create a virtualenv in /src/.venv/, @@ -172,7 +172,7 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - uses: JasonEtco/create-an-issue@1b14a70e4d8dc185e5cc76d3bec9eab20257b2c5 # v2.9.2 env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/CHANGES.md b/CHANGES.md index 38213bd2ff..6fbfa1adfd 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,3 +1,236 @@ +# Synapse 1.160.0 (2026-09-02) + +No significant changes since 1.160.0rc2. + + + + +# Synapse 1.160.0rc2 (2026-08-31) + +## Bugfixes + +- Fix sending custom profile field removals to legacy sync clients when the field is deleted using the profile field delete endpoint. ([\#20147](https://github.com/element-hq/synapse/issues/20147)) + + + +# Synapse 1.160.0rc1 (2026-08-25) + +## Features + +- Add experimental support for [MSC4502](https://github.com/matrix-org/matrix-spec-proposals/pull/4502): Targeted and unrestricted room member queries. ([\#19974](https://github.com/element-hq/synapse/issues/19974)) +- Add optional support for [MSC4262: Profile Updates for Sliding Sync](https://github.com/matrix-org/matrix-spec-proposals/pull/4262). + Currently defaults to disabled, and is limited to local users only for the sync results. ([\#20003](https://github.com/element-hq/synapse/issues/20003)) +- Allow specifying multiple `action_name` and `status` query parameters when listing scheduled tasks via the admin API. ([\#20067](https://github.com/element-hq/synapse/issues/20067)) + +## Bugfixes + +- Fix a bug where stream positions (presence, to-device message, etc.) could stop being sent to clients if a request was cancelled while a write was allocating a stream ID. Contributed by @FrenchGithubUser @Famedly. ([\#20090](https://github.com/element-hq/synapse/issues/20090)) +- Thumbnail WebP images that use transparency as PNG rather than JPEG, to preserve transparency during thumbnailing. Contributed by @catfromplan9. ([\#20094](https://github.com/element-hq/synapse/issues/20094)) +- Fix sync stream not being woken up when a user updates a profile field without belonging to any rooms. ([\#20135](https://github.com/element-hq/synapse/issues/20135)) + +## Improved Documentation + +- Document lighttpd reverse proxy configuration example. Contributed by JaxLUG from the Jacksonville Linux Users Group Inc.. ([\#19875](https://github.com/element-hq/synapse/issues/19875)) +- Fix the documentation on the `federation_domain_whitelist` config option. ([\#20089](https://github.com/element-hq/synapse/issues/20089)) + +## Internal Changes + +- Update release script to check more often for actions being completed so you don't have to wait around as much. ([\#20093](https://github.com/element-hq/synapse/issues/20093)) +- Speed up the conversion of device list changes into outbound federation pokes, and add a metric for how far behind the conversion is. ([\#20098](https://github.com/element-hq/synapse/issues/20098)) +- Fix the schema diff CI not using `faketime` for SQLite. ([\#20099](https://github.com/element-hq/synapse/issues/20099)) +- Fix the schema diff CI breaking when the Rust module was changed. ([\#20117](https://github.com/element-hq/synapse/issues/20117), [\#20129](https://github.com/element-hq/synapse/issues/20129)) +- Reduce database CPU usage when marking device list changes as sent over federation. ([\#20120](https://github.com/element-hq/synapse/issues/20120)) +- Fix cache `__len__` of Sliding Sync `PerConnectionState` ignoring account data entries. ([\#20124](https://github.com/element-hq/synapse/issues/20124)) +- Update Synapse repo link in inconsistent stream error. ([\#20128](https://github.com/element-hq/synapse/issues/20128)) +- Update rustls-webpki to address [GHSA-82j2-j2ch-gfr8](https://github.com/advisories/GHSA-82j2-j2ch-gfr8). ([\#20131](https://github.com/element-hq/synapse/issues/20131)) +- Update pyo3 to address [GHSA-36hh-v3qg-5jq4](https://github.com/advisories/GHSA-36hh-v3qg-5jq4) and [GHSA-chgr-c6px-7xpp](https://github.com/advisories/GHSA-chgr-c6px-7xpp). ([\#20131](https://github.com/element-hq/synapse/issues/20131)) + + + + +# Synapse 1.159.0 (2026-08-18) + +No significant changes since 1.159.0rc1. + + +# Synapse 1.159.0rc1 (2026-08-11) + +Administrators using the Debian/Ubuntu packages from `packages.matrix.org`, please check +[the relevant section in the upgrade notes](https://github.com/element-hq/synapse/blob/release-v1.159/docs/upgrade.md#upgrading-to-v11590) +as we have recently updated the expiry date on the repository's GPG signing key. The old version of the key will expire on `2027-03-15`. + +## Features + +- Add optional support for [MSC4429: Profile Updates for Legacy Sync](https://github.com/matrix-org/matrix-spec-proposals/pull/4429). + Currently defaults to not enabled, and is limited to local users only for the sync results. ([\#19556](https://github.com/element-hq/synapse/issues/19556)) + +## Bugfixes + +- Fix thumbnail generation failing for MPO images. Animations that cannot be decoded now fall back to a static thumbnail. ([\#20025](https://github.com/element-hq/synapse/issues/20025)) +- Fix the `quarantined_media` replication stream never being sent when the configured `quarantined_media_changes` stream writer is a worker. Introduced in v1.152.0. ([\#20085](https://github.com/element-hq/synapse/issues/20085)) + +## Updates to the Docker image + +- Run with `PYTHONUNBUFFERED=1` to ensure that we can always see log output when things go wrong. ([\#20075](https://github.com/element-hq/synapse/issues/20075)) + +## Improved Documentation + +- Correct the documentation for the `on_media_upload_limit_exceeded` module callback with regards to where it is called from. ([\#20018](https://github.com/element-hq/synapse/issues/20018)) +- Add upgrade notes to point out updated Debian package signing key. ([\#20066](https://github.com/element-hq/synapse/issues/20066)) +- Update stream cheatsheet docs to re-link `synapse/config/workers.py` which has more references. ([\#20086](https://github.com/element-hq/synapse/issues/20086)) + +## Internal Changes + +- Fix tests that use `homeserver_to_use=GenericWorkerServer` not being able to be run standalone. ([\#20017](https://github.com/element-hq/synapse/issues/20017)) +- Fix `RemoteJoinHelper` test helper to handle room version "12" rooms. Contributed by @famedly @jason-famedly. ([\#20021](https://github.com/element-hq/synapse/issues/20021)) +- Fix release script announcement to link to correct release branch of changelog. ([\#20023](https://github.com/element-hq/synapse/issues/20023)) +- Dust off `make_full_schema` and add CI using it to show schema diffs. ([\#20027](https://github.com/element-hq/synapse/issues/20027)) +- Remove broken `DROP` statements for SQLite in `make_full_schema` script. ([\#20028](https://github.com/element-hq/synapse/issues/20028)) +- Document how to capture a JSON snapshot of a Grafana dashboard to aid in debugging. ([\#20048](https://github.com/element-hq/synapse/issues/20048)) +- Routinely purge old cancelled tasks from the database. ([\#20068](https://github.com/element-hq/synapse/issues/20068)) +- Introduce an `RdataSafeValue` type and correct some minor type annotation mistakes. ([\#20071](https://github.com/element-hq/synapse/issues/20071)) +- Set `idle_in_transaction_session_timeout` (default 30 minutes) on new PostgreSQL connections, so that wedged connections don't hold locks or block vacuum indefinitely. ([\#20077](https://github.com/element-hq/synapse/issues/20077)) + + + + +# Synapse 1.158.0 (2026-08-04) + +## Deprecations and Removals + +- Remove package build targets for Ubuntu 25.10 'Questing Quokka' (end-of-life 2026-07-01). ([\#20039](https://github.com/element-hq/synapse/issues/20039)) + +## Internal Changes + +- Add package build targets for Ubuntu 26.04 'Resolute Raccoon'. ([\#20039](https://github.com/element-hq/synapse/issues/20039)) + + + + +# Synapse 1.158.0rc1 (2026-07-30) + +## Features + +- Change default room version to 11, implementing [MSC4239](https://github.com/matrix-org/matrix-spec-proposals/pull/4239) as part of Matrix v1.14. ([\#18680](https://github.com/element-hq/synapse/issues/18680)) +- Add animation support to the media thumbnailer, gated behind the `animated` query parameter on the thumbnail endpoint (defaults to off). ([\#18831](https://github.com/element-hq/synapse/issues/18831)) +- Return `M_USER_LIMIT_EXCEEDED` error code for media upload limits from [MSC4335](https://github.com/matrix-org/matrix-spec-proposals/pull/4335). ([\#18876](https://github.com/element-hq/synapse/issues/18876)) +- Add Synapse Module API hook that notifies modules when events are delivered over federation (`register_federation_callbacks(...)`). ([\#20019](https://github.com/element-hq/synapse/issues/20019)) + +## Bugfixes + +- Fix third-party (3pid) invites over federation failing intermittently in version 12 rooms, whose room IDs no longer encode a server name. ([#19898](https://github.com/element-hq/synapse/issues/19898)) +- Fix `/createRoom` intermittently failing with a 500 error in version 12 rooms when the same user creates several rooms at once, due to colliding room IDs. ([\#19898](https://github.com/element-hq/synapse/issues/19898)) +- Fix server key cache invalidations being silently dropped on workers. ([\#19966](https://github.com/element-hq/synapse/issues/19966)) +- Fix `HomeServer.shutdown()` not being able to cleanly shutdown the homeserver (caused by Rust code referencing Python `DatabasePool`). ([\#20009](https://github.com/element-hq/synapse/issues/20009)) + +## Improved Documentation + +- Clarify the usage of the `guests` parameter when using the [List Accounts (V2) admin API](https://element-hq.github.io/synapse/develop/admin_api/user_admin_api.html#list-accounts-v2) with the Matrix Authentication Service integration enabled. ([\#19963](https://github.com/element-hq/synapse/issues/19963)) + +## Internal Changes + +- Update release script JSON schema find/replace task to be compatible with macOS. ([\#19962](https://github.com/element-hq/synapse/issues/19962)) +- Remove alert silencing deploy step from release script instructions as it's no longer necessary. ([\#19968](https://github.com/element-hq/synapse/issues/19968)) +- Link to changelog instead of duplicating content in the tag/release. ([\#19984](https://github.com/element-hq/synapse/issues/19984)) +- Fix `RemoteJoinHelper` test helper signing events with the default room version (room version mismatch). ([\#20015](https://github.com/element-hq/synapse/issues/20015)) +- Fix `assertIncludes` printing `None` at the end of the message. ([\#20020](https://github.com/element-hq/synapse/issues/20020)) + + + + +# Synapse 1.157.2 (2026-07-28) + +This security release addresses several vulnerabilities. + +Please upgrade when you can, particularly if your homeserver participates in open federation +and/or has untrusted local users. + +## Security Fixes + +High severity: + +- Fix [ELEMENTSEC-2026-1071](https://github.com/element-hq/synapse/security/advisories/GHSA-fp53-rw9v-hcf9) +- Fix [ELEMENTSEC-2024-1520](https://github.com/element-hq/synapse/security/advisories/GHSA-rgv2-84w7-5j9p) +- Fix [ELEMENTSEC-2026-1717](https://github.com/element-hq/synapse/security/advisories/GHSA-27p5-4f45-gx76) +- Fix [ELEMENTSEC-2026-1721](https://github.com/element-hq/synapse/security/advisories/GHSA-95fh-hv8c-chvq) +- Fix [ELEMENTSEC-2026-1729](https://github.com/element-hq/synapse/security/advisories/GHSA-cjh7-rcpx-xpf8) +- Fix [ELEMENTSEC-2026-1740](https://github.com/element-hq/synapse/security/advisories/GHSA-6wjm-9p2x-gvpm) + +Moderate severity: + +- Fix [ELEMENTSEC-2026-1714](https://github.com/element-hq/synapse/security/advisories/GHSA-qcjr-46gf-7f4r) +- Fix [ELEMENTSEC-2026-1718](https://github.com/element-hq/synapse/security/advisories/GHSA-r66v-qhwx-8rg4) +- Fix [ELEMENTSEC-2026-1751](https://github.com/element-hq/synapse/security/advisories/GHSA-jhcg-5392-5mjw) + +Low severity: + +- Fix [ELEMENTSEC-2026-1703](https://github.com/element-hq/synapse/security/advisories/GHSA-vh4c-pqh4-w3wq) +- Fix [ELEMENTSEC-2026-1760](https://github.com/element-hq/synapse/security/advisories/GHSA-hgcg-p9gx-fq5f) + + +# Synapse 1.157.1 (2026-07-22) + +## Bugfixes + +- Fix config regression around falsy `experimental_features` no longer being accepted. ([\#19987](https://github.com/element-hq/synapse/issues/19987)) + + +# Synapse 1.157.0 (2026-07-21) + +No significant changes since 1.157.0rc1. + +Please check [the relevant section in the upgrade notes](https://github.com/element-hq/synapse/blob/develop/docs/upgrade.md#upgrading-to-v11570) as this release removes support for the deprecated MSC3861 Auth Delegation (`experimental_features.msc3861`). + + +# Synapse 1.157.0rc1 (2026-07-14) + +## Features + +- [MSC4140: Cancellable delayed events](https://github.com/matrix-org/matrix-spec-proposals/pull/4140): Limit how many delayed events a user may have scheduled at once. ([\#19539](https://github.com/element-hq/synapse/issues/19539)) +- Support [MSC4446](https://github.com/matrix-org/matrix-spec-proposals/pull/4446) for moving fully read markers backwards. Contributed by @SpiritCroc @ Beeper. ([\#19663](https://github.com/element-hq/synapse/issues/19663)) +- Add before and after time filters to the ['Redact events of a user'](https://element-hq.github.io/synapse/v1.157/admin_api/user_admin_api.html#redact-events-of-a-user) Admin API. ([\#19802](https://github.com/element-hq/synapse/issues/19802)) +- Updated experimental support for [MSC4388: Secure out-of-band channel for sign in with QR](https://github.com/matrix-org/matrix-spec-proposals/pull/4388). ([\#19808](https://github.com/element-hq/synapse/issues/19808)) +- Add an `exclude_rooms_from_presence` configuration option to stop presence being routed between users solely because they share one of the listed rooms. ([\#19935](https://github.com/element-hq/synapse/issues/19935)) + +## Bugfixes + +- [MSC4140: Cancellable delayed events](https://github.com/matrix-org/matrix-spec-proposals/pull/4140): Update error responses to match their format in the current draft of the MSC. ([\#19539](https://github.com/element-hq/synapse/issues/19539)) +- Lock Sliding Sync connections when inserting lazy members, to prevent repeated deadlocks. ([\#19826](https://github.com/element-hq/synapse/issues/19826)) +- Fix the `flag_existing_quarantined_media` background update skipping some quarantined remote media. Introduced in v1.152.0. ([\#19901](https://github.com/element-hq/synapse/issues/19901)) +- Fix a bug introduced in Synapse v1.150.0 where reactivating a deactivated and erased user did not restore their profile, breaking login, name changes, and invitations. + Contributed by @m4us1ne. ([\#19902](https://github.com/element-hq/synapse/issues/19902)) +- Fix a regression where application services that opted into ephemeral events using the legacy `de.sorunome.msc2409.push_ephemeral` registration flag stopped receiving ephemeral events (including to-device messages used for encryption). Introduced in v1.156.0. ([\#19928](https://github.com/element-hq/synapse/issues/19928)) +- Fix a bug causing device list pruning to skip some rows when the transaction gets retried. ([\#19947](https://github.com/element-hq/synapse/issues/19947)) +- Fix presence states being shown to clients forever after presence is disabled, by marking any previously only users as offline. ([\#19948](https://github.com/element-hq/synapse/issues/19948)) +- Fix `SYNAPSE_ASYNC_IO_REACTOR=1` on Python 3.14. ([\#19949](https://github.com/element-hq/synapse/issues/19949)) + +## Deprecations and Removals + +- Remove support for experimental [MSC3861](https://github.com/matrix-org/matrix-spec-proposals/pull/3861) auth delegation, in favour of the stable Matrix Authentication Service integration support. See [the upgrade notes](https://element-hq.github.io/synapse/v1.157/upgrade.html#upgrading-to-v11570). ([\#19895](https://github.com/element-hq/synapse/issues/19895)) + +## Internal Changes + +- Port the synchronous core of client event serialization to Rust. ([\#19837](https://github.com/element-hq/synapse/issues/19837), [\#19922](https://github.com/element-hq/synapse/issues/19922)) +- Update `HomeserverTestCase.get_success(...)` and friends to drive async Rust (Tokio runtime/thread pool). ([\#19871](https://github.com/element-hq/synapse/issues/19871), [\#19879](https://github.com/element-hq/synapse/issues/19879)) +- Allow Rust code to have database access via Python database connection pool. ([\#19878](https://github.com/element-hq/synapse/issues/19878)) +- Add `golangci-lint` to CI. ([\#19888](https://github.com/element-hq/synapse/issues/19888)) +- Remove wall-clock dependency of `test_redact_messages_all_rooms` test, as this caused flakiness. ([\#19890](https://github.com/element-hq/synapse/issues/19890)) +- Change the [MSC3814](https://github.com/matrix-org/matrix-spec-proposals/pull/3814) dehydrated device `/events` endpoint from `POST` to `GET`. ([\#19896](https://github.com/element-hq/synapse/issues/19896)) +- Change the [MSC3814](https://github.com/matrix-org/matrix-spec-proposals/pull/3814) dehydrated device `/events` endpoint paging to match spec conventions. ([\#19897](https://github.com/element-hq/synapse/issues/19897)) +- Fix storage type mismatches where values were bound with a type that didn't match their database column. ([\#19911](https://github.com/element-hq/synapse/issues/19911)) +- Speed up deletion of old sliding sync connections by adding an index. ([\#19912](https://github.com/element-hq/synapse/issues/19912)) +- Add note to 3PID email token request unit tests that the endpoint being tested can have an expected, artificial delay of up to 1s. ([\#19916](https://github.com/element-hq/synapse/issues/19916)) +- Add an index to `sliding_sync_connection_lazy_members` to speed up deleting old sliding sync connection positions. ([\#19923](https://github.com/element-hq/synapse/issues/19923)) +- Fix `test_lock_contention` being flaky when running against PostgreSQL by budgeting CPU time rather than wall-clock time. ([\#19929](https://github.com/element-hq/synapse/issues/19929)) +- Fix Complement test flake when restarting Synapse workers (cross-test pollution caused by nginx upstreams being temporarily unavailable). ([\#19936](https://github.com/element-hq/synapse/issues/19936)) +- Add clean deploy `FIXME` note for `TestOIDCProviderUnavailable` (problem tracked by [#19937](https://github.com/element-hq/synapse/issues/19937)). ([\#19938](https://github.com/element-hq/synapse/issues/19938)) +- Minor presence performance improvements for large servers. ([\#19939](https://github.com/element-hq/synapse/issues/19939)) +- Reduce replication traffic caused by presence. ([\#19941](https://github.com/element-hq/synapse/issues/19941)) +- Add `last_active_granularity`, `sync_online_timeout` and `idle_timeout` options to the `presence` config section to allow tuning the presence state machine timers. ([\#19942](https://github.com/element-hq/synapse/issues/19942)) + + + + # Synapse 1.156.0 (2026-07-07) No significant changes since 1.156.0rc1. diff --git a/Cargo.lock b/Cargo.lock index 87f50ea5df..6062122c15 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -13,9 +13,9 @@ dependencies = [ [[package]] name = "anyhow" -version = "1.0.102" +version = "1.0.104" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f202df86484c868dbad7eaa557ef785d5c66295e41b460ef922eca0723b842c" +checksum = "330a5ed07fa54e4702c9d6c4174f74427fc0ef6e214bbd677ae50a5099946470" [[package]] name = "arc-swap" @@ -25,13 +25,13 @@ checksum = "69f7f8c3906b62b754cd5326047894316021dcfe5a194c8ea52bdd94934a3457" [[package]] name = "async-trait" -version = "0.1.89" +version = "0.1.92" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9035ad2d096bed7955a320ee7e2230574d28fd3c3a0f186cbea1ff3c7eed5dbb" +checksum = "82f6aeea286b8eb4dd3431a1be1b59d290ace00f5bfd8e2a159bc2a05e2c1667" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 3.0.3", ] [[package]] @@ -78,9 +78,9 @@ checksum = "46c5e41b57b8bba42a04676d81cb89e9ee8e859a1a66f80a5a72e1cb76b34d43" [[package]] name = "bytes" -version = "1.11.1" +version = "1.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e748733b7cbc798e1434b6ac524f0c1ff2ab456fe201501e6497c8417a4fc33" +checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04" [[package]] name = "cc" @@ -103,6 +103,17 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724" +[[package]] +name = "chacha20" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d524456ba66e72eb8b115ff89e01e497f8e6d11d78b70b1aa13c0fbd97540a81" +dependencies = [ + "cfg-if", + "cpufeatures 0.3.0", + "rand_core 0.10.1", +] + [[package]] name = "core-foundation" version = "0.10.1" @@ -137,6 +148,15 @@ dependencies = [ "libc", ] +[[package]] +name = "cpufeatures" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b2a41393f66f16b0823bb79094d54ac5fbd34ab292ddafb9a0456ac9f87d201" +dependencies = [ + "libc", +] + [[package]] name = "crypto-common" version = "0.1.6" @@ -166,7 +186,7 @@ checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.104", ] [[package]] @@ -198,9 +218,9 @@ dependencies = [ [[package]] name = "futures" -version = "0.3.32" +version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b147ee9d1f6d097cef9ce628cd2ee62288d963e16fb287bd9286455b241382d" +checksum = "9a31d2a3fbaaeb2af2368bbdd904aa8e812d3c04a1ee10d3171f52d556e5d0a3" dependencies = [ "futures-channel", "futures-core", @@ -213,9 +233,9 @@ dependencies = [ [[package]] name = "futures-channel" -version = "0.3.32" +version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "07bbe89c50d7a535e539b8c17bc0b49bdb77747034daa8087407d655f3f7cc1d" +checksum = "b1f9e3d69d39e4862ffed03ed071a76f9a13ba1d9109d355b0f0aa6b15e393c4" dependencies = [ "futures-core", "futures-sink", @@ -223,15 +243,15 @@ dependencies = [ [[package]] name = "futures-core" -version = "0.3.32" +version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e3450815272ef58cec6d564423f6e755e25379b217b0bc688e295ba24df6b1d" +checksum = "92d699e522242e69e3003b94ecc1f960f3a5e015aa7c5d7486e65ad01dd94f5e" [[package]] name = "futures-executor" -version = "0.3.32" +version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "baf29c38818342a3b26b5b923639e7b1f4a61fc5e76102d4b1981c6dc7a7579d" +checksum = "031b47cf1a3c6cc8bc2fc76cd437f521619387907d469316e7c0bc278f1f5432" dependencies = [ "futures-core", "futures-task", @@ -240,38 +260,38 @@ dependencies = [ [[package]] name = "futures-io" -version = "0.3.32" +version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cecba35d7ad927e23624b22ad55235f2239cfa44fd10428eecbeba6d6a717718" +checksum = "53c0fa8157de1303bfffdaa1cc2a673bfffb60102f76b0ef4441659124373fed" [[package]] name = "futures-macro" -version = "0.3.32" +version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e835b70203e41293343137df5c0664546da5745f82ec9b84d40be8336958447b" +checksum = "9fb9654ba8355388abeb8dcb4fc62f511300867002afc858860463bdd9fe0c44" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 3.0.3", ] [[package]] name = "futures-sink" -version = "0.3.32" +version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c39754e157331b013978ec91992bde1ac089843443c49cbc7f46150b0fad0893" +checksum = "1944426bf7d03f1d14f708785e4b33efd750b36d48a157b836b3efc15ede8e1d" [[package]] name = "futures-task" -version = "0.3.32" +version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "037711b3d59c33004d3856fbdc83b99d4ff37a24768fa1be9ce3538a1cde4393" +checksum = "cd417de3d1d015fc3bfd2b1ea46dfc7bab72ef86f1cc7cc9c78e728b34a6d1fd" [[package]] name = "futures-util" -version = "0.3.32" +version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "389ca41296e6190b48053de0321d02a77f32f8a5d2461dd38762c0593805c6d6" +checksum = "0d50a92467f8ba5dd6e3ee5d4bd04d73ab2e4e1c44474a0674821dfce14b79bc" dependencies = [ "futures-channel", "futures-core", @@ -312,12 +332,24 @@ name = "getrandom" version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "26145e563e54f2cadc477553f1ec5ee650b00862f0a58bcd12cbdc5f0ea2d2f4" +dependencies = [ + "cfg-if", + "libc", + "r-efi 5.3.0", + "wasi 0.14.2+wasi-0.2.4", +] + +[[package]] +name = "getrandom" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "300e883d756b2e4ec94e02791f39b04b522276138852cfc41d9fb7e904106099" dependencies = [ "cfg-if", "js-sys", "libc", - "r-efi", - "wasi 0.14.2+wasi-0.2.4", + "r-efi 6.0.0", + "rand_core 0.10.1", "wasm-bindgen", ] @@ -384,9 +416,9 @@ checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" [[package]] name = "http" -version = "1.4.2" +version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6970f50e31d6fc17d3fa27329444bfa74e196cf62e95052a3f6fee181dba6425" +checksum = "918d3568bebf352712bc2ef3d46a8bcf1a75b373be6539de198e9105cbbf9ce0" dependencies = [ "bytes", "itoa", @@ -404,9 +436,9 @@ dependencies = [ [[package]] name = "http-body-util" -version = "0.1.3" +version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b021d93e26becf5dc7e1b75b1bed1fd93124b374ceb73f43d4d4eafec896a64a" +checksum = "23169fe34a5fbcdd3f3862e78fb9b6fccd5f02a6dc6f732547005d45631ce71c" dependencies = [ "bytes", "futures-core", @@ -718,9 +750,9 @@ checksum = "241eaef5fd12c88705a01fc1066c48c4b36e0dd4377dcdc7ec3942cea7a69956" [[package]] name = "log" -version = "0.4.32" +version = "0.4.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "953f07c43838f8e6f9758cab68bf5bed85465e7587ebe0b823f1bcd81978ad3a" +checksum = "0ceec5bc11778974d1bcb055b18002eba7f4b3518b6a0081b3af5f21666da9ad" [[package]] name = "lru-slab" @@ -811,9 +843,9 @@ dependencies = [ [[package]] name = "pyo3" -version = "0.28.3" +version = "0.29.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91fd8e38a3b50ed1167fb981cd6fd60147e091784c427b8f7183a7ee32c31c12" +checksum = "4688ddedf473e32662b9b067670129a8afb8c18e351482c70d62ba4a88171e8b" dependencies = [ "anyhow", "bytes", @@ -827,18 +859,18 @@ dependencies = [ [[package]] name = "pyo3-build-config" -version = "0.28.3" +version = "0.29.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e368e7ddfdeb98c9bca7f8383be1648fd84ab466bf2bc015e94008db6d35611e" +checksum = "f41027e41b4bd03f6e60f9f417fe24a6341a6bb744edd62b6f709f2a52ea30e9" dependencies = [ "target-lexicon", ] [[package]] name = "pyo3-ffi" -version = "0.28.3" +version = "0.29.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f29e10af80b1f7ccaf7f69eace800a03ecd13e883acfacc1e5d0988605f651e" +checksum = "e591a95526fead067432c3b3a33fc74770b87b1e04e73671090d9c2055a2b327" dependencies = [ "libc", "pyo3-build-config", @@ -857,34 +889,33 @@ dependencies = [ [[package]] name = "pyo3-macros" -version = "0.28.3" +version = "0.29.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df6e520eff47c45997d2fc7dd8214b25dd1310918bbb2642156ef66a67f29813" +checksum = "73225868fc1cd84eef2c3c230ddb91273bf1de46aeb8a4248da76d32a0924a1c" dependencies = [ "proc-macro2", "pyo3-macros-backend", "quote", - "syn", + "syn 2.0.104", ] [[package]] name = "pyo3-macros-backend" -version = "0.28.3" +version = "0.29.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c4cdc218d835738f81c2338f822078af45b4afdf8b2e33cbb5916f108b813acb" +checksum = "571575aa3749fa6216757dd47d2a3e7ef360f329a40f0666a9fbd14889024952" dependencies = [ "heck", "proc-macro2", - "pyo3-build-config", "quote", - "syn", + "syn 2.0.104", ] [[package]] name = "pythonize" -version = "0.28.0" +version = "0.29.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b79f670c9626c8b651c0581011b57b6ba6970bb69faf01a7c4c0cfc81c43f95" +checksum = "6ec376e1216e0c929a74964ce2020012a1a39f32d80e78aa688721219ea7fb89" dependencies = [ "pyo3", "serde", @@ -913,14 +944,15 @@ dependencies = [ [[package]] name = "quinn-proto" -version = "0.11.14" +version = "0.11.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "434b42fec591c96ef50e21e886936e66d3cc3f737104fdb9b737c40ffb94c098" +checksum = "2f4bfc015262b9df63c8845072ce59068853ff5872180c2ce2f13038b970e560" dependencies = [ "bytes", - "getrandom 0.3.3", + "getrandom 0.4.3", "lru-slab", - "rand", + "rand 0.10.2", + "rand_pcg", "ring", "rustc-hash", "rustls", @@ -961,6 +993,12 @@ version = "5.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" +[[package]] +name = "r-efi" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" + [[package]] name = "rand" version = "0.9.4" @@ -968,7 +1006,18 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "44c5af06bb1b7d3216d91932aed5265164bf384dc89cd6ba05cf59a35f5f76ea" dependencies = [ "rand_chacha", - "rand_core", + "rand_core 0.9.3", +] + +[[package]] +name = "rand" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7f5fa3a058cd35567ef9bfa5e75732bee0f9e4c55fa90477bef2dfcdbc4be80" +dependencies = [ + "chacha20", + "getrandom 0.4.3", + "rand_core 0.10.1", ] [[package]] @@ -978,7 +1027,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb" dependencies = [ "ppv-lite86", - "rand_core", + "rand_core 0.9.3", ] [[package]] @@ -991,10 +1040,25 @@ dependencies = [ ] [[package]] -name = "regex" -version = "1.12.4" +name = "rand_core" +version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f1292b7759ae1cb9ec195452d1390a074f0cd8541ab7a5a8c31cd6db45d4a6ba" +checksum = "63b8176103e19a2643978565ca18b50549f6101881c443590420e4dc998a3c69" + +[[package]] +name = "rand_pcg" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "caa0f4137e1c0a72f4c651489402276c8e8e1cf081f3b0ba156d2cbeef09e86a" +dependencies = [ + "rand_core 0.10.1", +] + +[[package]] +name = "regex" +version = "1.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f020237b6c8eed93db2e2cb53c00c60a8e1bc73da7d073199a1180401450218d" dependencies = [ "aho-corasick", "memchr", @@ -1004,9 +1068,9 @@ dependencies = [ [[package]] name = "regex-automata" -version = "0.4.13" +version = "0.4.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5276caf25ac86c8d810222b3dbb938e512c55c6831a10f3e6ed1c93b84041f1c" +checksum = "8fcfdb36bda0c880c5931cdc7a2bcdc8ba4556847b9d912bca70bc94708711ad" dependencies = [ "aho-corasick", "memchr", @@ -1128,9 +1192,9 @@ dependencies = [ [[package]] name = "rustls-webpki" -version = "0.103.13" +version = "0.103.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61c429a8649f110dddef65e2a5ad240f747e85f7758a6bccc7e5777bd33f756e" +checksum = "0527518605e68109d875e248ea259b6758801cf165e4b2c2733ae3b51f12535a" dependencies = [ "ring", "rustls-pki-types", @@ -1189,9 +1253,9 @@ checksum = "d767eb0aabc880b29956c35734170f26ed551a859dbd361d140cdbeca61ab1e2" [[package]] name = "serde" -version = "1.0.228" +version = "1.0.229" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e" +checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba" dependencies = [ "serde_core", "serde_derive", @@ -1199,29 +1263,29 @@ dependencies = [ [[package]] name = "serde_core" -version = "1.0.228" +version = "1.0.229" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad" +checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.228" +version = "1.0.229" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" +checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 3.0.3", ] [[package]] name = "serde_json" -version = "1.0.150" +version = "1.0.151" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e8014e44b4736ed0538adeecded0fce2a272f22dc9578a7eb6b2d9993c74cfb9" +checksum = "c841b55ecdae098c80dcae9cf767f6f8a0c2cdb3416bbef72181df4d0fe73f14" dependencies = [ "itoa", "memchr", @@ -1249,7 +1313,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e3bf829a2d51ab4a5ddf1352d8470c140cadc8301b2ae1789db023f01cedd6ba" dependencies = [ "cfg-if", - "cpufeatures", + "cpufeatures 0.2.17", "digest", ] @@ -1260,7 +1324,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" dependencies = [ "cfg-if", - "cpufeatures", + "cpufeatures 0.2.17", "digest", ] @@ -1325,6 +1389,17 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "syn" +version = "3.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53e9bae58849f64dfa4f5d5ae372c8341f7305f82a3868709269343628b659a3" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + [[package]] name = "synapse" version = "0.1.0" @@ -1375,7 +1450,7 @@ checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.104", ] [[package]] @@ -1401,7 +1476,7 @@ checksum = "7f7cf42b4507d8ea322120659672cf1b9dbb93f8f2d4ecfd6e51350ff5b17a1d" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.104", ] [[package]] @@ -1548,7 +1623,7 @@ version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "470dbf6591da1b39d43c14523b2b469c86879a53e8b758c8e090a470fe7b1fbe" dependencies = [ - "rand", + "rand 0.9.4", "web-time", ] @@ -1633,7 +1708,7 @@ dependencies = [ "log", "proc-macro2", "quote", - "syn", + "syn 2.0.104", "wasm-bindgen-shared", ] @@ -1668,7 +1743,7 @@ checksum = "8ae87ea40c9f689fc23f209965b6fb8a99ad69aeeb0231408be24920604395de" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.104", "wasm-bindgen-backend", "wasm-bindgen-shared", ] @@ -1847,7 +1922,7 @@ checksum = "38da3c9736e16c5d3c8c597a9aaa5d1fa565d0532ae05e27c24aa62fb32c0ab6" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.104", "synstructure", ] @@ -1868,7 +1943,7 @@ checksum = "9ecf5b4cc5364572d7f4c329661bcc82724222973f2cab6f050a4e5c22f75181" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.104", ] [[package]] @@ -1888,7 +1963,7 @@ checksum = "d71e5d6e06ab090c67b5e44993ec16b72dcbaabc526db883a360057678b48502" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.104", "synstructure", ] @@ -1928,7 +2003,7 @@ checksum = "5b96237efa0c878c64bd89c436f661be4e46b2f3eff1ebb976f7ef2321d2f58f" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.104", ] [[package]] diff --git a/changelog.d/19718.misc b/changelog.d/19718.misc new file mode 100644 index 0000000000..5914943f34 --- /dev/null +++ b/changelog.d/19718.misc @@ -0,0 +1 @@ +Add storage functions for future [MSC4242](https://github.com/matrix-org/matrix-spec-proposals/pull/4242) work. diff --git a/changelog.d/19782.misc b/changelog.d/19782.misc new file mode 100644 index 0000000000..35f6c32385 --- /dev/null +++ b/changelog.d/19782.misc @@ -0,0 +1 @@ +Put the `redacts` key under `content` when generating [MSC3912](https://github.com/matrix-org/matrix-spec-proposals/pull/3912) (relation based redactions) for room versions greater than 10. Contributed by @famedly. diff --git a/changelog.d/19802.feature b/changelog.d/19802.feature deleted file mode 100644 index 455b64ce0c..0000000000 --- a/changelog.d/19802.feature +++ /dev/null @@ -1 +0,0 @@ -Add before and after time filters to the 'Redact events of a user' Admin API. diff --git a/changelog.d/19808.feature b/changelog.d/19808.feature deleted file mode 100644 index 69eca83681..0000000000 --- a/changelog.d/19808.feature +++ /dev/null @@ -1 +0,0 @@ -Updated experimental support for [MSC4388: Secure out-of-band channel for sign in with QR](https://github.com/matrix-org/matrix-spec-proposals/pull/4388). diff --git a/changelog.d/19826.bugfix b/changelog.d/19826.bugfix deleted file mode 100644 index 771b197b20..0000000000 --- a/changelog.d/19826.bugfix +++ /dev/null @@ -1 +0,0 @@ -Lock Sliding Sync connections when inserting lazy members, to prevent repeated deadlocks. \ No newline at end of file diff --git a/changelog.d/19837.misc b/changelog.d/19837.misc deleted file mode 100644 index 64a531d4ed..0000000000 --- a/changelog.d/19837.misc +++ /dev/null @@ -1 +0,0 @@ -Port the synchronous core of client event serialization to Rust. diff --git a/changelog.d/19871.misc b/changelog.d/19871.misc deleted file mode 100644 index be10ee0540..0000000000 --- a/changelog.d/19871.misc +++ /dev/null @@ -1 +0,0 @@ -Update `HomeserverTestCase.get_success(...)` and friends to drive async Rust (Tokio runtime/thread pool). diff --git a/changelog.d/19878.misc b/changelog.d/19878.misc deleted file mode 100644 index 9ea93729c9..0000000000 --- a/changelog.d/19878.misc +++ /dev/null @@ -1 +0,0 @@ -Allow Rust code to have database access via Python database connection pool. diff --git a/changelog.d/19879.misc b/changelog.d/19879.misc deleted file mode 100644 index be10ee0540..0000000000 --- a/changelog.d/19879.misc +++ /dev/null @@ -1 +0,0 @@ -Update `HomeserverTestCase.get_success(...)` and friends to drive async Rust (Tokio runtime/thread pool). diff --git a/changelog.d/19888.misc b/changelog.d/19888.misc deleted file mode 100644 index 0506b21d42..0000000000 --- a/changelog.d/19888.misc +++ /dev/null @@ -1 +0,0 @@ -Add `golangci-lint` to CI. \ No newline at end of file diff --git a/changelog.d/19890.misc b/changelog.d/19890.misc deleted file mode 100644 index b74dbc3086..0000000000 --- a/changelog.d/19890.misc +++ /dev/null @@ -1 +0,0 @@ -Remove wall-clock dependency of `test_redact_messages_all_rooms` test, as this caused flakiness. \ No newline at end of file diff --git a/changelog.d/19895.removal b/changelog.d/19895.removal deleted file mode 100644 index 1b39ffb139..0000000000 --- a/changelog.d/19895.removal +++ /dev/null @@ -1 +0,0 @@ -Remove support for experimental [MSC3861](https://github.com/matrix-org/matrix-spec-proposals/pull/3861) auth delegation, in favour of the stable Matrix Authentication Service integration support. See [the upgrade notes](https://element-hq.github.io/synapse/v1.157/upgrade.html#upgrading-to-v11570). diff --git a/changelog.d/19896.misc b/changelog.d/19896.misc deleted file mode 100644 index 2c6a8740ef..0000000000 --- a/changelog.d/19896.misc +++ /dev/null @@ -1 +0,0 @@ -Change the [MSC3814](https://github.com/matrix-org/matrix-spec-proposals/pull/3814) dehydrated device `/events` endpoint from `POST` to `GET`. diff --git a/changelog.d/19897.misc b/changelog.d/19897.misc deleted file mode 100644 index f1163dc1f1..0000000000 --- a/changelog.d/19897.misc +++ /dev/null @@ -1 +0,0 @@ -Change the [MSC3814](https://github.com/matrix-org/matrix-spec-proposals/pull/3814) dehydrated device `/events` endpoint paging to match spec conventions. diff --git a/changelog.d/19901.bugfix b/changelog.d/19901.bugfix deleted file mode 100644 index e31c665c18..0000000000 --- a/changelog.d/19901.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix the `flag_existing_quarantined_media` background update skipping some quarantined remote media. Introduced in v1.152.0. diff --git a/changelog.d/19902.bugfix b/changelog.d/19902.bugfix deleted file mode 100644 index fb5f75c295..0000000000 --- a/changelog.d/19902.bugfix +++ /dev/null @@ -1,2 +0,0 @@ -Fix a bug introduced in Synapse v1.150.0 where reactivating a deactivated and erased user did not restore their profile, breaking login, name changes, and invitations. -Contributed by @m4us1ne. diff --git a/changelog.d/19911.misc b/changelog.d/19911.misc deleted file mode 100644 index 11331455aa..0000000000 --- a/changelog.d/19911.misc +++ /dev/null @@ -1 +0,0 @@ -Fix storage type mismatches where values were bound with a type that didn't match their database column. diff --git a/changelog.d/19912.misc b/changelog.d/19912.misc deleted file mode 100644 index e9491ab335..0000000000 --- a/changelog.d/19912.misc +++ /dev/null @@ -1 +0,0 @@ -Speed up deletion of old sliding sync connections by adding an index. diff --git a/changelog.d/19913.misc b/changelog.d/19913.misc deleted file mode 100644 index 31b5a4d37b..0000000000 --- a/changelog.d/19913.misc +++ /dev/null @@ -1 +0,0 @@ -Fix a flake in 3PID inhibit error unit tests, causing occasional failures in CI. \ No newline at end of file diff --git a/changelog.d/19915.feature b/changelog.d/19915.feature new file mode 100644 index 0000000000..39446ae6a1 --- /dev/null +++ b/changelog.d/19915.feature @@ -0,0 +1 @@ +Don't validate signatures with unknown algorithms for master keys, and allow updates to signatures. \ No newline at end of file diff --git a/changelog.d/19922.misc b/changelog.d/19922.misc deleted file mode 100644 index 64a531d4ed..0000000000 --- a/changelog.d/19922.misc +++ /dev/null @@ -1 +0,0 @@ -Port the synchronous core of client event serialization to Rust. diff --git a/changelog.d/19923.misc b/changelog.d/19923.misc deleted file mode 100644 index 4a52d1cca2..0000000000 --- a/changelog.d/19923.misc +++ /dev/null @@ -1 +0,0 @@ -Add an index to `sliding_sync_connection_lazy_members` to speed up deleting old sliding sync connection positions. diff --git a/changelog.d/19926.feature b/changelog.d/19926.feature new file mode 100644 index 0000000000..88f3111079 --- /dev/null +++ b/changelog.d/19926.feature @@ -0,0 +1 @@ +[MSC4140: Cancellable delayed events](https://github.com/matrix-org/matrix-spec-proposals/pull/4140): Add an endpoint for getting a single delayed event. diff --git a/changelog.d/19929.misc b/changelog.d/19929.misc deleted file mode 100644 index 4778f9fa32..0000000000 --- a/changelog.d/19929.misc +++ /dev/null @@ -1 +0,0 @@ -Fix `test_lock_contention` being flaky when running against PostgreSQL by budgeting CPU time rather than wall-clock time. diff --git a/changelog.d/19936.misc b/changelog.d/19936.misc deleted file mode 100644 index 5c1f7eef52..0000000000 --- a/changelog.d/19936.misc +++ /dev/null @@ -1 +0,0 @@ -Fix Complement test flake when restarting Synapse workers (cross-test pollution caused by nginx upstreams being temporarily unavailable). diff --git a/changelog.d/19938.misc b/changelog.d/19938.misc deleted file mode 100644 index d02c9bf3b7..0000000000 --- a/changelog.d/19938.misc +++ /dev/null @@ -1 +0,0 @@ -Add clean deploy `FIXME` note for `TestOIDCProviderUnavailable` (problem tracked by [#19937](https://github.com/element-hq/synapse/issues/19937)). diff --git a/changelog.d/19941.misc b/changelog.d/19941.misc deleted file mode 100644 index e928f68bc5..0000000000 --- a/changelog.d/19941.misc +++ /dev/null @@ -1 +0,0 @@ -Reduce replication traffic caused by presence. diff --git a/changelog.d/19942.misc b/changelog.d/19942.misc deleted file mode 100644 index bdd95c3c0f..0000000000 --- a/changelog.d/19942.misc +++ /dev/null @@ -1 +0,0 @@ -Add `last_active_granularity`, `sync_online_timeout` and `idle_timeout` options to the `presence` config section to allow tuning the presence state machine timers. diff --git a/changelog.d/19972.feature b/changelog.d/19972.feature new file mode 100644 index 0000000000..56e3b27136 --- /dev/null +++ b/changelog.d/19972.feature @@ -0,0 +1 @@ +Add experimental support for letting application services proxy namespaces in the C-S and S-S API as per MSC4512. diff --git a/changelog.d/20036.bugfix b/changelog.d/20036.bugfix new file mode 100644 index 0000000000..b2f7c3e79f --- /dev/null +++ b/changelog.d/20036.bugfix @@ -0,0 +1 @@ +Apply the `rc_reports` rate limit to the [room reporting endpoint](https://spec.matrix.org/v1.19/client-server-api/#post_matrixclientv3roomsroomidreport), which the spec declares as rate-limited. diff --git a/changelog.d/20050.bugfix b/changelog.d/20050.bugfix new file mode 100644 index 0000000000..d7a0025039 --- /dev/null +++ b/changelog.d/20050.bugfix @@ -0,0 +1 @@ +Fix `m.call.invite` state events not being given the proper power level on rooms created with the `public_chat` preset. Contributed by @famedly @itsoyou. diff --git a/changelog.d/20101.bugfix b/changelog.d/20101.bugfix new file mode 100644 index 0000000000..60e1d3ada2 --- /dev/null +++ b/changelog.d/20101.bugfix @@ -0,0 +1 @@ +Return the `M_INVALID_PARAM` error code specified by Matrix v1.13 ([MSC4178](https://github.com/matrix-org/matrix-spec-proposals/pull/4178)) when a malformed email address or country code is submitted to `/account/3pid/{email,msisdn}/requestToken`, and report an unsupported medium ahead of the denied/in-use checks on the msisdn variant. diff --git a/changelog.d/20104.bugfix b/changelog.d/20104.bugfix new file mode 100644 index 0000000000..f04945660c --- /dev/null +++ b/changelog.d/20104.bugfix @@ -0,0 +1 @@ +Fix a regression where `client_secret` request parameters were not validated against the character set required by the spec on Pydantic-validated endpoints, regressed in Synapse 1.66.0 (originally fixed for https://github.com/matrix-org/synapse/issues/6766). diff --git a/changelog.d/20106.bugfix b/changelog.d/20106.bugfix new file mode 100644 index 0000000000..bd88506868 --- /dev/null +++ b/changelog.d/20106.bugfix @@ -0,0 +1 @@ +Do not send a duplicate `m.room.encryption` event on room creation when the client already supplies one in the initial state and `encryption_enabled_by_default_for_room_type` is enabled. Contributed by @FrenchGithubUser @Famedly. diff --git a/changelog.d/20107.misc b/changelog.d/20107.misc new file mode 100644 index 0000000000..fb7a845252 --- /dev/null +++ b/changelog.d/20107.misc @@ -0,0 +1 @@ +Declare types that already appear in the module API's public signatures (such as `Requester`, `SynapseRequest` and `UserInfo`) in `synapse.module_api.__all__`. diff --git a/changelog.d/20114.bugfix b/changelog.d/20114.bugfix new file mode 100644 index 0000000000..967a3717c0 --- /dev/null +++ b/changelog.d/20114.bugfix @@ -0,0 +1 @@ +Fix a long-standing bug where private read receipts of users outside an application service's namespaces were sent to application services that opted in to receiving ephemeral events ([MSC2409](https://github.com/matrix-org/matrix-spec-proposals/pull/2409)). diff --git a/changelog.d/20119.bugfix b/changelog.d/20119.bugfix new file mode 100644 index 0000000000..a09602941a --- /dev/null +++ b/changelog.d/20119.bugfix @@ -0,0 +1 @@ +Fix a bug where the `event_search` background reindex skipped all `m.room.topic` events, making room topics unsearchable after a search index rebuild. diff --git a/changelog.d/20127.feature b/changelog.d/20127.feature new file mode 100644 index 0000000000..5e6738e035 --- /dev/null +++ b/changelog.d/20127.feature @@ -0,0 +1 @@ +Add experimental federation client support for [MSC4242](https://github.com/matrix-org/matrix-spec-proposals/pull/4242): State DAGs. diff --git a/changelog.d/20132.bugfix b/changelog.d/20132.bugfix new file mode 100644 index 0000000000..542596af63 --- /dev/null +++ b/changelog.d/20132.bugfix @@ -0,0 +1 @@ +Fixed joining rooms with unrecognized restricted join rules being rejected outright instead of returning the proper `M_UNABLE_TO_AUTHORISE_JOIN` error code. Contributed by @tulir @ Beeper. diff --git a/changelog.d/20136.misc b/changelog.d/20136.misc new file mode 100644 index 0000000000..3a335cdd6e --- /dev/null +++ b/changelog.d/20136.misc @@ -0,0 +1 @@ +Add missing tests for parse_stripped_state_event. Contributed by @guillemo12. diff --git a/changelog.d/20138.feature b/changelog.d/20138.feature new file mode 100644 index 0000000000..654a1e67c3 --- /dev/null +++ b/changelog.d/20138.feature @@ -0,0 +1 @@ +Add a config option that limits the time period in which local users can redact their own messages. Contributed by @defaultdino. diff --git a/changelog.d/20140.doc b/changelog.d/20140.doc new file mode 100644 index 0000000000..9d6432631f --- /dev/null +++ b/changelog.d/20140.doc @@ -0,0 +1 @@ +Make `federation_domain_whitelist` nullable in config schema. \ No newline at end of file diff --git a/changelog.d/20146.feature b/changelog.d/20146.feature new file mode 100644 index 0000000000..262ae828f8 --- /dev/null +++ b/changelog.d/20146.feature @@ -0,0 +1 @@ +Deprecate `livekit_service_url` and add support for specifying the SFU WebSocket URL for configured LiveKit transports. Please check [the relevant section in the upgrade notes](https://github.com/element-hq/synapse/blob/develop/docs/upgrade.md#upgrading-to-v11600). diff --git a/changelog.d/20148.bugfix b/changelog.d/20148.bugfix new file mode 100644 index 0000000000..2983484208 --- /dev/null +++ b/changelog.d/20148.bugfix @@ -0,0 +1 @@ +Fix a bug where, until Synapse was restarted, new events in a room would fail to be persisted if the database went down while an event was being persisted in the room. Bug introduced in v1.124.0. diff --git a/changelog.d/20149.bugfix b/changelog.d/20149.bugfix new file mode 100644 index 0000000000..2440618acf --- /dev/null +++ b/changelog.d/20149.bugfix @@ -0,0 +1 @@ +Fix the `/profile` endpoint, when querying custom fields, returning a 500 error instead of 404 when the profile does not exist at all. \ No newline at end of file diff --git a/changelog.d/20152.bugfix b/changelog.d/20152.bugfix new file mode 100644 index 0000000000..4233499d33 --- /dev/null +++ b/changelog.d/20152.bugfix @@ -0,0 +1 @@ +When not using MSC3866, omit the approval flag from the response of `GET /_synapse/admin/v2/users`. diff --git a/changelog.d/20156.misc b/changelog.d/20156.misc new file mode 100644 index 0000000000..116a238d8f --- /dev/null +++ b/changelog.d/20156.misc @@ -0,0 +1 @@ +[MSC4140: Cancellable delayed events](https://github.com/matrix-org/matrix-spec-proposals/pull/4140): Update the error response for requesting to schedule a delayed event with a delay that exceeds the server-enforced maximum delay. diff --git a/changelog.d/20163.removal b/changelog.d/20163.removal new file mode 100644 index 0000000000..44d8ccb144 --- /dev/null +++ b/changelog.d/20163.removal @@ -0,0 +1 @@ +Drop `GET /_matrix/client/unstable/org.matrix.msc2965/auth_issuer` endpoint which never ended up being used. \ No newline at end of file diff --git a/changelog.d/20169.bugfix b/changelog.d/20169.bugfix new file mode 100644 index 0000000000..1845478a22 --- /dev/null +++ b/changelog.d/20169.bugfix @@ -0,0 +1 @@ +Fix `/sync` returning membership events from after the user's leave in `state_after` for left rooms when lazy-loading room members (experimental [MSC4222](https://github.com/matrix-org/matrix-spec-proposals/pull/4222) implementation). diff --git a/changelog.d/20172.bugfix b/changelog.d/20172.bugfix new file mode 100644 index 0000000000..a25b5e1250 --- /dev/null +++ b/changelog.d/20172.bugfix @@ -0,0 +1 @@ +Fix a bug where a server admin setting a custom profile field for a user with no profile received a 500 error; this now succeeds for existing (e.g. deactivated) users and returns a 404 error if the user does not exist. diff --git a/changelog.d/20173.bugfix b/changelog.d/20173.bugfix new file mode 100644 index 0000000000..3cb83fcb44 --- /dev/null +++ b/changelog.d/20173.bugfix @@ -0,0 +1 @@ +Fix `PUT`/`DELETE` on a profile field returning HTTP 400 instead of 403 (with errcode `M_FORBIDDEN`) when profile changes are disabled via `enable_set_displayname` or `enable_set_avatar_url`. diff --git a/changelog.d/20180.bugfix b/changelog.d/20180.bugfix new file mode 100644 index 0000000000..72f4fc51d0 --- /dev/null +++ b/changelog.d/20180.bugfix @@ -0,0 +1 @@ +Return the stable `M_APPSERVICE_LOGIN_UNSUPPORTED` error code, added in Matrix 1.17, instead of its unstable MSC4190-prefixed identifier. diff --git a/changelog.d/20189.bugfix b/changelog.d/20189.bugfix new file mode 100644 index 0000000000..af3cba350e --- /dev/null +++ b/changelog.d/20189.bugfix @@ -0,0 +1 @@ +Fix missing validation of `membership` when making `make_*` requests over federation. Contributed by @tulir @ Beeper. diff --git a/changelog.d/20192.removal b/changelog.d/20192.removal new file mode 100644 index 0000000000..b1de7efa95 --- /dev/null +++ b/changelog.d/20192.removal @@ -0,0 +1 @@ +Remove support for the unstable `org.matrix.msc3202.device_id` query parameter for application service device masquerading. diff --git a/complement/go.mod b/complement/go.mod index aa2333a5a0..cea013573b 100644 --- a/complement/go.mod +++ b/complement/go.mod @@ -51,7 +51,7 @@ require ( go.opentelemetry.io/otel/sdk v1.43.0 // indirect go.opentelemetry.io/otel/sdk/metric v1.43.0 // indirect go.opentelemetry.io/otel/trace v1.43.0 // indirect - golang.org/x/crypto v0.51.0 // indirect + golang.org/x/crypto v0.52.0 // indirect golang.org/x/net v0.55.0 // indirect golang.org/x/sync v0.20.0 // indirect golang.org/x/sys v0.45.0 // indirect diff --git a/complement/go.sum b/complement/go.sum index 49c3724e83..4858bc78c8 100644 --- a/complement/go.sum +++ b/complement/go.sum @@ -122,8 +122,8 @@ go.opentelemetry.io/proto/otlp v1.10.0/go.mod h1:/CV4QoCR/S9yaPj8utp3lvQPoqMtxXd golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.51.0 h1:IBPXwPfKxY7cWQZ38ZCIRPI50YLeevDLlLnyC5wRGTI= -golang.org/x/crypto v0.51.0/go.mod h1:8AdwkbraGNABw2kOX6YFPs3WM22XqI4EXEd8g+x7Oc8= +golang.org/x/crypto v0.52.0 h1:RMs7fP2rXdep0CftQlK8Uf+kibLm7qkCcradZWYz988= +golang.org/x/crypto v0.52.0/go.mod h1:1QgfPxDqh0T2M/elOJtp9RvuR95kVjir0e6/BvEmGbc= golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 h1:2dVuKD2vS7b0QIHQbpyTISPd0LeHDbnYEryqj5Q1ug8= golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56/go.mod h1:M4RDyNAINzryxdtnbRXRL/OHtkFuWGRjvuhBJpk2IlY= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= diff --git a/contrib/grafana/README.md b/contrib/grafana/README.md index 0bbd57439e..4ccfb9b60b 100644 --- a/contrib/grafana/README.md +++ b/contrib/grafana/README.md @@ -4,3 +4,60 @@ 1. Have your Prometheus scrape your Synapse. https://element-hq.github.io/synapse/latest/metrics-howto.html 2. Import dashboard into Grafana. Download `synapse.json`. Import it to Grafana and select the correct Prometheus datasource. http://docs.grafana.org/reference/export_import/ 3. Set up required recording rules. [contrib/prometheus](../prometheus) + + +## Sharing a JSON snapshot of a Grafana dashboard + +To aid in debugging, you can share the dashboard with others by creating a snapshot of +the Grafana dashboard and exporting it as JSON. The snapshot will contain all of the +current values of the metrics visible on the dashboard. + +**To capture the JSON snapshot:** + + 1. Visit the Grafana dashboard in your browser + 1. Expand all of the sections on the dashboard and let all of the panels load in (the + snapshot only captures what's loaded on your page) + 1. Use the Grafana UI to capture the snapshot: **Share** (drop down arrow) -> **Share + Snapshot** -> **Publish Snapshot** + - If you run into `413` (`Content Too Large`) errors, you're probably just running + into the upload limit set on your reverse proxy (like nginx) in front of your + Grafana instance. Just increase it and try again. + - You may also run into `400` (`Bad Request`) which appear as `bad request data` + in the Grafana UI if the snapshot is larger than 100 MB. Grafana introduced a + [100 MB + limit](https://github.com/grafana/grafana/blob/555d6dde60b0f49acd453c7293b1cd518fda3592/pkg/web/binding.go#L13-L14) + as part of their [2026 June security + releases](https://github.com/grafana/grafana/pull/125789). There is no + workaround on this app limit, so you will have to reduce the time window, + number of panels shown, etc. + 1. Grab the snapshot ID from the link generated in the last step or find it from the + list of snapshots on https://localhost:3000/dashboard/snapshots + 1. To export the JSON, you have to use the [API for getting a + snapshot](https://grafana.com/docs/grafana/latest/developer-resources/api-reference/http-api/api-legacy/snapshot/#get-snapshot-by-key) + (update the snapshot ID in the command below): + ```shell + curl --request GET \ + --header 'Content-Type: application/json' \ + --output ~/Downloads/2026-08-16-synapse-myhomeserver.com.json \ + http://admin:admin@localhost:3000/api/snapshots/nerimdSEDz530rM6CiwkEFi09A1841yF + ``` + 1. If you're trying to upload to GitHub, keep in mind that GitHub has a 25MB limit for + attachments on issues. As an alternative, you could create a [GitHub + Gist](https://gist.github.com/). If the snapshot is too big to upload via the GitHub + UI, you can create a blank/empty gist and add it via git (gists are git repos). + 1. Once you have the JSON file, you can delete the snapshot from your Grafana instance + to free up space (from the snaphots page, + https://localhost:3000/dashboard/snapshots). The JSON file will still be valid and + can be shared with others. + +**To import the JSON snapshot into Grafana**, you have to use the [API for creating a snapshot](https://grafana.com/docs/grafana/latest/developer-resources/api-reference/http-api/snapshot/#create-new-snapshot) (passing in the whole JSON). + + 1. Import example: + ```shell + cat ~/Downloads/2026-08-16-synapse-myhomeserver.com.json \ + | jq '. += {"name": "2026-08-16-synapse-myhomeserver.com"}' \ + | curl --request POST \ + --header 'Content-Type: application/json' \ + --data @- http://admin:admin@localhost:3000/api/snapshots + ``` + 1. Then you can find the snapshot on https://localhost:3000/dashboard/snapshots to view it. diff --git a/debian/changelog b/debian/changelog index c1522f068d..529bfb865c 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,69 @@ +matrix-synapse-py3 (1.160.0) stable; urgency=medium + + * New synapse release 1.160.0. + + -- Synapse Packaging team Wed, 02 Sep 2026 21:22:51 +0000 + +matrix-synapse-py3 (1.160.0~rc2) stable; urgency=medium + + * New synapse release 1.160.0rc2. + + -- Synapse Packaging team Mon, 31 Aug 2026 22:25:15 +0000 + +matrix-synapse-py3 (1.160.0~rc1) stable; urgency=medium + + * New synapse release 1.160.0rc1. + + -- Synapse Packaging team Tue, 25 Aug 2026 16:17:54 +0000 + +matrix-synapse-py3 (1.159.0) stable; urgency=medium + + * New synapse release 1.159.0. + + -- Synapse Packaging team Tue, 18 Aug 2026 16:02:56 +0000 + +matrix-synapse-py3 (1.159.0~rc1) stable; urgency=medium + + * New synapse release 1.159.0rc1. + + -- Synapse Packaging team Tue, 11 Aug 2026 16:25:42 +0000 + +matrix-synapse-py3 (1.158.0) stable; urgency=medium + + * New synapse release 1.158.0. + + -- Synapse Packaging team Tue, 04 Aug 2026 17:52:05 +0000 + +matrix-synapse-py3 (1.158.0~rc1) stable; urgency=medium + + * New synapse release 1.158.0rc1. + + -- Synapse Packaging team Thu, 30 Jul 2026 14:53:30 +0000 + +matrix-synapse-py3 (1.157.2) stable; urgency=medium + + * New Synapse release 1.157.2. + + -- Synapse Packaging team Tue, 28 Jul 2026 14:01:25 +0100 + +matrix-synapse-py3 (1.157.1) stable; urgency=medium + + * New synapse release 1.157.1. + + -- Synapse Packaging team Wed, 22 Jul 2026 14:30:55 +0000 + +matrix-synapse-py3 (1.157.0) stable; urgency=medium + + * New synapse release 1.157.0. + + -- Synapse Packaging team Tue, 21 Jul 2026 15:17:02 +0000 + +matrix-synapse-py3 (1.157.0~rc1) stable; urgency=medium + + * New synapse release 1.157.0rc1. + + -- Synapse Packaging team Tue, 14 Jul 2026 21:18:16 +0000 + matrix-synapse-py3 (1.156.0) stable; urgency=medium * New Synapse release 1.156.0. diff --git a/docker/Dockerfile b/docker/Dockerfile index f395127da0..30c6b670b6 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -203,6 +203,9 @@ EXPOSE 8008/tcp 8448/tcp # SYNAPSE_ENABLE_METRICS=1). EXPOSE 19090/tcp +# Python's `print()` buffers output by default, this tells Python to disable buffering. +ENV PYTHONUNBUFFERED=1 + ENTRYPOINT ["/start.py"] HEALTHCHECK --start-period=5s --interval=15s --timeout=5s \ diff --git a/docker/complement/conf/workers-shared-extra.yaml.j2 b/docker/complement/conf/workers-shared-extra.yaml.j2 index e829292aca..4dc4eb932b 100644 --- a/docker/complement/conf/workers-shared-extra.yaml.j2 +++ b/docker/complement/conf/workers-shared-extra.yaml.j2 @@ -15,6 +15,8 @@ enable_registration_without_verification: true bcrypt_rounds: 4 url_preview_enabled: true url_preview_ip_range_blacklist: [] +# MSC4429 and MSC4262 Profile updates down sync +include_profile_updates_in_sync: true ## Registration ## diff --git a/docker/configure_workers_and_start.py b/docker/configure_workers_and_start.py index 38f8649b44..c5d785fc38 100755 --- a/docker/configure_workers_and_start.py +++ b/docker/configure_workers_and_start.py @@ -517,6 +517,7 @@ def add_worker_roles_to_shared_config( "typing", "push_rules", "thread_subscriptions", + "quarantined_media_changes", } # Worker-type specific sharding config. Now a single worker can fulfill multiple diff --git a/docs/SUMMARY.md b/docs/SUMMARY.md index 980f51d078..709e1a01e2 100644 --- a/docs/SUMMARY.md +++ b/docs/SUMMARY.md @@ -50,6 +50,7 @@ - [Background update controller callbacks](modules/background_update_controller_callbacks.md) - [Account data callbacks](modules/account_data_callbacks.md) - [Add extra fields to client events unsigned section callbacks](modules/add_extra_fields_to_client_events_unsigned.md) + - [Federation callbacks](modules/federation_callbacks.md) - [Media repository callbacks](modules/media_repository_callbacks.md) - [Ratelimit callbacks](modules/ratelimit_callbacks.md) - [Porting a legacy module to the new interface](modules/porting_legacy_module.md) diff --git a/docs/admin_api/scheduled_tasks.md b/docs/admin_api/scheduled_tasks.md index 949a03ee39..7d7c68d987 100644 --- a/docs/admin_api/scheduled_tasks.md +++ b/docs/admin_api/scheduled_tasks.md @@ -31,8 +31,10 @@ It returns a JSON body like the following: **Query parameters:** * `action_name`: string - Is optional. Returns only the scheduled tasks with the given action name. + May be given multiple times to return tasks matching any of the given action names. * `resource_id`: string - Is optional. Returns only the scheduled tasks with the given resource id. -* `status`: string - Is optional. Returns only the scheduled tasks matching the given status, one of +* `status`: string - Is optional. Returns only the scheduled tasks matching the given status. + May be given multiple times to return tasks matching any of the given statuses. The status must be one of - "scheduled" - Task is scheduled but not active - "active" - Task is active and probably running, and if not will be run on next scheduler loop run - "complete" - Task has completed successfully diff --git a/docs/admin_api/user_admin_api.md b/docs/admin_api/user_admin_api.md index 44d0985697..ca82ccbecd 100644 --- a/docs/admin_api/user_admin_api.md +++ b/docs/admin_api/user_admin_api.md @@ -227,7 +227,8 @@ The following parameters should be set in the URL: - `name` - Is optional and filters to only return users with user ID localparts **or** displaynames that contain this value. - `guests` - string representing a bool - Is optional and if `false` will **exclude** guest users. - Defaults to `true` to include guest users. This parameter is not supported when Matrix Authentication Service integration is enabled. [See #15582](https://github.com/matrix-org/synapse/pull/15582) + Defaults to `true` to include guest users. Setting this option to `true` is not supported when Matrix Authentication + Service integration is enabled. [See #15582](https://github.com/matrix-org/synapse/pull/15582) - `admins` - Optional flag to filter admins. If `true`, only admins are queried. If `false`, admins are excluded from the query. When the flag is absent (the default), **both** admins and non-admins are included in the search results. - `deactivated` - string representing a bool - Is optional and if `true` will **include** deactivated users. diff --git a/docs/development/database_schema.md b/docs/development/database_schema.md index 620d1c16b0..25153a06ad 100644 --- a/docs/development/database_schema.md +++ b/docs/development/database_schema.md @@ -84,17 +84,11 @@ reference only. If you want to recreate these schemas, they need to be made from a database that has had all background updates run. -To do so, use `scripts-dev/make_full_schema.sh`. This will produce new -`full.sql.postgres` and `full.sql.sqlite` files. +To do so, use the `make_full_schema.sh` script. This will produce new +`full.sql.postgres` and `full.sql.sqlite` files, after having applied ALL +schema deltas and background updates. -Ensure postgres is installed, then run: - -```sh -./scripts-dev/make_full_schema.sh -p postgres_username -o output_dir/ -``` - -NB at the time of writing, this script predates the split into separate `state`/`main` -databases so will require updates to handle that correctly. +Run `poetry run scripts-dev/make_full_schema.sh -h` for usage instructions. ## Delta files diff --git a/docs/development/synapse_architecture/streams.md b/docs/development/synapse_architecture/streams.md index e7ab79091e..a647082524 100644 --- a/docs/development/synapse_architecture/streams.md +++ b/docs/development/synapse_architecture/streams.md @@ -162,8 +162,9 @@ necessary registration and event handling. - Update `synapse/_scripts/synapse_port_db.py` so it knows about your new `SEQUENCE`: [add a new `_setup_sequence(...)`](https://github.com/element-hq/synapse/blob/35b55e962aa0bed3b2da5a3c12e3783ddf7604ca/synapse/_scripts/synapse_port_db.py#L883C24-L888) - [create a stream class and stream row class](https://github.com/element-hq/synapse/blob/4367fb2d078c52959aeca0fe6874539c53e8360d/synapse/replication/tcp/streams/_base.py#L728) - will need an [ID generator](https://github.com/element-hq/synapse/blob/4367fb2d078c52959aeca0fe6874539c53e8360d/synapse/storage/databases/main/thread_subscriptions.py#L75) - - may need [writer configuration](https://github.com/element-hq/synapse/blob/4367fb2d078c52959aeca0fe6874539c53e8360d/synapse/config/workers.py#L177), if there isn't already an obvious source of configuration for which workers should be designated as writers to your new stream. + - may need [writer configuration](https://github.com/element-hq/synapse/blob/62a4bc46203880dd5034483b0e84156d03a3a8c6/synapse/config/workers.py#L184-L187), if there isn't already an obvious source of configuration for which workers should be designated as writers to your new stream. - if adding new writer configuration, add Docker-worker configuration, which lets us configure the writer worker in Complement tests: [[1]](https://github.com/element-hq/synapse/blob/4367fb2d078c52959aeca0fe6874539c53e8360d/docker/configure_workers_and_start.py#L331), [[2]](https://github.com/element-hq/synapse/blob/4367fb2d078c52959aeca0fe6874539c53e8360d/docker/configure_workers_and_start.py#L440) + - Ensure that it's been correctly added to `synapse/replication/tcp/handler.py` and it's `streams_to_replicate` attribute to ensure that changes are actually replicated. - most of the time, you will likely introduce a new datastore class for the concept represented by the new stream, unless there is already an obvious datastore that covers it. - consider whether it may make sense to introduce a handler diff --git a/docs/modules/federation_callbacks.md b/docs/modules/federation_callbacks.md new file mode 100644 index 0000000000..09154e5640 --- /dev/null +++ b/docs/modules/federation_callbacks.md @@ -0,0 +1,39 @@ +# Federation callbacks + +Federation callbacks can be registered using the module API's `register_federation_callbacks` method. + +## Callbacks + +The available federation callbacks are: + +### `on_event_delivered_over_federation` + +_First introduced in Synapse v1.158.0_ + +```python +async def on_event_delivered_over_federation( + event: FederationEventDeliveryEvent, +) -> None: +``` + +Called when an event has been delivered over federation. +See `FederationEventDeliveryEvent` for detailed information available on the event. + +Note that depending on the specific method, delivery may not have +actually been acknowledged by the other homeserver. +See `FederatedEventDeliveryMethod` for details on which cases imply acknowledgment. + +Modules should anticipate more methods being added to the `FederatedEventDeliveryMethod` enum +over time (it is non-exhaustive). + +Only methods that deliver full, signed PDUs are included in this mechanism. +Some notable examples of excluded endpoints: +- `/send_knock` is excluded as it only returns unsigned 'stripped state'. +- `/timestamp_to_event` is excluded as it only returns event IDs, not events themselves. + +If multiple modules implement this callback, Synapse runs them all in order. +Exceptions are logged and otherwise ignored. + +Performance note: +- Registering this hook causes a performance (caching) optimisation on the + Federation `/state` endpoint to be bypassed. diff --git a/docs/modules/media_repository_callbacks.md b/docs/modules/media_repository_callbacks.md index d7c9074bde..c55d962512 100644 --- a/docs/modules/media_repository_callbacks.md +++ b/docs/modules/media_repository_callbacks.md @@ -116,10 +116,6 @@ may change without notice. Called when a user attempts to upload media that would exceed a [configured media upload limit](../usage/configuration/config_documentation.html#media_upload_limits). -This callback will only be called on workers which handle -[POST /_matrix/media/v3/upload](https://spec.matrix.org/v1.15/client-server-api/#post_matrixmediav3upload) -requests. - This could be used to inform the user that they have reached a media upload limit through some external method. diff --git a/docs/reverse_proxy.md b/docs/reverse_proxy.md index 0e3303df57..fc38c06335 100644 --- a/docs/reverse_proxy.md +++ b/docs/reverse_proxy.md @@ -4,8 +4,10 @@ It is recommended to put a reverse proxy such as [nginx](https://nginx.org/en/docs/http/ngx_http_proxy_module.html), [Apache](https://httpd.apache.org/docs/current/mod/mod_proxy_http.html), [Caddy](https://caddyserver.com/docs/quick-starts/reverse-proxy), -[HAProxy](https://www.haproxy.org/) or -[relayd](https://man.openbsd.org/relayd.8) in front of Synapse. +[HAProxy](https://www.haproxy.org/), +[relayd](https://man.openbsd.org/relayd.8) or +[lighttpd](https://www.lighttpd.net/) +in front of Synapse. This has the advantage of being able to expose the default HTTPS port (443) to Matrix clients without requiring Synapse to bind to a privileged port (port numbers less than 1024), avoiding the need for `CAP_NET_BIND_SERVICE` or running as root. @@ -312,6 +314,88 @@ relay "matrix_federation" { } ``` +### lighttpd +```conf +server.modules = ( + "mod_rewrite", + "mod_redirect", + "mod_access", + "mod_setenv", + "mod_openssl", + "mod_proxy", + "mod_accesslog" +) + +server.username = "lighttpd" +server.groupname = "lighttpd" + +# We set this to "disable" and use IPv6 `[::]` explicitly below, +# in order to listen on all incoming IPv6 addresses. +# +# If you only want to listen on specific IPv6 addresses, set this +# to "enable" and specify said addresses below. +server.use-ipv6 = "disable" + +ssl.pemfile = "/etc/lighttpd/cert+privkey.pem" +ssl.ca-file = "/etc/lighttpd/fullchain.pem" + +# redirect HTTP traffic to HTTPS, same for IPv6 below +$SERVER["socket"] == "0.0.0.0:80" { + url.redirect = ( + "" => "https://${url.authority.noport}${url.path}${qsa}" + ) +} +$SERVER["socket"] == "0.0.0.0:443" { ssl.engine = "enable" } +$SERVER["socket"] == "0.0.0.0:8448" { ssl.engine = "enable" } +$SERVER["socket"] == "[::]:80" { + url.redirect = ( + "" => "https://${url.authority.noport}${url.path}${qsa}" + ) +} +$SERVER["socket"] == "[::]:443" { ssl.engine = "enable" } +$SERVER["socket"] == "[::]:8448" { ssl.engine = "enable" } + + + +# both lighttpd and synapse need permissions for socket r/w +$HTTP["url"] =~ "(/_matrix|_synapse/admin|/_synapse/client)" { + proxy.balance = "hash" + proxy.server = ( + "" => ( + "backend-socket" => ( + "host" => "/var/lib/synapse/main_public.sock", + "port" => 0 + ) + ) + ) + proxy.forwarded = ( + "for" => 1, + "proto" => 1, + "host" => 1, + ) +} +# protect admin access IPv6 ULA only +$HTTP["remoteip"] !="fd00::/8" { + $HTTP["url"] =~ "^/_synapse/admin/" { + url.access-deny = ( "" ) + } +} +``` + +[Delegation](delegate.md) example: +```conf +url.rewrite-once = ( + "^/\.well-known/matrix/client$" => "/.well-known/matrix/client.json", + "^/\.well-known/matrix/server$" => "/.well-known/matrix/server.json" +) + +# This condition intentionally matches the post-rewrite URLs. +$HTTP["url"] =~ "^/\.well-known/matrix/(client|server)\.json$" { + mimetype.assign = ( ".json" => "application/json" ) + setenv.set-response-header = ( "Access-Control-Allow-Origin" => "*" ) +} +``` + ## Health check endpoint diff --git a/docs/upgrade.md b/docs/upgrade.md index 372e9d147b..9eac9c5ebc 100644 --- a/docs/upgrade.md +++ b/docs/upgrade.md @@ -117,6 +117,49 @@ each upgrade are complete before moving on to the next upgrade, to avoid stacking them up. You can monitor the currently running background updates with [the Admin API](usage/administration/admin_api/background_updates.html#status). + +# Upgrading to v1.161.0 + +## Deprecation of `matrix_rtc.livekit_service_url` + +When configuring the MatrixRTC LiveKit transport, the `livekit_service_url` is now +deprecated but should continue to be listed to ensure backwards compatibility with +older clients. A new sibling `url` config property is added that should be set to +your SFU's WebSocket URL. Clients that support `url` will use the Client-Server API +to (indirectly) interact with the LiveKit authorization service. The service needs +to be set up as an application service in order to support these endpoints. See +https://github.com/element-hq/lk-jwt-service and +https://element-hq.github.io/synapse/v1.161/usage/configuration/config_documentation.html#matrix_rtc +for further details. + +# Upgrading to v1.159.0 + +## Change of signing key expiry date for the Debian/Ubuntu package repository (2026) + +Administrators using the Debian/Ubuntu packages from `packages.matrix.org`, +please be aware that we have recently updated the expiry date on the repository's GPG signing key, +but this change must be imported into your keyring. + +If you have the `matrix-org-archive-keyring` package installed and it updates before the current key expires, this should +happen automatically. + +Otherwise, if you see an error similar to `The following signatures were invalid: EXPKEYSIG F473DD4473365DE1`, you +will need to get a fresh copy of the keys. You can do so with: + +```sh +sudo wget -O /usr/share/keyrings/matrix-org-archive-keyring.gpg https://packages.matrix.org/debian/matrix-org-archive-keyring.gpg +``` + +The old version of the key will expire on `2027-03-15`. + +# Upgrading to v1.158.0 + +## Drop support for Ubuntu 25.10 'Questing Quokka', add support for Ubuntu 26.04 'Resolute Raccoon' + +Ubuntu 25.10 'Questing Quokka' [is end-of-life as of +2026-07-01](https://endoflife.date/ubuntu). This release drops support for Ubuntu 25.10, +and in its place adds support for Ubuntu 26.04 'Resolute Raccoon'. + # Upgrading to v1.157.0 ## MSC3861 Auth Delegation must be migrated to stable Matrix Authentication Service integration diff --git a/docs/usage/configuration/config_documentation.md b/docs/usage/configuration/config_documentation.md index ec4098fdf6..4882569b7c 100644 --- a/docs/usage/configuration/config_documentation.md +++ b/docs/usage/configuration/config_documentation.md @@ -336,6 +336,17 @@ Example configuration: include_profile_data_on_invite: false ``` --- +### `include_profile_updates_in_sync` + +*(boolean)* Use this option to include updates of other users' profiles in sync responses, for users who share rooms. +For legacy sync clients, requires [MSC4429](https://github.com/matrix-org/matrix-spec-proposals/pull/4429) compatibility. For sliding sync clients, requires [MSC4262](https://github.com/matrix-org/matrix-spec-proposals/pull/4262) compatibility. Note, profile updates via sync are currently limited to local users only. +This feature is under development and should be used with caution on busy servers or servers which depend on `limit_profile_requests_to_users_who_share_rooms` for ensuring profile information doesn't leak across rooms. Defaults to `false`. + +Example configuration: +```yaml +include_profile_updates_in_sync: true +``` +--- ### `allow_public_rooms_without_auth` *(boolean)* If set to true, removes the need for authentication to access the server's public rooms directory through the client API, meaning that anyone can query the room directory. Defaults to `false`. @@ -363,8 +374,9 @@ Known room versions are listed [here](https://spec.matrix.org/latest/rooms/#comp For example, for room version 1, `default_room_version` should be set to "1". _Changed in Synapse 1.76:_ the default version room version was increased from [9](https://spec.matrix.org/v1.5/rooms/v9/) to [10](https://spec.matrix.org/v1.5/rooms/v10/). +_Changed in Synapse 1.157:_ the default version room version was increased from [10](https://spec.matrix.org/v1.12/rooms/v10/) to [11](https://spec.matrix.org/v1.12/rooms/v11/). -Defaults to `"10"`. +Defaults to `"11"`. Example configuration: ```yaml @@ -1061,6 +1073,21 @@ Example configuration: redaction_retention_period: 28d ``` --- +### `redaction_allowed_period` + +How long after an `m.room.message` was sent a local user is still allowed to redact it. If a local user tries to redact a `m.room.message` older than this period Synapse responds with `403 M_FORBIDDEN` and does not redact the event. + +Only applies to `m.room.message` events redacted by local users. Redactions of other event types and redactions received over federation are unaffected. When the target of the redaction is an edit (`m.replace`), the age and type are taken from the original event and not the edit. + +Set to `null` (the default) to disable, allowing events to be redacted at any time. + +Defaults to `null`. + +Example configuration: +```yaml +redaction_allowed_period: 7d +``` +--- ### `forgotten_room_retention_period` How long to keep locally forgotten rooms before purging them from the DB. A value of `null` means it's disabled. Defaults to `null`. @@ -1275,11 +1302,15 @@ Options related to federation. --- ### `federation_domain_whitelist` -*(array)* Restrict federation to the given whitelist of domains. N.B. we recommend also firewalling your federation listener to limit inbound federation traffic as early as possible, rather than relying purely on this application-layer restriction. If not specified, the default is to whitelist everything. +*(null|array)* Restrict federation to the given whitelist of domains. N.B. we recommend also firewalling your federation listener to limit inbound federation traffic as early as possible, rather than relying purely on this application-layer restriction. + +If specified as an empty list (`[]`), federation will be denied with all servers. Specifying an empty list (`[]`) here is the recommended way of disabling federation. + +If unset or null, allows federation with all servers. Note: this does not stop a server from joining rooms that servers not on the whitelist are in. As such, this option is really only useful to establish a "private federation", where a group of servers all whitelist each other and have the same whitelist. -Defaults to `[]`. +Defaults to `null`. Example configuration: ```yaml @@ -2227,13 +2258,26 @@ These settings can be overridden using the `get_media_upload_limits_for_user` mo Defaults to `[]`. +Options for each entry include: + +* `time_period` (duration): The time period over which the limit applies. Required. + +* `max_size` (byte size): Amount of data that can be uploaded in the time period by the user. Required. + +* `info_uri` (string): URI returned to the client for where the user can find information about the upload limit and how users can reduce their upload usage or request an upload limit increase. Optional. If not set, Synapse serves a built-in page (customisable via the `media_upload_limit_exceeded.html` template) and uses its URL instead. + +* `can_upgrade` (boolean): Value returned to the client for whether the limit can be increased. Defaults to `false`. + Example configuration: ```yaml media_upload_limits: - time_period: 1h max_size: 100M + info_uri: https://example.com/quota#hour - time_period: 1w max_size: 500M + info_uri: https://example.com/quota + can_upgrade: true ``` --- ### `max_image_pixels` @@ -2638,13 +2682,20 @@ This setting has the following sub-options: * `type` (string): The type of transport to use to connect to the selective forwarding unit (SFU). - * `livekit_service_url` (string): The base URL of the LiveKit service. Should only be used with LiveKit-based transports. + * `url` (string): The WebSocket URL of the LiveKit SFU. If type is "livekit", either this or `livekit_service_url` is required. + + Clients that support `url` will use the Client-Server API to (indirectly) interact with the LiveKit authorization service. The service needs to be set up as an application service in order to support these endpoints. See https://github.com/element-hq/lk-jwt-service for further details. + + * `livekit_service_url` (string): Deprecated. The HTTP URL of the LiveKit authorization service. If type is "livekit", either this or `url` is required. + + Clients that don't support `url` will use `livekit_service_url` to directly interact with the LiveKit authorization service. This mode of operation is deprecated and should only be used for backwards compatibility. Example configuration: ```yaml matrix_rtc: transports: - type: livekit + url: wss://livekit.example.com livekit_service_url: https://matrix-rtc.example.com/livekit/jwt ``` --- @@ -3808,7 +3859,7 @@ This setting has the following sub-options: Defaults to `null`. -* `update_profile_information` (boolean): Use this setting to keep a user's profile fields in sync with information from the identity provider. Fields are checked on every SSO login, and are updated if necessary. Note that enabling this option will override user profile information, regardless of whether users have opted-out of syncing that information when first signing in. Fields that will be synced: +* `update_profile_information` (boolean): Use this setting to keep a user's profile fields in sync with information from the identity provider. Fields are checked on every SSO login, and are updated if necessary. Note that enabling this option will override user profile information, regardless of whether users have opted-out of syncing that information when first signing in. Fields that will be synced: * displayname * picture - only if Synapse media repository is running in the main process (i.e. not workerized) and media is stored locally Defaults to `false`. @@ -3938,6 +3989,25 @@ push: jitter_delay: 10s ``` --- +### `push_rules` + +*(object)* Options for push rules + +This setting has the following sub-options: + +* `limits` (object): Limits on the size of push rules that users can have + + This setting has the following sub-options: + + * `rule_count` (integer): This is the total number of push rules that each user can have. Power users may expect to have one push rule per room. Defaults to `10000`. + + * `rule_id_length` (integer): This is the maximum length of a push rule ID, in bytes. Push rule IDs need to be allowed to be at least as long as a room ID (which are [limited to 255 bytes per specification](https://spec.matrix.org/v1.19/appendices/#room-ids)) + It's recommended to leave this option as it is. We expect to remove this option if/when the specification standardises on a limit. Defaults to `300`. + + * `rule_size` (integer): This is the maximum size of a push rule's body, in bytes. + The exact mechanism for calculating this size is currently an implementation detail, subject to change. This limit should be treated as a coarse sanity limit rather than something to fine-tune. + It's recommended to leave this option as it is. We expect to remove this option if/when the specification standardises on a limit and a mechanism for calculating it. Defaults to `1024`. +--- ## Rooms Config options relating to rooms. @@ -3955,6 +4025,8 @@ Possible options are "all", "invite", and "off". They are defined as: Note that this option will only affect rooms created after it is set. It will also not affect rooms created by other servers. +A client may supply its own `m.room.encryption` event in the `initial_state` of its `/createRoom` request. If that event is valid (it specifies an `algorithm` as a string), it takes precedence and this option will not overwrite it, allowing the client to, for example, choose a different encryption algorithm. An empty or otherwise invalid `m.room.encryption` event does not disable forced encryption: the default will still be applied on top of it. + Defaults to `"off"`. Example configuration: @@ -4310,6 +4382,16 @@ exclude_rooms_from_sync: - '!foo:example.com' ``` --- +### `exclude_rooms_from_presence` + +*(array)* A list of rooms to exclude from presence updates. Presence will not be routed between two users solely because they share one of these rooms. Users who also share a non-excluded room continue to exchange presence as normal. Defaults to `[]`. + +Example configuration: +```yaml +exclude_rooms_from_presence: +- '!foo:example.com' +``` +--- ## Opentracing Configuration options related to Opentracing support. diff --git a/docs/workers.md b/docs/workers.md index 92b606607c..8959becea6 100644 --- a/docs/workers.md +++ b/docs/workers.md @@ -572,10 +572,15 @@ configured as stream writer for the `device_lists` stream: ##### The `quarantined_media_changes` stream The `quarantined_media_changes` stream supports multiple writers. The following endpoints -can be handled by any worker, but should be routed directly to one of the workers -configured as stream writer for the `quarantined_media_changes` stream: +must be routed directly to one of the workers configured as stream writer for the +`quarantined_media_changes` stream (which must also be able to run the media +repository, as these endpoints are only registered on media-capable workers): ^/_synapse/admin/v1/quarantine_media/.*$ + ^/_synapse/admin/v1/room/.*/media/quarantine$ + ^/_synapse/admin/v1/user/.*/media/quarantine$ + ^/_synapse/admin/v1/media/quarantine/.*$ + ^/_synapse/admin/v1/media/unquarantine/.*$ #### Restrict outbound federation traffic to a specific set of workers diff --git a/poetry.lock b/poetry.lock index 2d03b641e8..508379acd5 100644 --- a/poetry.lock +++ b/poetry.lock @@ -31,7 +31,7 @@ description = "The ultimate Python library in building OAuth and OpenID Connect optional = true python-versions = ">=3.9" groups = ["main"] -markers = "extra == \"oidc\" or extra == \"jwt\" or extra == \"all\"" +markers = "extra == \"all\" or extra == \"jwt\" or extra == \"oidc\"" files = [ {file = "authlib-1.6.12-py2.py3-none-any.whl", hash = "sha256:e9229ad7fde610b139dd12f5edbe97eab9ee78bfb85691247e767727850b99ab"}, {file = "authlib-1.6.12.tar.gz", hash = "sha256:0656d8482f28fc8221929d5f35b2bde5d13e10555ebc06b4561b0d622e83b1bd"}, @@ -62,7 +62,7 @@ description = "Backport of CPython tarfile module" optional = false python-versions = ">=3.8" groups = ["dev"] -markers = "python_version < \"3.12\" and platform_machine != \"ppc64le\" and platform_machine != \"s390x\"" +markers = "platform_machine != \"ppc64le\" and platform_machine != \"s390x\" and python_version < \"3.12\"" files = [ {file = "backports.tarfile-1.2.0-py3-none-any.whl", hash = "sha256:77e284d754527b01fb1e6fa8a1afe577858ebe4e9dad8919e34c862cb399bc34"}, {file = "backports_tarfile-1.2.0.tar.gz", hash = "sha256:d75e02c268746e1b8144c278978b6e98e85de6ad16f8e4b0844a154557eca991"}, @@ -532,7 +532,7 @@ description = "XML bomb protection for Python stdlib modules" optional = true python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" groups = ["main"] -markers = "extra == \"saml2\" or extra == \"all\"" +markers = "extra == \"all\" or extra == \"saml2\"" files = [ {file = "defusedxml-0.7.1-py2.py3-none-any.whl", hash = "sha256:a352e7e428770286cc899e2542b6cdaedb2b4953ff269a210103ec58f6198a61"}, {file = "defusedxml-0.7.1.tar.gz", hash = "sha256:1bb3032db185915b62d7c6209c5a8792be6a32ab2fedacc84e01b52c51aa3e69"}, @@ -557,7 +557,7 @@ description = "XPath 1.0/2.0/3.0/3.1 parsers and selectors for ElementTree and l optional = true python-versions = ">=3.8" groups = ["main"] -markers = "extra == \"saml2\" or extra == \"all\"" +markers = "extra == \"all\" or extra == \"saml2\"" files = [ {file = "elementpath-4.8.0-py3-none-any.whl", hash = "sha256:5393191f84969bcf8033b05ec4593ef940e58622ea13cefe60ecefbbf09d58d9"}, {file = "elementpath-4.8.0.tar.gz", hash = "sha256:5822a2560d99e2633d95f78694c7ff9646adaa187db520da200a8e9479dc46ae"}, @@ -583,14 +583,14 @@ smmap = ">=3.0.1,<6" [[package]] name = "gitpython" -version = "3.1.50" +version = "3.1.58" description = "GitPython is a Python library used to interact with Git repositories" optional = false python-versions = ">=3.7" groups = ["dev"] files = [ - {file = "gitpython-3.1.50-py3-none-any.whl", hash = "sha256:d352abe2908d07355014abdd21ddf798c2a961469239afec4962e9da884858f9"}, - {file = "gitpython-3.1.50.tar.gz", hash = "sha256:80da2d12504d52e1f998772dc5baf6e553f8d2fcfe1fcc226c9d9a2ee3372dcc"}, + {file = "gitpython-3.1.58-py3-none-any.whl", hash = "sha256:d331e722577f0fd7fc1f857419b3ecc07af66282b933d2a4d95f84a042fdd50f"}, + {file = "gitpython-3.1.58.tar.gz", hash = "sha256:621416df10ef3fd0e19fabf9172ddeed0fa704d353d04f194eec56a625a95b22"}, ] [package.dependencies] @@ -598,7 +598,7 @@ gitdb = ">=4.0.1,<5" [package.extras] doc = ["sphinx (>=7.4.7,<8)", "sphinx-autodoc-typehints", "sphinx_rtd_theme"] -test = ["coverage[toml]", "ddt (>=1.1.1,!=1.4.3)", "mock ; python_version < \"3.8\"", "mypy (==1.18.2) ; python_version >= \"3.9\"", "pre-commit", "pytest (>=7.3.1)", "pytest-cov", "pytest-instafail", "pytest-mock", "pytest-sugar", "typing-extensions ; python_version < \"3.11\""] +test = ["basedpyright (==1.39.9) ; python_version >= \"3.9\" and sys_platform != \"cygwin\"", "coverage[toml]", "ddt (>=1.1.1,!=1.4.3)", "mock ; python_version < \"3.8\"", "mypy (==1.18.2) ; python_version >= \"3.9\"", "pre-commit", "pytest (>=7.3.1)", "pytest-cov", "pytest-instafail", "pytest-mock", "pytest-sugar", "typing-extensions ; python_version < \"3.11\""] [[package]] name = "hiredis" @@ -607,7 +607,7 @@ description = "Python wrapper for hiredis" optional = true python-versions = ">=3.8" groups = ["main"] -markers = "extra == \"redis\" or extra == \"all\"" +markers = "extra == \"all\" or extra == \"redis\"" files = [ {file = "hiredis-3.3.1-cp310-cp310-macosx_10_15_universal2.whl", hash = "sha256:f525734382a47f9828c9d6a1501522c78d5935466d8e2be1a41ba40ca5bb922b"}, {file = "hiredis-3.3.1-cp310-cp310-macosx_10_15_x86_64.whl", hash = "sha256:6e2e1024f0a021777740cb7c633a0efb2c4a4bc570f508223a8dcbcf79f99ef9"}, @@ -890,7 +890,7 @@ description = "Read metadata from Python packages" optional = false python-versions = ">=3.9" groups = ["dev"] -markers = "python_version < \"3.12\" and platform_machine != \"ppc64le\" and platform_machine != \"s390x\"" +markers = "platform_machine != \"ppc64le\" and platform_machine != \"s390x\" and python_version < \"3.12\"" files = [ {file = "importlib_metadata-8.7.1-py3-none-any.whl", hash = "sha256:5a1f80bf1daa489495071efbb095d75a634cf28a8bc299581244063b53176151"}, {file = "importlib_metadata-8.7.1.tar.gz", hash = "sha256:49fef1ae6440c182052f407c8d34a68f72efc36db9ca90dc0113398f2fdde8bb"}, @@ -931,7 +931,7 @@ description = "Jaeger Python OpenTracing Tracer implementation" optional = true python-versions = ">=3.7" groups = ["main"] -markers = "extra == \"opentracing\" or extra == \"all\"" +markers = "extra == \"all\" or extra == \"opentracing\"" files = [ {file = "jaeger-client-4.8.0.tar.gz", hash = "sha256:3157836edab8e2c209bd2d6ae61113db36f7ee399e66b1dcbb715d87ab49bfe0"}, ] @@ -1123,7 +1123,7 @@ description = "A strictly RFC 4510 conforming LDAP V3 pure Python client library optional = true python-versions = "*" groups = ["main"] -markers = "extra == \"matrix-synapse-ldap3\" or extra == \"all\"" +markers = "extra == \"all\" or extra == \"matrix-synapse-ldap3\"" files = [ {file = "ldap3-2.9.1-py2.py3-none-any.whl", hash = "sha256:5869596fc4948797020d3f03b7939da938778a0f9e2009f7a072ccf92b8e8d70"}, {file = "ldap3-2.9.1.tar.gz", hash = "sha256:f3e7fc4718e3f09dda568b57100095e0ce58633bcabbed8667ce3f8fbaa4229f"}, @@ -1240,7 +1240,7 @@ description = "Powerful and Pythonic XML processing library combining libxml2/li optional = true python-versions = ">=3.8" groups = ["main"] -markers = "extra == \"url-preview\" or extra == \"all\"" +markers = "extra == \"all\" or extra == \"url-preview\"" files = [ {file = "lxml-6.1.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:41dcc4c7b10484257cbd6c37b83ddb26df2b0e5aff5ac00d095689015af868ec"}, {file = "lxml-6.1.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:a31286dbb5e74c8e9a5344465b77ab4c5bd511a253b355b5ca2fae7e579fafec"}, @@ -1548,7 +1548,7 @@ description = "An LDAP3 auth provider for Synapse" optional = true python-versions = ">=3.10" groups = ["main"] -markers = "extra == \"matrix-synapse-ldap3\" or extra == \"all\"" +markers = "extra == \"all\" or extra == \"matrix-synapse-ldap3\"" files = [ {file = "matrix_synapse_ldap3-0.4.0-py3-none-any.whl", hash = "sha256:bf080037230d2af5fd3639cb87266de65c1cad7a68ea206278c5b4bf9c1a17f3"}, {file = "matrix_synapse_ldap3-0.4.0.tar.gz", hash = "sha256:cff52ba780170de5e6e8af42863d2648ee23f3bf0a9fea6db52372f9fc00be2b"}, @@ -1833,7 +1833,7 @@ description = "OpenTracing API for Python. See documentation at http://opentraci optional = true python-versions = "*" groups = ["main"] -markers = "extra == \"opentracing\" or extra == \"all\"" +markers = "extra == \"all\" or extra == \"opentracing\"" files = [ {file = "opentracing-2.4.0.tar.gz", hash = "sha256:a173117e6ef580d55874734d1fa7ecb6f3655160b8b8974a2a1e98e5ec9c840d"}, ] @@ -1901,103 +1901,99 @@ files = [ [[package]] name = "pillow" -version = "12.2.0" +version = "12.3.0" description = "Python Imaging Library (fork)" optional = false python-versions = ">=3.10" groups = ["main"] files = [ - {file = "pillow-12.2.0-cp310-cp310-macosx_10_10_x86_64.whl", hash = "sha256:a4e8f36e677d3336f35089648c8955c51c6d386a13cf6ee9c189c5f5bd713a9f"}, - {file = "pillow-12.2.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:2e589959f10d9824d39b350472b92f0ce3b443c0a3442ebf41c40cb8361c5b97"}, - {file = "pillow-12.2.0-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:a52edc8bfff4429aaabdf4d9ee0daadbbf8562364f940937b941f87a4290f5ff"}, - {file = "pillow-12.2.0-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:975385f4776fafde056abb318f612ef6285b10a1f12b8570f3647ad0d74b48ec"}, - {file = "pillow-12.2.0-cp310-cp310-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:bd9c0c7a0c681a347b3194c500cb1e6ca9cab053ea4d82a5cf45b6b754560136"}, - {file = "pillow-12.2.0-cp310-cp310-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:88d387ff40b3ff7c274947ed3125dedf5262ec6919d83946753b5f3d7c67ea4c"}, - {file = "pillow-12.2.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:51c4167c34b0d8ba05b547a3bb23578d0ba17b80a5593f93bd8ecb123dd336a3"}, - {file = "pillow-12.2.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:34c0d99ecccea270c04882cb3b86e7b57296079c9a4aff88cb3b33563d95afaa"}, - {file = "pillow-12.2.0-cp310-cp310-win32.whl", hash = "sha256:b85f66ae9eb53e860a873b858b789217ba505e5e405a24b85c0464822fe88032"}, - {file = "pillow-12.2.0-cp310-cp310-win_amd64.whl", hash = "sha256:673aa32138f3e7531ccdbca7b3901dba9b70940a19ccecc6a37c77d5fdeb05b5"}, - {file = "pillow-12.2.0-cp310-cp310-win_arm64.whl", hash = "sha256:3e080565d8d7c671db5802eedfb438e5565ffa40115216eabb8cd52d0ecce024"}, - {file = "pillow-12.2.0-cp311-cp311-macosx_10_10_x86_64.whl", hash = "sha256:8be29e59487a79f173507c30ddf57e733a357f67881430449bb32614075a40ab"}, - {file = "pillow-12.2.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:71cde9a1e1551df7d34a25462fc60325e8a11a82cc2e2f54578e5e9a1e153d65"}, - {file = "pillow-12.2.0-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:f490f9368b6fc026f021db16d7ec2fbf7d89e2edb42e8ec09d2c60505f5729c7"}, - {file = "pillow-12.2.0-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:8bd7903a5f2a4545f6fd5935c90058b89d30045568985a71c79f5fd6edf9b91e"}, - {file = "pillow-12.2.0-cp311-cp311-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:3997232e10d2920a68d25191392e3a4487d8183039e1c74c2297f00ed1c50705"}, - {file = "pillow-12.2.0-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e74473c875d78b8e9d5da2a70f7099549f9eb37ded4e2f6a463e60125bccd176"}, - {file = "pillow-12.2.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:56a3f9c60a13133a98ecff6197af34d7824de9b7b38c3654861a725c970c197b"}, - {file = "pillow-12.2.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:90e6f81de50ad6b534cab6e5aef77ff6e37722b2f5d908686f4a5c9eba17a909"}, - {file = "pillow-12.2.0-cp311-cp311-win32.whl", hash = "sha256:8c984051042858021a54926eb597d6ee3012393ce9c181814115df4c60b9a808"}, - {file = "pillow-12.2.0-cp311-cp311-win_amd64.whl", hash = "sha256:6e6b2a0c538fc200b38ff9eb6628228b77908c319a005815f2dde585a0664b60"}, - {file = "pillow-12.2.0-cp311-cp311-win_arm64.whl", hash = "sha256:9a8a34cc89c67a65ea7437ce257cea81a9dad65b29805f3ecee8c8fe8ff25ffe"}, - {file = "pillow-12.2.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:2d192a155bbcec180f8564f693e6fd9bccff5a7af9b32e2e4bf8c9c69dbad6b5"}, - {file = "pillow-12.2.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:f3f40b3c5a968281fd507d519e444c35f0ff171237f4fdde090dd60699458421"}, - {file = "pillow-12.2.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:03e7e372d5240cc23e9f07deca4d775c0817bffc641b01e9c3af208dbd300987"}, - {file = "pillow-12.2.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:b86024e52a1b269467a802258c25521e6d742349d760728092e1bc2d135b4d76"}, - {file = "pillow-12.2.0-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:7371b48c4fa448d20d2714c9a1f775a81155050d383333e0a6c15b1123dda005"}, - {file = "pillow-12.2.0-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:62f5409336adb0663b7caa0da5c7d9e7bdbaae9ce761d34669420c2a801b2780"}, - {file = "pillow-12.2.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:01afa7cf67f74f09523699b4e88c73fb55c13346d212a59a2db1f86b0a63e8c5"}, - {file = "pillow-12.2.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:fc3d34d4a8fbec3e88a79b92e5465e0f9b842b628675850d860b8bd300b159f5"}, - {file = "pillow-12.2.0-cp312-cp312-win32.whl", hash = "sha256:58f62cc0f00fd29e64b29f4fd923ffdb3859c9f9e6105bfc37ba1d08994e8940"}, - {file = "pillow-12.2.0-cp312-cp312-win_amd64.whl", hash = "sha256:7f84204dee22a783350679a0333981df803dac21a0190d706a50475e361c93f5"}, - {file = "pillow-12.2.0-cp312-cp312-win_arm64.whl", hash = "sha256:af73337013e0b3b46f175e79492d96845b16126ddf79c438d7ea7ff27783a414"}, - {file = "pillow-12.2.0-cp313-cp313-ios_13_0_arm64_iphoneos.whl", hash = "sha256:8297651f5b5679c19968abefd6bb84d95fe30ef712eb1b2d9b2d31ca61267f4c"}, - {file = "pillow-12.2.0-cp313-cp313-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:50d8520da2a6ce0af445fa6d648c4273c3eeefbc32d7ce049f22e8b5c3daecc2"}, - {file = "pillow-12.2.0-cp313-cp313-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:766cef22385fa1091258ad7e6216792b156dc16d8d3fa607e7545b2b72061f1c"}, - {file = "pillow-12.2.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:5d2fd0fa6b5d9d1de415060363433f28da8b1526c1c129020435e186794b3795"}, - {file = "pillow-12.2.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:56b25336f502b6ed02e889f4ece894a72612fe885889a6e8c4c80239ff6e5f5f"}, - {file = "pillow-12.2.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:f1c943e96e85df3d3478f7b691f229887e143f81fedab9b20205349ab04d73ed"}, - {file = "pillow-12.2.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:03f6fab9219220f041c74aeaa2939ff0062bd5c364ba9ce037197f4c6d498cd9"}, - {file = "pillow-12.2.0-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5cdfebd752ec52bf5bb4e35d9c64b40826bc5b40a13df7c3cda20a2c03a0f5ed"}, - {file = "pillow-12.2.0-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:eedf4b74eda2b5a4b2b2fb4c006d6295df3bf29e459e198c90ea48e130dc75c3"}, - {file = "pillow-12.2.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:00a2865911330191c0b818c59103b58a5e697cae67042366970a6b6f1b20b7f9"}, - {file = "pillow-12.2.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:1e1757442ed87f4912397c6d35a0db6a7b52592156014706f17658ff58bbf795"}, - {file = "pillow-12.2.0-cp313-cp313-win32.whl", hash = "sha256:144748b3af2d1b358d41286056d0003f47cb339b8c43a9ea42f5fea4d8c66b6e"}, - {file = "pillow-12.2.0-cp313-cp313-win_amd64.whl", hash = "sha256:390ede346628ccc626e5730107cde16c42d3836b89662a115a921f28440e6a3b"}, - {file = "pillow-12.2.0-cp313-cp313-win_arm64.whl", hash = "sha256:8023abc91fba39036dbce14a7d6535632f99c0b857807cbbbf21ecc9f4717f06"}, - {file = "pillow-12.2.0-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:042db20a421b9bafecc4b84a8b6e444686bd9d836c7fd24542db3e7df7baad9b"}, - {file = "pillow-12.2.0-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:dd025009355c926a84a612fecf58bb315a3f6814b17ead51a8e48d3823d9087f"}, - {file = "pillow-12.2.0-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:88ddbc66737e277852913bd1e07c150cc7bb124539f94c4e2df5344494e0a612"}, - {file = "pillow-12.2.0-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:d362d1878f00c142b7e1a16e6e5e780f02be8195123f164edf7eddd911eefe7c"}, - {file = "pillow-12.2.0-cp313-cp313t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:2c727a6d53cb0018aadd8018c2b938376af27914a68a492f59dfcaca650d5eea"}, - {file = "pillow-12.2.0-cp313-cp313t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:efd8c21c98c5cc60653bcb311bef2ce0401642b7ce9d09e03a7da87c878289d4"}, - {file = "pillow-12.2.0-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:9f08483a632889536b8139663db60f6724bfcb443c96f1b18855860d7d5c0fd4"}, - {file = "pillow-12.2.0-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:dac8d77255a37e81a2efcbd1fc05f1c15ee82200e6c240d7e127e25e365c39ea"}, - {file = "pillow-12.2.0-cp313-cp313t-win32.whl", hash = "sha256:ee3120ae9dff32f121610bb08e4313be87e03efeadfc6c0d18f89127e24d0c24"}, - {file = "pillow-12.2.0-cp313-cp313t-win_amd64.whl", hash = "sha256:325ca0528c6788d2a6c3d40e3568639398137346c3d6e66bb61db96b96511c98"}, - {file = "pillow-12.2.0-cp313-cp313t-win_arm64.whl", hash = "sha256:2e5a76d03a6c6dcef67edabda7a52494afa4035021a79c8558e14af25313d453"}, - {file = "pillow-12.2.0-cp314-cp314-ios_13_0_arm64_iphoneos.whl", hash = "sha256:3adc9215e8be0448ed6e814966ecf3d9952f0ea40eb14e89a102b87f450660d8"}, - {file = "pillow-12.2.0-cp314-cp314-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:6a9adfc6d24b10f89588096364cc726174118c62130c817c2837c60cf08a392b"}, - {file = "pillow-12.2.0-cp314-cp314-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:6a6e67ea2e6feda684ed370f9a1c52e7a243631c025ba42149a2cc5934dec295"}, - {file = "pillow-12.2.0-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:2bb4a8d594eacdfc59d9e5ad972aa8afdd48d584ffd5f13a937a664c3e7db0ed"}, - {file = "pillow-12.2.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:80b2da48193b2f33ed0c32c38140f9d3186583ce7d516526d462645fd98660ae"}, - {file = "pillow-12.2.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:22db17c68434de69d8ecfc2fe821569195c0c373b25cccb9cbdacf2c6e53c601"}, - {file = "pillow-12.2.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:7b14cc0106cd9aecda615dd6903840a058b4700fcb817687d0ee4fc8b6e389be"}, - {file = "pillow-12.2.0-cp314-cp314-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8cbeb542b2ebc6fcdacabf8aca8c1a97c9b3ad3927d46b8723f9d4f033288a0f"}, - {file = "pillow-12.2.0-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:4bfd07bc812fbd20395212969e41931001fd59eb55a60658b0e5710872e95286"}, - {file = "pillow-12.2.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:9aba9a17b623ef750a4d11b742cbafffeb48a869821252b30ee21b5e91392c50"}, - {file = "pillow-12.2.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:deede7c263feb25dba4e82ea23058a235dcc2fe1f6021025dc71f2b618e26104"}, - {file = "pillow-12.2.0-cp314-cp314-win32.whl", hash = "sha256:632ff19b2778e43162304d50da0181ce24ac5bb8180122cbe1bf4673428328c7"}, - {file = "pillow-12.2.0-cp314-cp314-win_amd64.whl", hash = "sha256:4e6c62e9d237e9b65fac06857d511e90d8461a32adcc1b9065ea0c0fa3a28150"}, - {file = "pillow-12.2.0-cp314-cp314-win_arm64.whl", hash = "sha256:b1c1fbd8a5a1af3412a0810d060a78b5136ec0836c8a4ef9aa11807f2a22f4e1"}, - {file = "pillow-12.2.0-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:57850958fe9c751670e49b2cecf6294acc99e562531f4bd317fa5ddee2068463"}, - {file = "pillow-12.2.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:d5d38f1411c0ed9f97bcb49b7bd59b6b7c314e0e27420e34d99d844b9ce3b6f3"}, - {file = "pillow-12.2.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:5c0a9f29ca8e79f09de89293f82fc9b0270bb4af1d58bc98f540cc4aedf03166"}, - {file = "pillow-12.2.0-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:1610dd6c61621ae1cf811bef44d77e149ce3f7b95afe66a4512f8c59f25d9ebe"}, - {file = "pillow-12.2.0-cp314-cp314t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0a34329707af4f73cf1782a36cd2289c0368880654a2c11f027bcee9052d35dd"}, - {file = "pillow-12.2.0-cp314-cp314t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8e9c4f5b3c546fa3458a29ab22646c1c6c787ea8f5ef51300e5a60300736905e"}, - {file = "pillow-12.2.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:fb043ee2f06b41473269765c2feae53fc2e2fbf96e5e22ca94fb5ad677856f06"}, - {file = "pillow-12.2.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:f278f034eb75b4e8a13a54a876cc4a5ab39173d2cdd93a638e1b467fc545ac43"}, - {file = "pillow-12.2.0-cp314-cp314t-win32.whl", hash = "sha256:6bb77b2dcb06b20f9f4b4a8454caa581cd4dd0643a08bacf821216a16d9c8354"}, - {file = "pillow-12.2.0-cp314-cp314t-win_amd64.whl", hash = "sha256:6562ace0d3fb5f20ed7290f1f929cae41b25ae29528f2af1722966a0a02e2aa1"}, - {file = "pillow-12.2.0-cp314-cp314t-win_arm64.whl", hash = "sha256:aa88ccfe4e32d362816319ed727a004423aab09c5cea43c01a4b435643fa34eb"}, - {file = "pillow-12.2.0-pp311-pypy311_pp73-macosx_10_15_x86_64.whl", hash = "sha256:0538bd5e05efec03ae613fd89c4ce0368ecd2ba239cc25b9f9be7ed426b0af1f"}, - {file = "pillow-12.2.0-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:394167b21da716608eac917c60aa9b969421b5dcbbe02ae7f013e7b85811c69d"}, - {file = "pillow-12.2.0-pp311-pypy311_pp73-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:5d04bfa02cc2d23b497d1e90a0f927070043f6cbf303e738300532379a4b4e0f"}, - {file = "pillow-12.2.0-pp311-pypy311_pp73-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:0c838a5125cee37e68edec915651521191cef1e6aa336b855f495766e77a366e"}, - {file = "pillow-12.2.0-pp311-pypy311_pp73-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4a6c9fa44005fa37a91ebfc95d081e8079757d2e904b27103f4f5fa6f0bf78c0"}, - {file = "pillow-12.2.0-pp311-pypy311_pp73-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:25373b66e0dd5905ed63fa3cae13c82fbddf3079f2c8bf15c6fb6a35586324c1"}, - {file = "pillow-12.2.0-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:bfa9c230d2fe991bed5318a5f119bd6780cda2915cca595393649fc118ab895e"}, - {file = "pillow-12.2.0.tar.gz", hash = "sha256:a830b1a40919539d07806aa58e1b114df53ddd43213d9c8b75847eee6c0182b5"}, + {file = "pillow-12.3.0-cp310-cp310-macosx_10_10_x86_64.whl", hash = "sha256:6c0016e7b354317c4e9e525b937ac8596c38d2d232b419529b9cd7a1cd46e39a"}, + {file = "pillow-12.3.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:bcc33feacfaefce60c12fd500a277533bdc02b10a19f7f6d348763d8140bbba7"}, + {file = "pillow-12.3.0-cp310-cp310-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5594fc43d548a7ed94949d139aa1341b270f1863f11cfd37f5a6c8b778a6b67f"}, + {file = "pillow-12.3.0-cp310-cp310-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f0606c8bf2cdefea14a43530f7657cbbb7ecf1c4222512492ef4a4434a9501ec"}, + {file = "pillow-12.3.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:85f998ea1848bc6757289e739cfbdda3a04adfd58b02fc018ce54d754a5ce468"}, + {file = "pillow-12.3.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:25b9b82bb22e6e2b3cd07b39c68b7b862001226cb3dff7130d1cb914121b39ed"}, + {file = "pillow-12.3.0-cp310-cp310-win32.whl", hash = "sha256:37dc8f7bbb66efe481bb60defacef820c950c24713fb44962ed6aa2a50966de1"}, + {file = "pillow-12.3.0-cp310-cp310-win_amd64.whl", hash = "sha256:300557495eb45ebb8aec96c2da9c4be642fbf7cd937278b4013ba894ea8eb0eb"}, + {file = "pillow-12.3.0-cp310-cp310-win_arm64.whl", hash = "sha256:514435a37670e3e5e08f3945b68718b6ed329bb84367777e16f9f4dfe1e61a0f"}, + {file = "pillow-12.3.0-cp311-cp311-macosx_10_10_x86_64.whl", hash = "sha256:00808c5e14ef63ac5161091d242999076604ff74b883423a11e5d7bbb38bf756"}, + {file = "pillow-12.3.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:37d6d0a00072fd2948eb22bce7e1475f34569d90c87c59f7a2ec59541b77f7a6"}, + {file = "pillow-12.3.0-cp311-cp311-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:bcb46e2f9feff8d06323983bd83ed00c201fdcab3d74973e7072a889b3979fcd"}, + {file = "pillow-12.3.0-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:23d27a3e0307ec2244cc51e7287b919aa68d097504ebe19df4e76a98a3eea5bd"}, + {file = "pillow-12.3.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:4f883547d4b7f0495ebe7056b0cc2aea76094e7a4abc8e933540f3271df27d9c"}, + {file = "pillow-12.3.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:236ff70b9312fb68943c703aa842ca6a758abfa45ac187a5e7c1452e96ef72b5"}, + {file = "pillow-12.3.0-cp311-cp311-win32.whl", hash = "sha256:10e41f0fbf1eec8cfd234b8fe17a4caac7c9d0db4c204d3c173a8f9f6ef3232b"}, + {file = "pillow-12.3.0-cp311-cp311-win_amd64.whl", hash = "sha256:8e95e1385e4998ae9694eeaa4730ba5457ff61185b3a55e2e7bea0880aef452a"}, + {file = "pillow-12.3.0-cp311-cp311-win_arm64.whl", hash = "sha256:ebaea975e03d3141d9d3a507df75c9b3ec90fa9d2ffd07567b3a978d9d790b26"}, + {file = "pillow-12.3.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:ba09209fbe443b4acccebe845d8a138b89a8f4fbaeedd44953490b5315d5e965"}, + {file = "pillow-12.3.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:ffd0c5368496f41b0944be820fcb7a838aa6e623d250b01acf2643939c3f99d7"}, + {file = "pillow-12.3.0-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d9c7f76c0673154f044e9d78c8655fb4213f6ca31a836df48b40fe5d187717b9"}, + {file = "pillow-12.3.0-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:78cb2c6865a35ab8ff8b75fd122f6033b92a62c82801110e48ddd6c936a45d91"}, + {file = "pillow-12.3.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:e491916b378fba47242221bb9ead245211b70d504f495d105d17b14a24b4907c"}, + {file = "pillow-12.3.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:0dd2064cbc55aaec028ef5fbb60fa47bb6c3e7918e07ff17935284b227a9d2df"}, + {file = "pillow-12.3.0-cp312-cp312-win32.whl", hash = "sha256:dbce0b29841537a2fa4a214c2bbf14de3587c9680caa9b4e217568472490b28f"}, + {file = "pillow-12.3.0-cp312-cp312-win_amd64.whl", hash = "sha256:a2b55dd6b2a4c4b7d87ffa56bdb33fdc5fdb9a462173861a7bc097f17d91cb09"}, + {file = "pillow-12.3.0-cp312-cp312-win_arm64.whl", hash = "sha256:331b624368d4f1d069149002f25f44bc61c8919ce8ddb3c45bdad8f6e2d89510"}, + {file = "pillow-12.3.0-cp313-cp313-ios_13_0_arm64_iphoneos.whl", hash = "sha256:21900ce7ba264168cd50defae43cd75d25c833ad4ad6e73ffc5596d12e25ac89"}, + {file = "pillow-12.3.0-cp313-cp313-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:4e8c2a84d977f50b9daed6eeaf3baef67d00d5d74d932288f02cb94518ee3ace"}, + {file = "pillow-12.3.0-cp313-cp313-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:ae26d61dfa7a47befdc7572b521024e8745f3d809bd95ca9505a7bba9ef849ec"}, + {file = "pillow-12.3.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:7a743ff716f746fc19a9557f60dab1600d4613255f8a7aeb3cdde4db7eb15a66"}, + {file = "pillow-12.3.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:d69141514cc30b774ceea5e3ed3a6635c8d8a96edf664689b890f4089111fb35"}, + {file = "pillow-12.3.0-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f7401aebd7f581d7f83a439d87d474999317ee099218e5ad25d125290990ba65"}, + {file = "pillow-12.3.0-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0847a763afefb695bc912d7c131e7e0632d4edc1d8698f58ddabec8e46b8b6d3"}, + {file = "pillow-12.3.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:571b9fcb07b97ef3a492028fb3d2dc0993ca23a06138b0315286566d29ef718a"}, + {file = "pillow-12.3.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:756c768d0c9c2955feb7a56c37ea24aea2e369f8d36a88da270b6a9f19e62b5e"}, + {file = "pillow-12.3.0-cp313-cp313-win32.whl", hash = "sha256:a876864214e136f0eb367788dbd7df045f4806801518e2cfe9e13229cfe06d8f"}, + {file = "pillow-12.3.0-cp313-cp313-win_amd64.whl", hash = "sha256:1cca606cd25738df4ed873d5ad46bbdb3d83b5cbca291f6b4ff13a4df6b0bbe8"}, + {file = "pillow-12.3.0-cp313-cp313-win_arm64.whl", hash = "sha256:b629de27fda84b42cde7edef0d85f13b958b47f6e9bbcbba9b673c562a89bd8b"}, + {file = "pillow-12.3.0-cp314-cp314-ios_13_0_arm64_iphoneos.whl", hash = "sha256:9cf95fe4d0f84c82d282745d9bb08ad9f926efa00be4697e767b814ce40d4330"}, + {file = "pillow-12.3.0-cp314-cp314-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:8728f216dcdb6e6d555cf971cb34076139ad74b31fc2c14da4fafc741c5f6217"}, + {file = "pillow-12.3.0-cp314-cp314-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:a45650e8ce7fafffd731db8550230db6b0d306d181a90b67d3e6bca2f1990930"}, + {file = "pillow-12.3.0-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:ba54cfebe86920a559a7c4d6b9050791c20513650a1952ebe3368c7dc70306f8"}, + {file = "pillow-12.3.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:e158cb00350dc278f3b91551101aa7d12415a66ebf2c91d8d5ac14e56ddd3ad0"}, + {file = "pillow-12.3.0-cp314-cp314-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e9aeb04d6aef139de265b29683e119b638208f88cf73cdd1658aa07221165321"}, + {file = "pillow-12.3.0-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:251bf95b67017e27b13d82f5b326234ca62d70f9cf4c2b9032de2358a3b12c7b"}, + {file = "pillow-12.3.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:fe3cca2e4e8a592be0f269a1ca4835c25199d9f3ce815c8491048f785b0a0198"}, + {file = "pillow-12.3.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:23aceaa007d6172b02c277f0cd359c79492bbb14f7072b4ede9fbcaf20648130"}, + {file = "pillow-12.3.0-cp314-cp314-win32.whl", hash = "sha256:af8d94b0db561cf68b88a267c5c44b49e134f525d0dc2cb7ed413a66bc23559a"}, + {file = "pillow-12.3.0-cp314-cp314-win_amd64.whl", hash = "sha256:fdafc9cce40277e0f7a0feabce0ee50dd2fa1800f3b38015e51296b5e814048d"}, + {file = "pillow-12.3.0-cp314-cp314-win_arm64.whl", hash = "sha256:e91206ee562682b51b98ef4b26a6ef48fd84e15fd4c4bc5ec768eb641d206838"}, + {file = "pillow-12.3.0-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:164b31cd1a0490ab6efae01aa5df49da7061be0af1b30e035b6e9a1bfe34ee6e"}, + {file = "pillow-12.3.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:5afb51d599ea772b8365ae807ae557f18bccfe46ab261fd1c2a9ed700fc6eb17"}, + {file = "pillow-12.3.0-cp314-cp314t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:3edce1d53195db527e0191f84b71d02022de0540bf43a16ed734ed7537b07385"}, + {file = "pillow-12.3.0-cp314-cp314t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:bf16ba1b4d0b6b7c8e534936632270cf70eb00dbe09005bc345b2677b726855c"}, + {file = "pillow-12.3.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:24870b09b224f7ae3c39ed07d10e819d06f8720bc551847b1d623832b5b0e28d"}, + {file = "pillow-12.3.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:30f2aa603c41533cc25c05acd0da21636e84a315768feb631c937177db558931"}, + {file = "pillow-12.3.0-cp314-cp314t-win32.whl", hash = "sha256:4b0a7fe987b14c31ebda6083f74f22b561fd3739bc0ac51e019622e3d72668c7"}, + {file = "pillow-12.3.0-cp314-cp314t-win_amd64.whl", hash = "sha256:962864dc93511324d51ddbb5b9f8731bf71675b93ca612a07441896f4688fb8c"}, + {file = "pillow-12.3.0-cp314-cp314t-win_arm64.whl", hash = "sha256:0740a512dc522224c77d9aa5a8d70d8b7d73fb91f2c21125d8d025d3b8990e45"}, + {file = "pillow-12.3.0-cp315-cp315-ios_13_0_arm64_iphoneos.whl", hash = "sha256:0feb2e9d6ad6c9e3c06effe9d00f3f1e618a6643273576b016f591e9315a7139"}, + {file = "pillow-12.3.0-cp315-cp315-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:9e881fca225083806662a5c43d627d215f258ff43c890f831966c7d7ba9c7402"}, + {file = "pillow-12.3.0-cp315-cp315-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:4998562bf62a445225f22e07c896bb04b35b1b1f2eb6d760584c9c51d7a5f78c"}, + {file = "pillow-12.3.0-cp315-cp315-macosx_10_15_x86_64.whl", hash = "sha256:dc624f6bc473dacdf7ef7eb8678d0d08edf15cd94fad6ae5c7d6cc67a4e4902f"}, + {file = "pillow-12.3.0-cp315-cp315-macosx_11_0_arm64.whl", hash = "sha256:71d6097b330eea8fd15097780c8e89cb1a8ce7838669f48c5bacd6f663dd4701"}, + {file = "pillow-12.3.0-cp315-cp315-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:28ce87c5ab450a9dd970b52e5aca5fe63ed432d18a2eaddd1979a00a1ba24ace"}, + {file = "pillow-12.3.0-cp315-cp315-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:6b02afb9b97f65fbca5f31db6a2a3ba21aa93030225f150fa3f249717e938fb4"}, + {file = "pillow-12.3.0-cp315-cp315-musllinux_1_2_aarch64.whl", hash = "sha256:1182d52bc2d5e5d7d0949503aa7e36d12f42205dc287e4883f407b1988820d39"}, + {file = "pillow-12.3.0-cp315-cp315-musllinux_1_2_x86_64.whl", hash = "sha256:e795b7eb908249c4e43c7c99fac7c2c75dab0c43566e37db472a355f63693d71"}, + {file = "pillow-12.3.0-cp315-cp315-win32.whl", hash = "sha256:57b3d78c95ba9059768b10e28b813002261d3f3dfc55cc48b0c988f625175827"}, + {file = "pillow-12.3.0-cp315-cp315-win_amd64.whl", hash = "sha256:fa4ecea169a355be7a3ade2c783e2ed12f0e40d2c5621cda8b3297faf7fbb9f5"}, + {file = "pillow-12.3.0-cp315-cp315-win_arm64.whl", hash = "sha256:877c3f311ff35410f690861c4409e7ccbf0cd2f878e50628a28e5a0bb689e658"}, + {file = "pillow-12.3.0-cp315-cp315t-macosx_10_15_x86_64.whl", hash = "sha256:e9871b1ffbfa9656b60aeee92ed5136a5742696006fa322b29ea3d8da0ecc9cf"}, + {file = "pillow-12.3.0-cp315-cp315t-macosx_11_0_arm64.whl", hash = "sha256:53aa02d20d10c3d814d536aa4e5ac9b84ca0ff5a88377963b085ad6822f93e64"}, + {file = "pillow-12.3.0-cp315-cp315t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:446c34dcc4324b084a53b705127dc15717b22c5e140ae0a3c38349d4efec071e"}, + {file = "pillow-12.3.0-cp315-cp315t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:cf1845d02ad822a369a49f2bb9345b1614744267682e7a03527dc3bf6eea1777"}, + {file = "pillow-12.3.0-cp315-cp315t-musllinux_1_2_aarch64.whl", hash = "sha256:186941b6aef820ad110fb01fb06eb925374dc3a21b17e37ec9a53b250c6fe2d1"}, + {file = "pillow-12.3.0-cp315-cp315t-musllinux_1_2_x86_64.whl", hash = "sha256:f13c32a3abd6079a66d9526e18dad9b6d280384d49d7c54040cd57b6424041d9"}, + {file = "pillow-12.3.0-cp315-cp315t-win32.whl", hash = "sha256:1657923d2d45afb66526e5b933e5b3052e6bdea196c90d3abb2424e18c77dae8"}, + {file = "pillow-12.3.0-cp315-cp315t-win_amd64.whl", hash = "sha256:8cd2f7bdda092d99c9fc2fb7391354f306d01443d22785d0cbfafa2e2c8bb418"}, + {file = "pillow-12.3.0-cp315-cp315t-win_arm64.whl", hash = "sha256:06ff022112bc9cbf83b60f8e028d94ad87b60621706487e65f673de61610ab59"}, + {file = "pillow-12.3.0-pp311-pypy311_pp73-macosx_10_15_x86_64.whl", hash = "sha256:b3c777e849237620b022f7f297dd67705f9f5cf1685f09f02e46f93e92725468"}, + {file = "pillow-12.3.0-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:b343699e8308bdc51978310e1c959c584e7869cc8c40780058c87da7781a1e94"}, + {file = "pillow-12.3.0-pp311-pypy311_pp73-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:fbd139c8447d25dd750ab79ee274cc5e1fe80fc56340ab10b18a195e1b6eca3e"}, + {file = "pillow-12.3.0-pp311-pypy311_pp73-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e7e480451b9fa137494bccd3a7d69adbe8ac65a87d97be61e11f1b1050a5bac3"}, + {file = "pillow-12.3.0-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:04f01d28a6aaff387bf842a13be313df23ba0597a44f1a976c9feb3c6ff4711a"}, + {file = "pillow-12.3.0.tar.gz", hash = "sha256:3b8182a766685eaa002637e28b4ec8d6b18819a0c71f579bf0dbaa5830297cce"}, ] [package.extras] @@ -2005,7 +2001,7 @@ docs = ["furo", "olefile", "sphinx (>=8.2)", "sphinx-autobuild", "sphinx-copybut fpx = ["olefile"] mic = ["olefile"] test-arrow = ["arro3-compute", "arro3-core", "nanoarrow", "pyarrow"] -tests = ["check-manifest", "coverage (>=7.4.2)", "defusedxml", "markdown2", "olefile", "packaging", "pyroma (>=5)", "pytest", "pytest-cov", "pytest-timeout", "pytest-xdist", "trove-classifiers (>=2024.10.12)"] +tests = ["coverage (>=7.4.2)", "defusedxml", "markdown2", "olefile", "packaging", "pytest", "pytest-cov", "pytest-timeout", "pytest-xdist", "setuptools", "trove-classifiers (>=2024.10.12)"] xmp = ["defusedxml"] [[package]] @@ -2032,7 +2028,7 @@ description = "psycopg2 - Python-PostgreSQL Database Adapter" optional = true python-versions = ">=3.9" groups = ["main"] -markers = "extra == \"postgres\" or extra == \"all\"" +markers = "extra == \"all\" or extra == \"postgres\"" files = [ {file = "psycopg2-2.9.11-cp310-cp310-win_amd64.whl", hash = "sha256:103e857f46bb76908768ead4e2d0ba1d1a130e7b8ed77d3ae91e8b33481813e8"}, {file = "psycopg2-2.9.11-cp311-cp311-win_amd64.whl", hash = "sha256:210daed32e18f35e3140a1ebe059ac29209dd96468f2f7559aa59f75ee82a5cb"}, @@ -2050,7 +2046,7 @@ description = ".. image:: https://travis-ci.org/chtd/psycopg2cffi.svg?branch=mas optional = true python-versions = "*" groups = ["main"] -markers = "platform_python_implementation == \"PyPy\" and (extra == \"postgres\" or extra == \"all\")" +markers = "platform_python_implementation == \"PyPy\" and (extra == \"all\" or extra == \"postgres\")" files = [ {file = "psycopg2cffi-2.9.0.tar.gz", hash = "sha256:7e272edcd837de3a1d12b62185eb85c45a19feda9e62fa1b120c54f9e8d35c52"}, ] @@ -2066,7 +2062,7 @@ description = "A Simple library to enable psycopg2 compatability" optional = true python-versions = "*" groups = ["main"] -markers = "platform_python_implementation == \"PyPy\" and (extra == \"postgres\" or extra == \"all\")" +markers = "platform_python_implementation == \"PyPy\" and (extra == \"all\" or extra == \"postgres\")" files = [ {file = "psycopg2cffi-compat-1.1.tar.gz", hash = "sha256:d25e921748475522b33d13420aad5c2831c743227dc1f1f2585e0fdb5c914e05"}, ] @@ -2076,14 +2072,14 @@ psycopg2 = "*" [[package]] name = "pyasn1" -version = "0.6.3" +version = "0.6.4" description = "Pure-Python implementation of ASN.1 types and DER/BER/CER codecs (X.208)" optional = false python-versions = ">=3.8" groups = ["main"] files = [ - {file = "pyasn1-0.6.3-py3-none-any.whl", hash = "sha256:a80184d120f0864a52a073acc6fc642847d0be408e7c7252f31390c0f4eadcde"}, - {file = "pyasn1-0.6.3.tar.gz", hash = "sha256:697a8ecd6d98891189184ca1fa05d1bb00e2f84b5977c481452050549c8a72cf"}, + {file = "pyasn1-0.6.4-py3-none-any.whl", hash = "sha256:deda9277cfd454080ec40b207fb6df82206a3a2688735233cdcd8d3d565f088b"}, + {file = "pyasn1-0.6.4.tar.gz", hash = "sha256:9c447d8431c947fe4c8febc4ed9e760bc29011a5b01e5c74b67025bd9fb8ce81"}, ] [[package]] @@ -2346,7 +2342,7 @@ description = "A development tool to measure, monitor and analyze the memory beh optional = true python-versions = ">=3.6" groups = ["main"] -markers = "extra == \"cache-memory\" or extra == \"all\"" +markers = "extra == \"all\" or extra == \"cache-memory\"" files = [ {file = "Pympler-1.0.1-py3-none-any.whl", hash = "sha256:d260dda9ae781e1eab6ea15bacb84015849833ba5555f141d2d9b7b7473b307d"}, {file = "Pympler-1.0.1.tar.gz", hash = "sha256:993f1a3599ca3f4fcd7160c7545ad06310c9e12f70174ae7ae8d4e25f6c5d3fa"}, @@ -2478,7 +2474,7 @@ description = "Python implementation of SAML Version 2 Standard" optional = true python-versions = ">=3.9,<4.0" groups = ["main"] -markers = "extra == \"saml2\" or extra == \"all\"" +markers = "extra == \"all\" or extra == \"saml2\"" files = [ {file = "pysaml2-7.5.0-py3-none-any.whl", hash = "sha256:bc6627cc344476a83c757f440a73fda1369f13b6fda1b4e16bca63ffbabb5318"}, {file = "pysaml2-7.5.0.tar.gz", hash = "sha256:f36871d4e5ee857c6b85532e942550d2cf90ea4ee943d75eb681044bbc4f54f7"}, @@ -2503,7 +2499,7 @@ description = "Extensions to the standard Python datetime module" optional = true python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" groups = ["main"] -markers = "extra == \"saml2\" or extra == \"all\"" +markers = "extra == \"all\" or extra == \"saml2\"" files = [ {file = "python-dateutil-2.9.0.post0.tar.gz", hash = "sha256:37dd54208da7e1cd875388217d5e00ebd4179249f90fb72437e91a35459a0ad3"}, {file = "python_dateutil-2.9.0.post0-py2.py3-none-any.whl", hash = "sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427"}, @@ -2531,7 +2527,7 @@ description = "World timezone definitions, modern and historical" optional = true python-versions = "*" groups = ["main"] -markers = "extra == \"saml2\" or extra == \"all\"" +markers = "extra == \"all\" or extra == \"saml2\"" files = [ {file = "pytz-2026.1.post1-py2.py3-none-any.whl", hash = "sha256:f2fd16142fda348286a75e1a524be810bb05d444e5a081f37f7affc635035f7a"}, {file = "pytz-2026.1.post1.tar.gz", hash = "sha256:3378dde6a0c3d26719182142c56e60c7f9af7e968076f31aae569d72a0358ee1"}, @@ -2935,7 +2931,7 @@ description = "Python client for Sentry (https://sentry.io)" optional = true python-versions = ">=3.6" groups = ["main"] -markers = "extra == \"sentry\" or extra == \"all\"" +markers = "extra == \"all\" or extra == \"sentry\"" files = [ {file = "sentry_sdk-2.57.0-py2.py3-none-any.whl", hash = "sha256:812c8bf5ff3d2f0e89c82f5ce80ab3a6423e102729c4706af7413fd1eb480585"}, {file = "sentry_sdk-2.57.0.tar.gz", hash = "sha256:4be8d1e71c32fb27f79c577a337ac8912137bba4bcbc64a4ec1da4d6d8dc5199"}, @@ -3020,24 +3016,24 @@ tests = ["coverage[toml] (>=5.0.2)", "pytest"] [[package]] name = "setuptools" -version = "82.0.0" -description = "Easily download, build, install, upgrade, and uninstall Python packages" +version = "83.0.0" +description = "Most extensible Python build backend with support for C/C++ extension modules" optional = false -python-versions = ">=3.9" +python-versions = ">=3.10" groups = ["main"] files = [ - {file = "setuptools-82.0.0-py3-none-any.whl", hash = "sha256:70b18734b607bd1da571d097d236cfcfacaf01de45717d59e6e04b96877532e0"}, - {file = "setuptools-82.0.0.tar.gz", hash = "sha256:22e0a2d69474c6ae4feb01951cb69d515ed23728cf96d05513d36e42b62b37cb"}, + {file = "setuptools-83.0.0-py3-none-any.whl", hash = "sha256:29b23c360f22f414dc7336bb39178cc7bcbf6021ed2733cde173f09dba19abb3"}, + {file = "setuptools-83.0.0.tar.gz", hash = "sha256:025bccbbf0fa05b6192bc64ae1e7b16e001fd6d6d4d5de03c97b1c1ade523bef"}, ] [package.extras] -check = ["pytest-checkdocs (>=2.4)", "pytest-ruff (>=0.2.1) ; sys_platform != \"cygwin\"", "ruff (>=0.13.0) ; sys_platform != \"cygwin\""] -core = ["importlib_metadata (>=6) ; python_version < \"3.10\"", "jaraco.functools (>=4)", "jaraco.text (>=3.7)", "more_itertools", "more_itertools (>=8.8)", "packaging (>=24.2)", "platformdirs (>=4.2.2)", "tomli (>=2.0.1) ; python_version < \"3.11\"", "wheel (>=0.43.0)"] +check = ["pytest-checkdocs (>=2.14)", "pytest-ruff (>=0.2.1) ; sys_platform != \"cygwin\"", "ruff (>=0.13.0) ; sys_platform != \"cygwin\""] +core = ["importlib_metadata (>=6) ; python_version < \"3.10\"", "jaraco.functools (>=4)", "jaraco.text (>=3.7)", "more_itertools", "more_itertools (>=8.8)", "packaging (>=24.2)", "tomli (>=2.0.1) ; python_version < \"3.11\"", "wheel (>=0.43.0)"] cover = ["pytest-cov"] doc = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "pygments-github-lexers (==0.0.5)", "pyproject-hooks (!=1.1)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-favicon", "sphinx-inline-tabs", "sphinx-lint", "sphinx-notfound-page (>=1,<2)", "sphinx-reredirects", "sphinxcontrib-towncrier", "towncrier (<24.7)"] -enabler = ["pytest-enabler (>=2.2)"] +enabler = ["pytest-enabler (>=3.4)"] test = ["build[virtualenv] (>=1.0.3)", "filelock (>=3.4.0)", "ini2toml[lite] (>=0.14)", "jaraco.develop (>=7.21) ; python_version >= \"3.9\" and sys_platform != \"cygwin\"", "jaraco.envs (>=2.2)", "jaraco.path (>=3.7.2)", "jaraco.test (>=5.5)", "packaging (>=24.2)", "pip (>=19.1)", "pyproject-hooks (!=1.1)", "pytest (>=6,!=8.1.*)", "pytest-home (>=0.5)", "pytest-perf ; sys_platform != \"cygwin\"", "pytest-subprocess", "pytest-timeout", "pytest-xdist (>=3)", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel (>=0.44.0)"] -type = ["importlib_metadata (>=7.0.2) ; python_version < \"3.10\"", "jaraco.develop (>=7.21) ; sys_platform != \"cygwin\"", "mypy (==1.18.*)", "pytest-mypy"] +type = ["importlib_metadata (>=7.0.2) ; python_version < \"3.10\"", "jaraco.develop (>=7.21) ; sys_platform != \"cygwin\"", "mypy (==1.18.*)", "pytest-mypy (>=1.0.1) ; platform_python_implementation != \"PyPy\""] [[package]] name = "setuptools-rust" @@ -3135,7 +3131,7 @@ description = "Tornado IOLoop Backed Concurrent Futures" optional = true python-versions = "*" groups = ["main"] -markers = "extra == \"opentracing\" or extra == \"all\"" +markers = "extra == \"all\" or extra == \"opentracing\"" files = [ {file = "threadloop-1.0.2-py2-none-any.whl", hash = "sha256:5c90dbefab6ffbdba26afb4829d2a9df8275d13ac7dc58dccb0e279992679599"}, {file = "threadloop-1.0.2.tar.gz", hash = "sha256:8b180aac31013de13c2ad5c834819771992d350267bddb854613ae77ef571944"}, @@ -3146,20 +3142,55 @@ tornado = "*" [[package]] name = "thrift" -version = "0.22.0" +version = "0.24.0" description = "Python bindings for the Apache Thrift RPC system" optional = true python-versions = "*" groups = ["main"] -markers = "extra == \"opentracing\" or extra == \"all\"" +markers = "extra == \"all\" or extra == \"opentracing\"" files = [ - {file = "thrift-0.22.0.tar.gz", hash = "sha256:42e8276afbd5f54fe1d364858b6877bc5e5a4a5ed69f6a005b94ca4918fe1466"}, + {file = "thrift-0.24.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:efe85c4508adaf6c9e6f7fac2b1c3c9beb4b39b19c375519966335a70b28e64a"}, + {file = "thrift-0.24.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:a2baaec0c5cd7ba3eace54b26d81fdf0f5a85010468687cf4a131871d65abfed"}, + {file = "thrift-0.24.0-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:cad27a826e86739a79a327e6afae5a9bea36aeed2989c4318fc2943b6cfad095"}, + {file = "thrift-0.24.0-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:9e26ca0f346e5ec2b6be9778573fb2e9d8b3eb162dbce94e61906176158b448b"}, + {file = "thrift-0.24.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:498ae392090d7ae17ab59ebd8dfabda76528eeb22d3890a57c9f226003d7ed6e"}, + {file = "thrift-0.24.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:acbd02baacfa0d2017f85b189ed69cd6baa522785d1ad86476eb7ce8cd16d063"}, + {file = "thrift-0.24.0-cp310-cp310-win_amd64.whl", hash = "sha256:887a10d718d85275da70fe4e9d4740268ae2854f205fb5869909d24cc5576b79"}, + {file = "thrift-0.24.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:873c93d8496cf71589efd3128ddc5af0be350e6c1a0e615475d840eaa54f6124"}, + {file = "thrift-0.24.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:048d768e5822c436e8920b987d89a6a26e4d438ddf2de4b9d3f81444cd03cd04"}, + {file = "thrift-0.24.0-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:2592bb0bf232d0808583626a7a8d71e265851e974c182967d67dde1f16902aee"}, + {file = "thrift-0.24.0-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:ca1b1ba00151565dc4e6673c924debd557aaf25b2790f1570b3430cb35503671"}, + {file = "thrift-0.24.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:afb7977ccb8ecd7b1520a5141de64b58e94712528a26c10ac8f5b4ef220112a5"}, + {file = "thrift-0.24.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:c8d37a4311a87bda4bd2b8e4137eec638c18e3a57d11a1011bd98a8867523860"}, + {file = "thrift-0.24.0-cp311-cp311-win_amd64.whl", hash = "sha256:fbf461351940ddaa85bf8c2ee1754c9cfdd33bb78322e635e1ea5cd3947ae49a"}, + {file = "thrift-0.24.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:306e0fe3c96b300471c19cec60bd6816064fd93d04713488ef07120aff84653b"}, + {file = "thrift-0.24.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:8ac42199ea2a6fc6275b0aeec32ae09e7f9edb43890ff9a1af5e34191fb422cc"}, + {file = "thrift-0.24.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:8cdd5f927eb98d0e8f00ab148b0cb66ae2598a396e907bd2b6febcbdcc46d80c"}, + {file = "thrift-0.24.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:134cd1a0d80f928377808348d1f9f647284ada093573beaa47040bed5507e219"}, + {file = "thrift-0.24.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:518199062feabfc0982767a0f7bceccb4fd1b485654f75e02913837444c3c130"}, + {file = "thrift-0.24.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:d8c5de86af2a44641d423624c71c554a691d9f80c8b87709f15c7f98ccc6aeac"}, + {file = "thrift-0.24.0-cp312-cp312-win_amd64.whl", hash = "sha256:2f82bb16c4f2009dbc4fc9374604f05e998fb33be6a7787e095cea9842ecfa1d"}, + {file = "thrift-0.24.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:ebb8b389142d67554d0de814dd6c1d62b962751f68721537efca429c57b09327"}, + {file = "thrift-0.24.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:18e7693f1bcef45937ad31a02564add55dec754710d42afc8ee3fc26ecf13028"}, + {file = "thrift-0.24.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:937b398c311799fd5eddaeaffbd275510c68ead597eb1897e5e8113542fb2b50"}, + {file = "thrift-0.24.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:b89a83d4e9ae6029e14f6f7c73305044bf02739d729c6aa8025ef3b6faef0ec0"}, + {file = "thrift-0.24.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:6381093f2c54e0f108554004a8cac3f1ef7ba99d6c5a9909c0eb64f450142685"}, + {file = "thrift-0.24.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:28e034658d724aaa66007babb258ef12f0294036812fc449d7ce6fd15b07100f"}, + {file = "thrift-0.24.0-cp313-cp313-win_amd64.whl", hash = "sha256:f4a44391ae1e32817553639b2991b0753d84487259fe87f8111c956c6bdebf43"}, + {file = "thrift-0.24.0-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:7beb05268c76895f7c3130ce747a8a8cf35558fbc7f464f994e20a0c92181cbd"}, + {file = "thrift-0.24.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:f40bdfbaf8d2795b1593bb75b4d9c77831bbe96a1ef59da4634ccc125a17dca5"}, + {file = "thrift-0.24.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:272624d36faa7bee01b94e5da5efc7305d9d3da57d005382c93fcbde8c3edf6a"}, + {file = "thrift-0.24.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:7bc91ae93005bc16c7362edcb58818a547850b9b52fe9b8075cc6b8922bcde2d"}, + {file = "thrift-0.24.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:c05bb4eac921836c12cdf30c237283df8a42aff6540f7970cd72c5959b139970"}, + {file = "thrift-0.24.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:e5da6b391828d46df9471f22181374760a91773b7e07ad7eb44f351c90580662"}, + {file = "thrift-0.24.0-cp314-cp314-win_amd64.whl", hash = "sha256:829db909a053d4064cb9fe574cc1f5994c24d727f61cdf6446ca774cd3ba5268"}, + {file = "thrift-0.24.0.tar.gz", hash = "sha256:9ef601c49e988475ff0e741d8e1b45feec23b48514e524341efc274191f1789c"}, ] [package.extras] -all = ["tornado (>=4.0)", "twisted"] -tornado = ["tornado (>=4.0)"] -twisted = ["twisted"] +all = ["tornado (>=6.3.0)", "twisted (>=24.3.0)", "zope.interface (>=6.1)"] +tornado = ["tornado (>=6.3.0)"] +twisted = ["twisted (>=24.3.0)", "zope.interface (>=6.1)"] [[package]] name = "tomli" @@ -3226,7 +3257,7 @@ description = "Tornado is a Python web framework and asynchronous networking lib optional = true python-versions = ">=3.9" groups = ["main"] -markers = "extra == \"opentracing\" or extra == \"all\"" +markers = "extra == \"all\" or extra == \"opentracing\"" files = [ {file = "tornado-6.5.7-cp39-abi3-macosx_10_9_universal2.whl", hash = "sha256:148b2eb15c2c765a50796172c1e499649b35f30d2e3c3d3e15913cfa56bfb163"}, {file = "tornado-6.5.7-cp39-abi3-macosx_10_9_x86_64.whl", hash = "sha256:9da38de27f1da3b78a966f0dae12b5a1ea9afe72ca805d84ff06508272ddf100"}, @@ -3358,7 +3389,7 @@ description = "non-blocking redis client for python" optional = true python-versions = "*" groups = ["main"] -markers = "extra == \"redis\" or extra == \"all\"" +markers = "extra == \"all\" or extra == \"redis\"" files = [ {file = "txredisapi-1.4.11-py3-none-any.whl", hash = "sha256:ac64d7a9342b58edca13ef267d4fa7637c1aa63f8595e066801c1e8b56b22d0b"}, {file = "txredisapi-1.4.11.tar.gz", hash = "sha256:3eb1af99aefdefb59eb877b1dd08861efad60915e30ad5bf3d5bf6c5cedcdbc6"}, @@ -3619,7 +3650,7 @@ description = "An XML Schema validator and decoder" optional = true python-versions = ">=3.7" groups = ["main"] -markers = "extra == \"saml2\" or extra == \"all\"" +markers = "extra == \"all\" or extra == \"saml2\"" files = [ {file = "xmlschema-2.5.1-py3-none-any.whl", hash = "sha256:ec2b2a15c8896c1fcd14dcee34ca30032b99456c3c43ce793fdb9dca2fb4b869"}, {file = "xmlschema-2.5.1.tar.gz", hash = "sha256:4f7497de6c8b6dc2c28ad7b9ed6e21d186f4afe248a5bea4f54eedab4da44083"}, @@ -3640,7 +3671,7 @@ description = "Backport of pathlib-compatible object wrapper for zip files" optional = false python-versions = ">=3.9" groups = ["dev"] -markers = "python_version < \"3.12\" and platform_machine != \"ppc64le\" and platform_machine != \"s390x\"" +markers = "platform_machine != \"ppc64le\" and platform_machine != \"s390x\" and python_version < \"3.12\"" files = [ {file = "zipp-3.23.0-py3-none-any.whl", hash = "sha256:071652d6115ed432f5ce1d34c336c0adfd6a884660d1e9712a256d3d3bd4b14e"}, {file = "zipp-3.23.0.tar.gz", hash = "sha256:a07157588a12518c9d4034df3fbbee09c814741a33ff63c05fa29d26a2404166"}, diff --git a/pyproject.toml b/pyproject.toml index 6023740d6c..600660bd93 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "matrix-synapse" -version = "1.156.0" +version = "1.160.0" description = "Homeserver for the Matrix decentralised comms protocol" readme = "README.rst" authors = [ diff --git a/rust/Cargo.toml b/rust/Cargo.toml index 612ab09f6d..f6b12c4c1b 100644 --- a/rust/Cargo.toml +++ b/rust/Cargo.toml @@ -31,7 +31,7 @@ http = "1.1.0" lazy_static = "1.4.0" log = "0.4.17" mime = "0.3.17" -pyo3 = { version = "0.28.3", features = [ +pyo3 = { version = "0.29.0", features = [ "macros", "anyhow", "abi3", @@ -40,8 +40,8 @@ pyo3 = { version = "0.28.3", features = [ # https://docs.rs/pyo3/latest/pyo3/bytes/index.html "bytes", ] } -pyo3-log = "0.13.3" -pythonize = { version = "0.28.0", features = ["arbitrary_precision"] } +pyo3-log = "0.13.4" +pythonize = { version = "0.29.0", features = ["arbitrary_precision"] } regex = "1.6.0" sha2 = "0.10.8" serde = { version = "1.0.144", features = ["derive", "rc"] } diff --git a/rust/src/config/mod.rs b/rust/src/config/mod.rs index ef9ecb067c..db00197107 100644 --- a/rust/src/config/mod.rs +++ b/rust/src/config/mod.rs @@ -47,7 +47,8 @@ pub struct AuthConfig { } #[derive(FromPyObject, Clone)] pub struct ServerConfig { - pub max_event_delay_ms: Option, + pub msc4140_enabled: bool, + pub include_profile_updates_in_sync: bool, } #[derive(FromPyObject, Clone)] @@ -72,4 +73,6 @@ pub struct ExperimentalConfig { pub msc4222_enabled: bool, pub msc4491_enabled: bool, pub msc4143_enabled: bool, + pub msc4446_enabled: bool, + pub msc4502_enabled: bool, } diff --git a/rust/src/deferred.rs b/rust/src/deferred.rs index 62a1ce6b90..3da5defb83 100644 --- a/rust/src/deferred.rs +++ b/rust/src/deferred.rs @@ -141,7 +141,7 @@ pub(crate) async fn run_python_awaitable( make_awaitable: F, ) -> PyResult> where - F: for<'py> Fn(Python<'py>) -> PyResult> + Send + 'static, + F: for<'py> Fn(Python<'py>) -> PyResult> + Send + Sync + 'static, { // Resolves when the awaitable completes; carries the resolved value or error. let (tx, rx) = oneshot::channel::>>(); diff --git a/rust/src/handlers/mod.rs b/rust/src/handlers/mod.rs index d560423166..5df019adff 100644 --- a/rust/src/handlers/mod.rs +++ b/rust/src/handlers/mod.rs @@ -41,15 +41,14 @@ impl RustHandlers { // The Twisted reactor, used both to drive our Tokio runtime and to // marshal database work back onto the reactor thread. - let reactor: Py = homeserver.call_method0("get_reactor")?.unbind(); + let reactor = homeserver.call_method0("get_reactor")?.unbind(); // hs.get_datastores().main.db_pool - let db_pool_py: Py = homeserver + let db_pool_py = homeserver .call_method0("get_datastores")? .getattr("main")? - .getattr("db_pool")? - .unbind(); - let db_pool = PythonDatabasePoolWrapper::new(db_pool_py, reactor.clone_ref(py)); + .getattr("db_pool")?; + let db_pool = PythonDatabasePoolWrapper::new(&db_pool_py, reactor.clone_ref(py))?; // Store is shared across all of the handlers so let's use an `Arc` let store = Arc::new(Store { diff --git a/rust/src/handlers/versions.rs b/rust/src/handlers/versions.rs index db8bca82d6..ab4bd9a160 100644 --- a/rust/src/handlers/versions.rs +++ b/rust/src/handlers/versions.rs @@ -251,12 +251,18 @@ pub struct UnstableFeatureMap { /// MSC4169: Backwards-compatible redaction sending using `/send` #[serde(rename = "com.beeper.msc4169")] msc4169: bool, + /// MSC4262: Profile updates for simplified sliding sync. + #[serde(rename = "org.matrix.msc4262")] + msc4262: bool, /// MSC4354: Sticky events #[serde(rename = "org.matrix.msc4354")] msc4354: bool, /// MSC4380: Invite blocking #[serde(rename = "org.matrix.msc4380.stable")] msc4380: bool, + /// MSC4429: Profile updates for legacy /sync. + #[serde(rename = "org.matrix.msc4429")] + msc4429: bool, /// MSC4445: Sync timeline order #[serde(rename = "org.matrix.msc4445.initial_sync_timeline_topological_ordering")] msc4445_initial_sync_timeline_topological_ordering: bool, @@ -266,6 +272,12 @@ pub struct UnstableFeatureMap { /// MSC4143: Matrix RTC transports (LiveKit backend) #[serde(rename = "org.matrix.msc4143")] msc4143_enabled: bool, + /// MSC4446: Allow moving the fully read marker backwards. + #[serde(rename = "com.beeper.msc4446")] + msc4446_enabled: bool, + /// MSC4502: Targeted and unrestricted room member queries + #[serde(rename = "io.element.msc4502")] + msc4502: bool, // Whether new rooms will be set to encrypted or not (based on presets). #[serde(rename = "io.element.e2ee_forced.public")] @@ -304,21 +316,22 @@ pub fn synapse_config_to_global_unstable_feature_map( msc4028: config.experimental.msc4028_push_encrypted_events, msc4108: config.experimental.msc4108_enabled || (config.experimental.msc4108_delegation_endpoint.is_some()), - msc4140: config - .server - .max_event_delay_ms - .is_some_and(|max_event_delay_ms| max_event_delay_ms > 0), + msc4140: config.server.msc4140_enabled, msc3575: config.experimental.msc3575_enabled, msc4133: config.experimental.msc4133_enabled, msc4133_stable: true, msc4155: config.experimental.msc4155_enabled, msc4306: config.experimental.msc4306_enabled, msc4169: config.experimental.msc4169_enabled, + msc4262: config.server.include_profile_updates_in_sync, msc4354: config.experimental.msc4354_enabled, msc4380: true, + msc4429: config.server.include_profile_updates_in_sync, msc4445_initial_sync_timeline_topological_ordering: true, msc4491_enabled: config.experimental.msc4491_enabled, msc4143_enabled: config.experimental.msc4143_enabled, + msc4446_enabled: config.experimental.msc4446_enabled, + msc4502: config.experimental.msc4502_enabled, e2ee_forced_public: config .room .encryption_enabled_by_default_for_room_presets diff --git a/rust/src/storage/db/mod.rs b/rust/src/storage/db/mod.rs index fd40d52f08..175f66e76d 100644 --- a/rust/src/storage/db/mod.rs +++ b/rust/src/storage/db/mod.rs @@ -32,7 +32,7 @@ pub mod python_db_pool; /// It may be invoked multiple times under certain failure modes (serialization /// and deadlock errors), so it is `Fn` rather than `FnOnce`. pub type ErasedInteraction = - Box Fn(&'txn mut dyn Transaction) -> BoxFuture<'txn, ErasedResult> + Send>; + Box Fn(&'txn mut dyn Transaction) -> BoxFuture<'txn, ErasedResult> + Send + Sync>; /// The type-erased *result* of an [`ErasedInteraction`] /// [`DatabasePool::run_interaction_erased`]. @@ -114,6 +114,7 @@ pub trait DatabasePoolExt: DatabasePool { R: Send + 'static, F: for<'txn> Fn(&'txn mut dyn Transaction) -> BoxFuture<'txn, anyhow::Result> + Send + + Sync + 'static, { // Erase the concrete return type `R` into `Box` so we can call diff --git a/rust/src/storage/db/python_db_pool.rs b/rust/src/storage/db/python_db_pool.rs index 5e62b06656..89003d574b 100644 --- a/rust/src/storage/db/python_db_pool.rs +++ b/rust/src/storage/db/python_db_pool.rs @@ -34,7 +34,9 @@ use pyo3::{ exceptions::{PyAssertionError, PyRuntimeError, PyTypeError}, intern, prelude::*, - types::{PyBool, PyCFunction, PyFloat, PyInt, PyList, PyString}, + types::{ + PyBool, PyCFunction, PyFloat, PyInt, PyList, PyString, PyWeakrefMethods, PyWeakrefReference, + }, }; use crate::deferred::run_python_awaitable; @@ -95,23 +97,32 @@ impl DatabaseEngine { /// Wrapper for a `DatabasePool` from the Python side of Synapse. pub struct PythonDatabasePoolWrapper { - /// The underlying Python `DatabasePool` - database_pool_py: Py, + /// A *weak* reference to the underlying Python `DatabasePool`. + /// + /// We use a *weak* reference to ensure the homeserver can cleanly shut down as + /// otherwise we hold onto `DatabasePool` which references the homeserver and keeps + /// the homeserver from being garbage collected. + database_pool_py_ref: Py, - /// The Twisted reactor. We need this to marshal back onto the reactor thread - /// (via `callFromThread`) when starting transactions, since Twisted's thread - /// pool machinery must be driven from there. + /// The Twisted reactor. We need this to marshal database work back onto the + /// reactor thread (via `callFromThread`) when starting transactions, since + /// Twisted's thread pool machinery must be driven from there. + /// + /// A strong reference is fine here: the reactor is a process-global singleton that + /// never gets garbage collected and never points back at the homeserver, so it is + /// not part of any reference cycle. Ideally, we could worry about it but + /// practically probably doesn't matter. reactor: Py, } impl PythonDatabasePoolWrapper { /// Build a wrapper around the Python `DatabasePool` (e.g. /// `hs.get_datastores().main.db_pool`) and the Twisted `reactor`. - pub fn new(database_pool_py: Py, reactor: Py) -> Self { - Self { - database_pool_py, + pub fn new(database_pool: &Bound<'_, PyAny>, reactor: Py) -> PyResult { + Ok(Self { + database_pool_py_ref: PyWeakrefReference::new(database_pool)?.unbind(), reactor, - } + }) } } @@ -193,9 +204,24 @@ impl DatabasePool for PythonDatabasePoolWrapper { )? .unbind(); + // Upgrade our weak reference to the Python `DatabasePool` into a strong + // one for the duration of this interaction. + let database_pool_py = self + .database_pool_py_ref + .bind(py) + .upgrade() + .ok_or_else(|| { + PyRuntimeError::new_err( + "The Python `DatabasePool` has already been dropped \ + (the homeserver is likely shutdown), so we cannot \ + run the database interaction.", + ) + })? + .unbind(); + Ok(( callback, - self.database_pool_py.clone_ref(py), + database_pool_py, self.reactor.clone_ref(py), )) }) diff --git a/schema/synapse-config.schema.yaml b/schema/synapse-config.schema.yaml index 0d1aff071b..68ff696780 100644 --- a/schema/synapse-config.schema.yaml +++ b/schema/synapse-config.schema.yaml @@ -1,5 +1,5 @@ $schema: https://element-hq.github.io/synapse/latest/schema/v1/meta.schema.json -$id: https://element-hq.github.io/synapse/schema/synapse/v1.156/synapse-config.schema.json +$id: https://element-hq.github.io/synapse/schema/synapse/v1.160/synapse-config.schema.json type: object properties: modules: @@ -275,6 +275,23 @@ properties: default: true examples: - false + include_profile_updates_in_sync: + type: boolean + description: >- + Use this option to include updates of other users' profiles in sync responses, + for users who share rooms. + + For legacy sync clients, requires [MSC4429](https://github.com/matrix-org/matrix-spec-proposals/pull/4429) + compatibility. For sliding sync clients, requires + [MSC4262](https://github.com/matrix-org/matrix-spec-proposals/pull/4262) compatibility. Note, profile updates + via sync are currently limited to local users only. + + This feature is under development and should be used with caution on busy servers or + servers which depend on `limit_profile_requests_to_users_who_share_rooms` for ensuring + profile information doesn't leak across rooms. + default: false + examples: + - true allow_public_rooms_without_auth: type: boolean description: @@ -309,7 +326,11 @@ properties: _Changed in Synapse 1.76:_ the default version room version was increased from [9](https://spec.matrix.org/v1.5/rooms/v9/) to [10](https://spec.matrix.org/v1.5/rooms/v10/). - default: "10" + + _Changed in Synapse 1.157:_ the default version room version was increased + from [10](https://spec.matrix.org/v1.12/rooms/v10/) to + [11](https://spec.matrix.org/v1.12/rooms/v11/). + default: "11" examples: - "8" gc_thresholds: @@ -1257,6 +1278,28 @@ properties: default: 7d examples: - 28d + redaction_allowed_period: + oneOf: + - $ref: "#/$defs/duration" + - type: "null" + description: >- + How long after an `m.room.message` was sent a local user is still allowed + to redact it. If a local user tries to redact a `m.room.message` older + than this period Synapse responds with `403 M_FORBIDDEN` + and does not redact the event. + + + Only applies to `m.room.message` events redacted by local users. + Redactions of other event types and redactions received over federation + are unaffected. When the target of the redaction is an edit (`m.replace`), + the age and type are taken from the original event and not the edit. + + + Set to `null` (the default) to disable, allowing events to be redacted at + any time. + default: null + examples: + - 7d forgotten_room_retention_period: oneOf: - $ref: "#/$defs/duration" @@ -1554,13 +1597,19 @@ properties: - myCA2.pem - myCA3.pem federation_domain_whitelist: - type: array + type: ["null", "array"] description: >- Restrict federation to the given whitelist of domains. N.B. we recommend also firewalling your federation listener to limit inbound federation traffic as early as possible, rather than relying purely on this - application-layer restriction. If not specified, the default is to - whitelist everything. + application-layer restriction. + + + If specified as an empty list (`[]`), federation will be denied with all servers. + Specifying an empty list (`[]`) here is the recommended way of disabling federation. + + + If unset or null, allows federation with all servers. Note: this does not stop a server from joining rooms that servers not on @@ -1569,7 +1618,7 @@ properties: each other and have the same whitelist. items: type: string - default: [] + default: null examples: - - lon.example.com - nyc.example.com @@ -2408,7 +2457,7 @@ properties: Enable the local on-disk media storage provider. When disabled, media is stored only in configured `media_storage_providers` and temporary files are used for processing. - + **Warning:** If this option is set to `false` and no `media_storage_providers` are configured, all media requests will return 404 errors as there will be no storage backend available. @@ -2501,20 +2550,42 @@ properties: module API [callback](../../modules/media_repository_callbacks.md#get_media_upload_limits_for_user). default: [] items: - time_period: - type: "#/$defs/duration" - description: >- - The time period over which the limit applies. Required. - max_size: - type: "#/$defs/bytes" - description: >- - Amount of data that can be uploaded in the time period by the user. - Required. + type: object + required: + - time_period + - max_size + properties: + time_period: + $ref: "#/$defs/duration" + description: >- + The time period over which the limit applies. Required. + max_size: + $ref: "#/$defs/bytes" + description: >- + Amount of data that can be uploaded in the time period by the user. + Required. + info_uri: + type: string + description: >- + URI returned to the client for where the user can find information + about the upload limit and how users can reduce their upload usage + or request an upload limit increase. Optional. + If not set, Synapse serves a built-in page (customisable via the + `media_upload_limit_exceeded.html` template) and uses its URL + instead. + can_upgrade: + type: boolean + description: >- + Value returned to the client for whether the limit can be increased. + default: false examples: - - time_period: 1h max_size: 100M + info_uri: https://example.com/quota#hour - time_period: 1w max_size: 500M + info_uri: https://example.com/quota + can_upgrade: true max_image_pixels: $ref: "#/$defs/bytes" description: Maximum number of pixels that will be thumbnailed. @@ -2962,7 +3033,7 @@ properties: examples: - false matrix_rtc: - type: object + type: object description: >- Options related to MatrixRTC. properties: @@ -2977,19 +3048,37 @@ properties: type: string description: The type of transport to use to connect to the selective forwarding unit (SFU). example: livekit + url: + type: string + description: >- + The WebSocket URL of the LiveKit SFU. If type is "livekit", either this or `livekit_service_url` is + required. + + + Clients that support `url` will use the Client-Server API to (indirectly) interact with the LiveKit + authorization service. The service needs to be set up as an application service in order to support + these endpoints. See https://github.com/element-hq/lk-jwt-service for further details. + example: + wss://livekit.example.com livekit_service_url: type: string description: >- - The base URL of the LiveKit service. Should only be used with LiveKit-based transports. + Deprecated. The HTTP URL of the LiveKit authorization service. If type is "livekit", either this or `url` is + required. + + + Clients that don't support `url` will use `livekit_service_url` to directly interact with the LiveKit + authorization service. This mode of operation is deprecated and should only be used for backwards + compatibility. example: https://matrix-rtc.example.com/livekit/jwt - description: - A list of transport types and arguments to use for MatrixRTC connections. + description: A list of transport types and arguments to use for MatrixRTC connections. default: [] default: {} examples: - transports: - - type: livekit - livekit_service_url: https://matrix-rtc.example.com/livekit/jwt + - type: livekit + url: wss://livekit.example.com + livekit_service_url: https://matrix-rtc.example.com/livekit/jwt enable_registration: type: boolean description: >- @@ -4664,10 +4753,10 @@ properties: type: boolean description: >- Use this setting to keep a user's profile fields in sync with - information from the identity provider. Fields are checked on every - SSO login, and are updated if necessary. Note that enabling this - option will override user profile information, regardless of whether - users have opted-out of syncing that information when first signing + information from the identity provider. Fields are checked on every + SSO login, and are updated if necessary. Note that enabling this + option will override user profile information, regardless of whether + users have opted-out of syncing that information when first signing in. Fields that will be synced: * displayname @@ -4879,6 +4968,45 @@ properties: include_content: false group_unread_count_by_room: false jitter_delay: 10s + push_rules: + type: object + description: Options for push rules + properties: + limits: + type: object + description: Limits on the size of push rules that users can have + properties: + rule_count: + type: integer + default: 10000 + description: >- + This is the total number of push rules that each user can have. + Power users may expect to have one push rule per room. + rule_id_length: + type: integer + default: 300 + description: >- + This is the maximum length of a push rule ID, in bytes. + Push rule IDs need to be allowed to be at least as long + as a room ID (which are [limited to 255 bytes per specification](https://spec.matrix.org/v1.19/appendices/#room-ids)) + + It's recommended to leave this option as it is. + We expect to remove this option if/when the specification standardises on + a limit. + rule_size: + type: integer + default: 1024 + description: >- + This is the maximum size of a push rule's body, in bytes. + + The exact mechanism for calculating this size is currently an implementation + detail, subject to change. + This limit should be treated as a coarse sanity limit rather than something + to fine-tune. + + It's recommended to leave this option as it is. + We expect to remove this option if/when the specification standardises on + a limit and a mechanism for calculating it. encryption_enabled_by_default_for_room_type: type: string description: >- @@ -4899,6 +5027,15 @@ properties: Note that this option will only affect rooms created after it is set. It will also not affect rooms created by other servers. + + + A client may supply its own `m.room.encryption` event in the + `initial_state` of its `/createRoom` request. If that event is valid (it + specifies an `algorithm` as a string), it takes precedence and this option + will not overwrite it, allowing the client to, for example, choose a + different encryption algorithm. An empty or otherwise invalid + `m.room.encryption` event does not disable forced encryption: the default + will still be applied on top of it. enum: - all - invite @@ -5354,6 +5491,18 @@ properties: default: [] examples: - - "!foo:example.com" + exclude_rooms_from_presence: + type: array + description: >- + A list of rooms to exclude from presence updates. Presence will not be + routed between two users solely because they share one of these rooms. + Users who also share a non-excluded room continue to exchange presence as + normal. + items: + type: string + default: [] + examples: + - - "!foo:example.com" opentracing: type: object description: >- diff --git a/scripts-dev/build_debian_packages.py b/scripts-dev/build_debian_packages.py index cdde81ee2c..bd5c890711 100755 --- a/scripts-dev/build_debian_packages.py +++ b/scripts-dev/build_debian_packages.py @@ -27,12 +27,12 @@ from typing import Collection, Sequence # to remove references to the distibution across Synapse (search for "bookworm" for # example) DISTS = ( - "debian:bookworm", # (EOL 2026-06) (our EOL forced by Python 3.11 is 2027-10-24) + "debian:bookworm", # (EOL 2028-06-30) (our EOL forced by Python 3.11 is 2027-10-31) "debian:sid", # (rolling distro, no EOL) - "ubuntu:jammy", # 22.04 LTS (EOL 2027-04) (our EOL forced by Python 3.10 is 2026-10-04) - "ubuntu:noble", # 24.04 LTS (EOL 2029-06) - "ubuntu:questing", # 25.10 (EOL 2026-07) - "debian:trixie", # (EOL not specified yet) + "ubuntu:jammy", # 22.04 LTS (EOL 2027-04-01) (our EOL forced by Python 3.10 is 2026-10-31) + "ubuntu:noble", # 24.04 LTS (EOL 2029-05-31) (our EOL forced by Python 3.12 is 2028-10-31) + "debian:trixie", # (EOL 2030-06-30) (our EOL forced by Python 3.13 is 2029-10-31) + "ubuntu:resolute", # 26.04 (EOL 2031-04-30) (our EOL forced by Python 3.14 is 2030-10-31) ) DESC = """\ diff --git a/scripts-dev/check_schema_delta.py b/scripts-dev/check_schema_delta.py index 12ed5d258c..7f2500ec05 100755 --- a/scripts-dev/check_schema_delta.py +++ b/scripts-dev/check_schema_delta.py @@ -14,6 +14,11 @@ import sqlglot.expressions SCHEMA_FILE_REGEX = re.compile(r"^synapse/storage/schema/(.*)/delta/(.*)/(.*)$") +# Keep this in sync with synapse.storage.engines._base. The CI job for this +# script deliberately installs only its lightweight parsing dependencies, so we +# avoid importing Synapse here. +AUTO_INCREMENT_PRIMARY_KEYPLACEHOLDER = "$%AUTO_INCREMENT_PRIMARY_KEY%$" + # The base branch we want to check against. We use the main development branch # on the assumption that is what we are developing against. DEVELOP_BRANCH = "develop" @@ -81,7 +86,7 @@ def main(force_colors: bool) -> None: bad_delta_files = [] changed_delta_files = [] for diff in diffs: - if diff.b_path is None: + if diff.deleted_file or diff.b_path is None: # We don't lint deleted files. continue @@ -196,6 +201,10 @@ def check_schema_delta(delta_files: list[str], force_colors: bool) -> bool: ) return True + delta_contents = _replace_auto_increment_primary_key_placeholder( + delta_contents, sql_lang + ) + statements = sqlglot.parse(delta_contents, read=sql_lang) for statement in statements: @@ -244,5 +253,18 @@ def check_schema_delta(delta_files: list[str], force_colors: bool) -> bool: return success +def _replace_auto_increment_primary_key_placeholder( + delta_contents: str, sql_lang: str +) -> str: + """Replace Synapse's auto-increment PK placeholder with parseable SQL.""" + + if sql_lang == "sqlite": + replacement = "INTEGER PRIMARY KEY AUTOINCREMENT" + else: + replacement = "BIGINT PRIMARY KEY GENERATED ALWAYS AS IDENTITY" + + return delta_contents.replace(AUTO_INCREMENT_PRIMARY_KEYPLACEHOLDER, replacement) + + if __name__ == "__main__": main() diff --git a/scripts-dev/complement.sh b/scripts-dev/complement.sh index cca87d42a9..d8e553f446 100755 --- a/scripts-dev/complement.sh +++ b/scripts-dev/complement.sh @@ -286,6 +286,7 @@ main() { ./tests/msc4155 ./tests/msc4306 ./tests/msc4222 + ./tests/msc4429 ) # Export the list of test packages as a space-separated environment variable, so other diff --git a/scripts-dev/make_full_schema.sh b/scripts-dev/make_full_schema.sh index 473f54772a..3a40b02ff8 100755 --- a/scripts-dev/make_full_schema.sh +++ b/scripts-dev/make_full_schema.sh @@ -7,11 +7,14 @@ export PGHOST="localhost" POSTGRES_MAIN_DB_NAME="synapse_full_schema_main.$$" POSTGRES_COMMON_DB_NAME="synapse_full_schema_common.$$" POSTGRES_STATE_DB_NAME="synapse_full_schema_state.$$" -REQUIRED_DEPS=("matrix-synapse" "psycopg2") + +# Python package names that must be importable +REQUIRED_DEPS=("synapse" "sqlite3" "psycopg2") usage() { echo echo "Usage: $0 -p -o [-c] [-n ] [-h]" + echo "It is the caller's responsibility to be in the correct Python environment (e.g. using \`poetry run scripts-dev/make_full_schema.sh\`)." echo echo "-p " echo " Username to connect to local postgres instance. The password will be requested" @@ -24,14 +27,19 @@ usage() { echo "-n " echo " Schema number for the new snapshot. Used to set the location of files within " echo " the output directory, mimicking that of synapse/storage/schemas." + echo " NOTE: This does not influence which schema deltas are applied to build the full schema." + echo " Schema deltas past this version will still be applied; if you want to build a" + echo " full schema at a particular version, later deltas first need to be" + echo " temporarily deleted (as well as homeserver code temporarily tweaked" + echo " not to rely on any of those changes when applying background updates)" echo " Defaults to 9999." echo "-h" echo " Display this help text." echo "" echo "" echo "You probably want to invoke this with something like" - echo " docker run --rm -e POSTGRES_PASSWORD=postgres -e POSTGRES_USER=postgres -e POSTGRES_DB=synapse -p 5432:5432 postgres:11-alpine" - echo " echo postgres | scripts-dev/make_full_schema.sh -p postgres -n MY_SCHEMA_NUMBER -o synapse/storage/schema" + echo " docker run --rm -e POSTGRES_PASSWORD=postgres -e POSTGRES_USER=postgres -e POSTGRES_DB=synapse -p 5432:5432 postgres:14-alpine" + echo " echo postgres | poetry run scripts-dev/make_full_schema.sh -p postgres -n MY_SCHEMA_NUMBER -o synapse/storage/schema" echo "" echo " NB: make sure to run this against the *oldest* supported version of postgres," echo " or else pg_dump might output non-backwards-compatible syntax." @@ -69,10 +77,10 @@ done # Check that required dependencies are installed unsatisfied_requirements=() for dep in "${REQUIRED_DEPS[@]}"; do - pip show "$dep" --quiet || unsatisfied_requirements+=("$dep") + python -c "import $dep" &> /dev/null || unsatisfied_requirements+=("$dep") done if [ ${#unsatisfied_requirements} -ne 0 ]; then - echo "Please install the following python packages: ${unsatisfied_requirements[*]}" + echo 'Please `poetry install --extras postgres` first as the following Python modules are not importable: '"${unsatisfied_requirements[*]}" exit 1 fi @@ -232,34 +240,33 @@ psql "$POSTGRES_MAIN_DB_NAME" -w <<< "$DROP_COMMON_TABLES" psql "$POSTGRES_STATE_DB_NAME" -w <<< "$DROP_COMMON_TABLES" # For Reasons(TM), SQLite's `.schema` also dumps out "shadow tables", the implementation -# details behind full text search tables. Omit these from the dumps. - -sqlite3 "$SQLITE_MAIN_DB" <<< " -DROP TABLE event_search_content; -DROP TABLE event_search_segments; -DROP TABLE event_search_segdir; -DROP TABLE event_search_docsize; -DROP TABLE event_search_stat; -DROP TABLE user_directory_search_content; -DROP TABLE user_directory_search_segments; -DROP TABLE user_directory_search_segdir; -DROP TABLE user_directory_search_docsize; -DROP TABLE user_directory_search_stat; -" +# details behind full text search tables. +# Previously we omitted these from the dumps by dropping them beforehand, +# but nowadays it seems to be forbidden to drop those. +# The emitted dump adds `IF NOT EXISTS` text for them, so it's harmless. echo "Dumping SQLite3 schema..." mkdir -p "$OUTPUT_DIR/"{common,main,state}"/full_schemas/$SCHEMA_NUMBER" -sqlite3 "$SQLITE_COMMON_DB" ".schema" > "$OUTPUT_DIR/common/full_schemas/$SCHEMA_NUMBER/full.sql.sqlite" +# `--nosys` prevents emitting (some) SQLite internal tables like `sqlite_sequence` that +# aren't managed by the Synapse application and don't need to be part of our full schema. +# +# (SQLite creates `sqlite_sequence` automatically when the database contains at least one +# table with an AUTOINCREMENT column.) +sqlite3 "$SQLITE_COMMON_DB" ".schema --nosys" > "$OUTPUT_DIR/common/full_schemas/$SCHEMA_NUMBER/full.sql.sqlite" sqlite3 "$SQLITE_COMMON_DB" ".dump --data-only --nosys" >> "$OUTPUT_DIR/common/full_schemas/$SCHEMA_NUMBER/full.sql.sqlite" -sqlite3 "$SQLITE_MAIN_DB" ".schema" > "$OUTPUT_DIR/main/full_schemas/$SCHEMA_NUMBER/full.sql.sqlite" +sqlite3 "$SQLITE_MAIN_DB" ".schema --nosys" > "$OUTPUT_DIR/main/full_schemas/$SCHEMA_NUMBER/full.sql.sqlite" sqlite3 "$SQLITE_MAIN_DB" ".dump --data-only --nosys" >> "$OUTPUT_DIR/main/full_schemas/$SCHEMA_NUMBER/full.sql.sqlite" -sqlite3 "$SQLITE_STATE_DB" ".schema" > "$OUTPUT_DIR/state/full_schemas/$SCHEMA_NUMBER/full.sql.sqlite" +sqlite3 "$SQLITE_STATE_DB" ".schema --nosys" > "$OUTPUT_DIR/state/full_schemas/$SCHEMA_NUMBER/full.sql.sqlite" sqlite3 "$SQLITE_STATE_DB" ".dump --data-only --nosys" >> "$OUTPUT_DIR/state/full_schemas/$SCHEMA_NUMBER/full.sql.sqlite" cleanup_pg_schema() { # Cleanup as follows: # - Remove empty lines. pg_dump likes to output a lot of these. + # - Remove the `\restrict` and `\unrestrict` psql meta-commands. + # We don't run the pg_dump output through psql so no meta-commands are + # supported and so the security feature (which doesn't apply anyway + # as we trust the source database) is not relevant. # - Remove comment-only lines. pg_dump also likes to output a lot of these to visually # separate tables etc. # - Remove "public." prefix --- the schema name. @@ -284,6 +291,8 @@ cleanup_pg_schema() { # is `true` or omitted, this marks the given integer as having been consumed and # will NOT appear as the nextval. sed -e '/^$/d' \ + -e '/^\\restrict REMOVEME$/d' \ + -e '/^\\unrestrict REMOVEME$/d' \ -e '/^--/d' \ -e 's/public\.//g' \ -e '/^SET /d' \ @@ -292,12 +301,13 @@ cleanup_pg_schema() { echo "Dumping Postgres schema..." -pg_dump --format=plain --schema-only --no-tablespaces --no-acl --no-owner "$POSTGRES_COMMON_DB_NAME" | cleanup_pg_schema > "$OUTPUT_DIR/common/full_schemas/$SCHEMA_NUMBER/full.sql.postgres" -pg_dump --format=plain --data-only --inserts --no-tablespaces --no-acl --no-owner "$POSTGRES_COMMON_DB_NAME" | cleanup_pg_schema >> "$OUTPUT_DIR/common/full_schemas/$SCHEMA_NUMBER/full.sql.postgres" -pg_dump --format=plain --schema-only --no-tablespaces --no-acl --no-owner "$POSTGRES_MAIN_DB_NAME" | cleanup_pg_schema > "$OUTPUT_DIR/main/full_schemas/$SCHEMA_NUMBER/full.sql.postgres" -pg_dump --format=plain --data-only --inserts --no-tablespaces --no-acl --no-owner "$POSTGRES_MAIN_DB_NAME" | cleanup_pg_schema >> "$OUTPUT_DIR/main/full_schemas/$SCHEMA_NUMBER/full.sql.postgres" -pg_dump --format=plain --schema-only --no-tablespaces --no-acl --no-owner "$POSTGRES_STATE_DB_NAME" | cleanup_pg_schema > "$OUTPUT_DIR/state/full_schemas/$SCHEMA_NUMBER/full.sql.postgres" -pg_dump --format=plain --data-only --inserts --no-tablespaces --no-acl --no-owner "$POSTGRES_STATE_DB_NAME" | cleanup_pg_schema >> "$OUTPUT_DIR/state/full_schemas/$SCHEMA_NUMBER/full.sql.postgres" +# --restrict-key: set the \restrict key to a static value (normally a random string) for easy find/replacement in the code above. +pg_dump --restrict-key=REMOVEME --format=plain --schema-only --no-tablespaces --no-acl --no-owner "$POSTGRES_COMMON_DB_NAME" | cleanup_pg_schema > "$OUTPUT_DIR/common/full_schemas/$SCHEMA_NUMBER/full.sql.postgres" +pg_dump --restrict-key=REMOVEME --format=plain --data-only --inserts --no-tablespaces --no-acl --no-owner "$POSTGRES_COMMON_DB_NAME" | cleanup_pg_schema >> "$OUTPUT_DIR/common/full_schemas/$SCHEMA_NUMBER/full.sql.postgres" +pg_dump --restrict-key=REMOVEME --format=plain --schema-only --no-tablespaces --no-acl --no-owner "$POSTGRES_MAIN_DB_NAME" | cleanup_pg_schema > "$OUTPUT_DIR/main/full_schemas/$SCHEMA_NUMBER/full.sql.postgres" +pg_dump --restrict-key=REMOVEME --format=plain --data-only --inserts --no-tablespaces --no-acl --no-owner "$POSTGRES_MAIN_DB_NAME" | cleanup_pg_schema >> "$OUTPUT_DIR/main/full_schemas/$SCHEMA_NUMBER/full.sql.postgres" +pg_dump --restrict-key=REMOVEME --format=plain --schema-only --no-tablespaces --no-acl --no-owner "$POSTGRES_STATE_DB_NAME" | cleanup_pg_schema > "$OUTPUT_DIR/state/full_schemas/$SCHEMA_NUMBER/full.sql.postgres" +pg_dump --restrict-key=REMOVEME --format=plain --data-only --inserts --no-tablespaces --no-acl --no-owner "$POSTGRES_STATE_DB_NAME" | cleanup_pg_schema >> "$OUTPUT_DIR/state/full_schemas/$SCHEMA_NUMBER/full.sql.postgres" if [[ "$OUTPUT_DIR" == *synapse/storage/schema ]]; then echo "Updating contrib/datagrip symlinks..." diff --git a/scripts-dev/mypy_synapse_plugin.py b/scripts-dev/mypy_synapse_plugin.py index 7fe4d6cd86..ac1fee7f67 100644 --- a/scripts-dev/mypy_synapse_plugin.py +++ b/scripts-dev/mypy_synapse_plugin.py @@ -45,6 +45,7 @@ from mypy.types import ( AnyType, CallableType, Instance, + LiteralType, NoneType, Options, TupleType, @@ -813,6 +814,10 @@ def is_cacheable( if isinstance(rt, AnyType): return True, ("may be mutable" if verbose else None) + elif isinstance(rt, LiteralType): + # Literal[True] etc + return True, None + elif isinstance(rt, Instance): if ( rt.type.fullname in IMMUTABLE_VALUE_TYPES diff --git a/scripts-dev/release.py b/scripts-dev/release.py index ea4fb0f142..58d36f7dcc 100755 --- a/scripts-dev/release.py +++ b/scripts-dev/release.py @@ -31,6 +31,7 @@ import sys import time import urllib.request from os import path +from pathlib import Path from tempfile import TemporaryDirectory from typing import Any @@ -257,10 +258,16 @@ def _prepare() -> None: subprocess.check_output(["poetry", "version", new_version]) # Update config schema $id. - schema_file = "schema/synapse-config.schema.yaml" + schema_file_path = Path("schema/synapse-config.schema.yaml") major_minor_version = ".".join(new_version.split(".")[:2]) url = f"https://element-hq.github.io/synapse/schema/synapse/v{major_minor_version}/synapse-config.schema.json" - subprocess.check_output(["sed", "-i", f"0,/^\\$id: .*/s||$id: {url}|", schema_file]) + # Find/replace the `$id: ...` line in `schema/synapse-config.schema.yaml` with a new + # unique identifier for this release + schema_file_content = schema_file_path.read_text() + new_schema_file_content = re.sub( + r"^\$id: .*", f"$id: {url}", schema_file_content, count=1, flags=re.MULTILINE + ) + schema_file_path.write_text(new_schema_file_content) # Generate changelogs. generate_and_write_changelog(synapse_repo, current_version, new_version) @@ -366,19 +373,10 @@ def _tag(gh_token: str | None) -> None: ) click.get_current_context().abort() - # Get the appropriate changelogs and tag. - changes = get_changes_for_version(current_version) + # We simply point to the changelog instead of duplicating the content into the git tag/release + tag_message = f"Changelog: https://github.com/element-hq/synapse/blob/{repo.active_branch.name}/CHANGES.md" - click.echo_via_pager(changes) - if click.confirm("Edit text?", default=False): - edited_changes = click.edit(changes, require_save=False) - # This assert is for mypy's benefit. click's docs are a little unclear, but - # when `require_save=False`, not saving the temp file in the editor returns - # the original string. - assert edited_changes is not None - changes = edited_changes - - repo.create_tag(tag_name, message=changes, sign=True) + repo.create_tag(tag_name, message=tag_message, sign=True) if not click.confirm("Push tag to GitHub?", default=True): print("") @@ -413,7 +411,7 @@ def _tag(gh_token: str | None) -> None: release = gh_repo.create_git_release( tag=tag_name, name=tag_name, - message=changes, + message=tag_message, draft=True, prerelease=current_version.is_prerelease, ) @@ -602,9 +600,15 @@ def _wait_for_actions(gh_token: str | None) -> None: headers["authorization"] = f"token {gh_token}" req = urllib.request.Request(url, headers=headers) + # Initially, wait 10 minutes as we know the CI typically takes 15m+ anyway (no need + # to check over and over when we know it won't be finished yet) time.sleep(10 * 60) while True: - time.sleep(5 * 60) + # Then check once every minute. Short enough to not have to wait around too long + # while not spamming the GitHub API and running into the unauthenticated API + # request rate limit (60 requests per hour so 1 request/minute perfectly aligns + # to not run into any problems) + time.sleep(1 * 60) response = urllib.request.urlopen(req) resp = json.loads(response.read()) @@ -739,6 +743,7 @@ def _announce() -> None: """Generate markdown to announce the release.""" current_version = get_package_version() + release_branch_name = get_release_branch_name(current_version) tag_name = f"v{current_version}" is_rc = "rc" in tag_name @@ -757,7 +762,7 @@ Hi everyone. Synapse {current_version} has just been released. ) release_text += f""" -[notes](https://github.com/element-hq/synapse/releases/tag/{tag_name}) | \ +[notes](https://github.com/element-hq/synapse/blob/{release_branch_name}/CHANGES.md) | \ [docker](https://hub.docker.com/r/matrixdotorg/synapse/tags?name={tag_name}) | \ [debs](https://packages.matrix.org/debian/) | \ [pypi](https://pypi.org/project/matrix-synapse/{current_version}/)""" @@ -806,7 +811,6 @@ def full(gh_token: str) -> None: _prepare() click.echo("Deploy to matrix.org and ensure that it hasn't fallen over.") - click.echo("Remember to silence the alerts to prevent alert spam.") click.confirm("Deployed?", abort=True) click.echo("\n*** tag ***") diff --git a/synapse/__init__.py b/synapse/__init__.py index 3acfc1a0d7..a223066f04 100644 --- a/synapse/__init__.py +++ b/synapse/__init__.py @@ -49,7 +49,9 @@ if strtobool(os.environ.get("SYNAPSE_ASYNC_IO_REACTOR", "0")): from twisted.internet import asyncioreactor - asyncioreactor.install(asyncio.get_event_loop()) + loop = asyncio.new_event_loop() + asyncio.set_event_loop(loop) + asyncioreactor.install(loop) # Twisted and canonicaljson will fail to import when this file is executed to # get the __version__ during a fresh install. That's OK and subsequent calls to diff --git a/synapse/_scripts/synapse_port_db.py b/synapse/_scripts/synapse_port_db.py index 0b8a289d92..f4f598d27c 100755 --- a/synapse/_scripts/synapse_port_db.py +++ b/synapse/_scripts/synapse_port_db.py @@ -917,6 +917,10 @@ class Porter: "quarantined_media_id_seq", [("quarantined_media_changes", "stream_id")], ) + await self._setup_sequence( + "profile_updates_sequence", + [("profile_updates", "stream_id")], + ) # Step 3. Get tables. self.progress.set_state("Fetching tables") diff --git a/synapse/api/auth/__init__.py b/synapse/api/auth/__init__.py index 201c295f06..d8d3b31b9d 100644 --- a/synapse/api/auth/__init__.py +++ b/synapse/api/auth/__init__.py @@ -24,7 +24,7 @@ from prometheus_client import Histogram from twisted.web.server import Request -from synapse.appservice import ApplicationService +from synapse.appservice import ApplicationService, Scopes from synapse.http.site import SynapseRequest from synapse.metrics import SERVER_NAME_LABEL from synapse.types import Requester @@ -205,3 +205,9 @@ class Auth(Protocol): membership event ID of the user. If the user is not in the room and never has been, then `(Membership.JOIN, None)` is returned. """ + + def assert_requester_has_scope(self, requester: Requester, scope: Scopes) -> None: + """Asserts that the requester has the given scope, either directly + (e.g. via an OAuth token) or via the scopes registered against the + application service. + """ diff --git a/synapse/api/auth/base.py b/synapse/api/auth/base.py index 14e76b0cff..7b8214fabb 100644 --- a/synapse/api/auth/base.py +++ b/synapse/api/auth/base.py @@ -19,6 +19,7 @@ # # import logging +from http import HTTPStatus from typing import TYPE_CHECKING from netaddr import IPAddress @@ -33,7 +34,7 @@ from synapse.api.errors import ( MissingClientTokenError, UnstableSpecAuthError, ) -from synapse.appservice import ApplicationService +from synapse.appservice import ApplicationService, Scopes from synapse.http import get_request_user_agent from synapse.http.site import SynapseRequest from synapse.logging.opentracing import trace @@ -316,9 +317,6 @@ class BaseAuth: - The returned device ID, if present, has been checked to be a valid device ID for the returned user ID. """ - # TODO: We can drop unstable support after 2026-01-01 (couple months after stable support) - UNSTABLE_DEVICE_ID_ARG_NAME = b"org.matrix.msc3202.device_id" - app_service = self.store.get_app_service_by_token(access_token) if app_service is None: return None @@ -339,9 +337,7 @@ class BaseAuth: else: effective_user_id = app_service.sender - effective_device_id_args = request.args.get( - b"device_id", request.args.get(UNSTABLE_DEVICE_ID_ARG_NAME) - ) + effective_device_id_args = request.args.get(b"device_id") if effective_device_id_args: effective_device_id = effective_device_id_args[0].decode("utf8") # We only just set this so it can't be None! @@ -362,6 +358,21 @@ class BaseAuth: effective_user_id, app_service=app_service, device_id=effective_device_id ) + def assert_requester_has_scope(self, requester: Requester, scope: Scopes) -> None: + """Asserts that the requester has the given scope, either directly + (e.g. via an OAuth token) or via the scopes registered against the + application service. + """ + if scope in requester.scope: + return + + if requester.app_service_id is not None: + app_service = self.store.get_app_service_by_id(requester.app_service_id) + if app_service is not None and app_service.has_scope(scope): + return + + raise AuthError(HTTPStatus.FORBIDDEN, f"Missing {scope} scope") + async def _record_request( self, request: SynapseRequest, requester: Requester ) -> None: diff --git a/synapse/api/constants.py b/synapse/api/constants.py index acac057334..8e91b2e203 100644 --- a/synapse/api/constants.py +++ b/synapse/api/constants.py @@ -410,6 +410,61 @@ class ProfileFields: AVATAR_URL: Final = "avatar_url" +class ProfileUpdateAction(str, enum.Enum): + """ + Enum representing the action of a row in the profile updates stream tables. + The action determines whether a profile field update has occurred, or whether + something else has happened that the sync code should know about, for example + a user joining or leaving a room. + """ + + JOINED_ROOM = "joined_room" + """ + This profile update row action represents a user joining a room. + + When gathering an incremental sync non-lazy response for profile updates, + we always include the full profile of users who have joined a room the syncing + user is a member of, where full profile means all the current profile values the + client asked for, regardless of whether they have changed recently. This ensures + that clients have profiles re-populated for any users who have recently left + shared rooms. + + A scenario example would be as follows: + + * Alice leaves a room with Bob + * Bob's client clears all profile fields from Alice + * Alice joins a room with Bob + * Bob's client does an incremental non-lazy sync + + At the end of the flow Bob should receive all the profile fields the client + is interested in, not just the potential diff, which non-lazy incremental sync + normally includes. This update action currently has no meaning for sync responses + that are not incremental and non-lazy. + """ + + LEFT_ROOM = "left_room" + """ + This profile update row action represents a user leaving a room. + + Clients will want to know when they no longer share rooms with a user. This + profile action row allows the sync code to deliver a `null` response for those + profiles, so clients can clear their cache containing the users profile data + they are no longer interested in. + """ + + UPDATE = "update" + """ + This profile update row action represents a user updating one or more + profile fields. + 'Updating' could mean creating, changing the value of, or deleting a field. + + Depending on the type of sync (initial/incremental, lazy/non-lazy), either the + diff of profile field updates or all the current profile fields are included + in the sync response. In the latter case the profile update action row signifies + a change, but the client may still get fields that have not changed. + """ + + class StickyEventField(TypedDict): """ Dict content of the `sticky` part of an event. diff --git a/synapse/api/errors.py b/synapse/api/errors.py index 0c35b4a7ba..00b5fa9a76 100644 --- a/synapse/api/errors.py +++ b/synapse/api/errors.py @@ -137,11 +137,13 @@ class Codes(str, Enum): PROFILE_TOO_LARGE = "M_PROFILE_TOO_LARGE" KEY_TOO_LARGE = "M_KEY_TOO_LARGE" + DELAY_TOO_LARGE = "ORG.MATRIX.MSC4140_DELAY_TOO_LARGE" + # Part of MSC4155/MSC4380 INVITE_BLOCKED = "M_INVITE_BLOCKED" # Part of MSC4190 - APPSERVICE_LOGIN_UNSUPPORTED = "IO.ELEMENT.MSC4190.M_APPSERVICE_LOGIN_UNSUPPORTED" + APPSERVICE_LOGIN_UNSUPPORTED = "M_APPSERVICE_LOGIN_UNSUPPORTED" # Part of MSC4306: Thread Subscriptions MSC4306_CONFLICTING_UNSUBSCRIPTION = ( @@ -152,6 +154,8 @@ class Codes(str, Enum): # Part of MSC4326 UNKNOWN_DEVICE = "ORG.MATRIX.MSC4326.M_UNKNOWN_DEVICE" + USER_LIMIT_EXCEEDED = "M_USER_LIMIT_EXCEEDED" + class CodeMessageException(RuntimeError): """An exception with integer code, a message string attributes and optional headers. @@ -513,6 +517,34 @@ class ResourceLimitError(SynapseError): ) +class UserLimitExceededError(SynapseError): + """ + Implementation of M_USER_LIMIT_EXCEEDED error + """ + + def __init__( + self, + code: int, + msg: str, + *, + info_uri: str, + can_upgrade: bool = False, + ): + additional_fields: dict[str, str | bool] = { + "info_uri": info_uri, + } + + if can_upgrade: + additional_fields["can_upgrade"] = can_upgrade + + super().__init__( + code, + msg, + Codes.USER_LIMIT_EXCEEDED, + additional_fields=additional_fields, + ) + + class EventSizeError(SynapseError): """An error raised when an event is too big.""" @@ -823,20 +855,24 @@ class HttpResponseException(CodeMessageException): super().__init__(code, msg) self.response = response - def to_synapse_error(self) -> SynapseError: - """Make a SynapseError based on an HTTPResponseException + def unsafe_to_verbatim_synapse_error(self) -> SynapseError: + """Make a SynapseError directly based on a TRUSTED HTTPResponseException. This is useful when a proxied request has failed, and we need to decide how to map the failure onto a matrix error to send back to the client. - An attempt is made to parse the body of the http response as a matrix + An attempt is made to parse the body of the HTTP response as a Matrix error. If that succeeds, the errcode and error message from the body - are used as the errcode and error message in the new synapse error. + are copied verbatim into the new Synapse error. Otherwise, the errcode is set to M_UNKNOWN, and the error message is set to the reason code from the HTTP response. + Safety: + This must ONLY be used on errors from TRUSTED sources, + such as other Synapse workers. + Returns: The error converted to a SynapseError. """ @@ -851,10 +887,68 @@ class HttpResponseException(CodeMessageException): j = {} errcode = j.pop("errcode", Codes.UNKNOWN) + if not isinstance(errcode, str): + errcode = Codes.UNKNOWN errmsg = j.pop("error", self.msg) + if not isinstance(errmsg, str): + errmsg = self.msg return ProxiedRequestError(self.code, errmsg, errcode, j) + def to_synapse_error(self) -> SynapseError: + """Make a SynapseError directly based on a TRUSTED HTTPResponseException. + + This is useful when a proxied request has failed, and we need to + decide how to map the failure onto a matrix error to send back to the + client. + + An attempt is made to parse the body of the HTTP response as a Matrix + error. If that succeeds, the errcode and error message from the body + are copied verbatim into the new Synapse error, unless it's of + a forbidden type. + + Otherwise, the errcode is set to M_UNKNOWN, and the error message is + set to the reason code from the HTTP response. + + Safety: + This is the correct method to use when forwarding errors + from upstream requests (e.g. federation, policy servers). + + FIXME: restrict forwarded errors further + + Returns: + The error converted to a SynapseError. + """ + # try to parse the body as json, to get better errcode/msg, but + # default to M_UNKNOWN with the HTTP status as the error text + try: + j = json_decoder.decode(self.response.decode("utf-8")) + except ValueError: + j = {} + + if not isinstance(j, dict): + j = {} + + status = self.code + errcode = j.pop("errcode", Codes.UNKNOWN) + if not isinstance(errcode, str): + errcode = Codes.UNKNOWN + errmsg = j.pop("error", self.msg) + if not isinstance(errmsg, str): + errmsg = self.msg + + if errcode == Codes.UNKNOWN_TOKEN: + # We must not relay this error code back down to clients, + # because clients interpret this code to mean that they + # have been logged out. + # See: https://github.com/element-hq/synapse/security/advisories/GHSA-95fh-hv8c-chvq + errcode = Codes.UNKNOWN + + if status == HTTPStatus.UNAUTHORIZED: + status = HTTPStatus.BAD_REQUEST + + return ProxiedRequestError(status, errmsg, errcode, j) + class HomeServerNotSetupException(Exception): """ diff --git a/synapse/api/filtering.py b/synapse/api/filtering.py index 9b47c20437..cbae9133c6 100644 --- a/synapse/api/filtering.py +++ b/synapse/api/filtering.py @@ -123,6 +123,13 @@ USER_FILTER_SCHEMA = { "filter": FILTER_SCHEMA, "room_filter": ROOM_FILTER_SCHEMA, "room_event_filter": ROOM_EVENT_FILTER_SCHEMA, + "profile_fields_filter": { + "type": "object", + "properties": { + "ids": {"type": "array", "items": {"type": "string"}}, + }, + "additionalProperties": True, + }, }, "properties": { "presence": {"$ref": "#/definitions/filter"}, @@ -130,6 +137,9 @@ USER_FILTER_SCHEMA = { "room": {"$ref": "#/definitions/room_filter"}, "event_format": {"type": "string", "enum": ["client", "federation"]}, "event_fields": {"type": "array", "items": {"type": "string"}}, + "org.matrix.msc4429.profile_fields": { + "$ref": "#/definitions/profile_fields_filter" + }, }, "additionalProperties": True, # Allow new fields for forward compatibility } @@ -217,6 +227,13 @@ class FilterCollection: self.event_fields = filter_json.get("event_fields", []) self.event_format = filter_json.get("event_format", "client") + self.profile_fields: set[str] = set() + if hs.config.server.include_profile_updates_in_sync: + profile_fields_filter = filter_json.get("org.matrix.msc4429.profile_fields") + + if isinstance(profile_fields_filter, Mapping): + self.profile_fields = set(profile_fields_filter.get("ids", [])) + def __repr__(self) -> str: return "" % (json.dumps(self._filter_json),) diff --git a/synapse/appservice/__init__.py b/synapse/appservice/__init__.py index c55a83a879..20f7a410b6 100644 --- a/synapse/appservice/__init__.py +++ b/synapse/appservice/__init__.py @@ -63,6 +63,14 @@ TransactionOneTimeKeysCount = dict[str, dict[str, dict[str, int]]] TransactionUnusedFallbackKeys = dict[str, dict[str, list[str]]] +class Scopes(str, Enum): + """ + All known scopes assignable to application services for extended privileges. + """ + + QUERY_ROOM_MEMBERSHIP = "urn:matrix:client:io.element.msc4502:rooms:is_joined" + + class ApplicationServiceState(Enum): DOWN = "down" UP = "up" @@ -89,6 +97,11 @@ class ApplicationService: # values. NS_LIST = [NS_USERS, NS_ALIASES, NS_ROOMS] + # Prefixes are applied after the version segment(s) (either /vX/ or /unstable/foo/): + # - /_matrix/client/(unstable/[^/]+|v[^/]+)/{prefix}/.* + # - /_matrix/federation/(unstable/[^/]+|v[^/]+)/{prefix}/.* + ALLOWED_PROXY_PREFIXES = {"rtc/livekit"} + def __init__( self, token: str, @@ -104,11 +117,20 @@ class ApplicationService: supports_unstable_ephemeral: bool = False, msc3202_transaction_extensions: bool = False, msc4190_device_management: bool = False, + scopes: Iterable[str] = frozenset(), + proxy_prefix: str | None = None, + proxy_url: str | None = None, ): self.token = token self.url = ( url.rstrip("/") if isinstance(url, str) else None ) # url must not end with a slash + self.proxy_url = ( + proxy_url.rstrip("/") if isinstance(proxy_url, str) else None + ) # proxy_url must not end with a slash + self.proxy_prefix = ( + proxy_prefix.rstrip("/") if isinstance(proxy_prefix, str) else None + ) # proxy_prefix must not end with a slash self.hs_token = hs_token # The full Matrix ID for this application service's sender. self.sender = sender @@ -134,12 +156,25 @@ class ApplicationService: if "|" in self.id: raise Exception("application service ID cannot contain '|' character") + if (self.proxy_prefix is None) != (self.proxy_url is None): + raise KeyError("proxy_url and proxy_prefix must always be set together") + if proxy_prefix is not None: + if not proxy_prefix or not self.proxy_url: + raise ValueError("proxy_prefix and proxy_url must be non-empty strings") + if not self._is_proxy_prefix_allowed(proxy_prefix): + raise ValueError(f"cannot claim reserved proxy prefix {proxy_prefix!r}") + # .protocols is a publicly visible field if protocols: self.protocols = set(protocols) else: self.protocols = set() + self.scopes = set(scopes) + unknown_scopes = self.scopes - frozenset(Scopes) + if unknown_scopes: + raise ValueError(f"Unknown application service scope(s): {unknown_scopes}") + self.rate_limited = rate_limited def _check_namespaces( @@ -192,6 +227,12 @@ class ApplicationService: return namespace.exclusive return False + def _is_proxy_prefix_allowed(self, prefix: str) -> bool: + return any( + prefix == allowed or prefix.startswith(allowed + "/") + for allowed in ApplicationService.ALLOWED_PROXY_PREFIXES + ) + @cached(num_args=1, cache_context=True) async def _matches_user_in_member_list( self, @@ -379,6 +420,9 @@ class ApplicationService: def is_interested_in_protocol(self, protocol: str) -> bool: return protocol in self.protocols + def has_scope(self, scope: Scopes) -> bool: + return scope in self.scopes + def is_exclusive_alias(self, alias: str) -> bool: return self._is_exclusive(ApplicationService.NS_ALIASES, alias) diff --git a/synapse/config/_base.pyi b/synapse/config/_base.pyi index 7c371d161c..f226afc2b4 100644 --- a/synapse/config/_base.pyi +++ b/synapse/config/_base.pyi @@ -38,6 +38,7 @@ from synapse.config import ( # noqa: F401 oidc, password_auth_providers, push, + push_rules, ratelimiting, redis, registration, @@ -103,6 +104,7 @@ class RootConfig: worker: workers.WorkerConfig authproviders: password_auth_providers.PasswordAuthProviderConfig push: push.PushConfig + push_rules: push_rules.PushRulesConfig spamchecker: spam_checker.SpamCheckerConfig room: room.RoomConfig userdirectory: user_directory.UserDirectoryConfig diff --git a/synapse/config/_util.py b/synapse/config/_util.py index e09c68ebd4..8e1dd52c29 100644 --- a/synapse/config/_util.py +++ b/synapse/config/_util.py @@ -18,12 +18,13 @@ # [This file includes modifications made by New Vector Limited] # # -from typing import Any, TypeVar +from typing import Annotated, Any, TypeVar import jsonschema -from pydantic import BaseModel, TypeAdapter, ValidationError +from pydantic import BaseModel, BeforeValidator, StrictInt, TypeAdapter, ValidationError +from pydantic_core.core_schema import int_schema -from synapse.config._base import ConfigError +from synapse.config._base import Config, ConfigError from synapse.types import JsonDict, StrSequence @@ -97,3 +98,11 @@ def parse_and_validate_mapping( except ValidationError as e: raise ConfigError(str(e)) from e return instances + + +ConfigByteSize = Annotated[ + StrictInt, BeforeValidator(Config.parse_size), int_schema(ge=0) +] +""" +A size in bytes. Pydantic-compatible wrapper for `Config.parse_size` +""" diff --git a/synapse/config/appservice.py b/synapse/config/appservice.py index 7a629d10bf..1868985f80 100644 --- a/synapse/config/appservice.py +++ b/synapse/config/appservice.py @@ -68,6 +68,7 @@ def load_appservices( # Dicts of value -> filename seen_as_tokens: dict[str, str] = {} seen_ids: dict[str, str] = {} + seen_proxy_prefixes: dict[str, str] = {} appservices = [] @@ -93,6 +94,17 @@ def load_appservices( ) ) seen_as_tokens[appservice.token] = config_file + if appservice.proxy_prefix is not None: + for seen_prefix, seen_file in seen_proxy_prefixes.items(): + if _proxy_prefixes_overlap( + appservice.proxy_prefix, seen_prefix + ): + raise ConfigError( + "io.element.msc4512.proxy_prefix values must not overlap across " + "application services: " + f"{appservice.proxy_prefix} (files: {config_file}, {seen_file})" + ) + seen_proxy_prefixes[appservice.proxy_prefix] = config_file logger.info("Loaded application service: %s", appservice) appservices.append(appservice) except Exception as e: @@ -102,6 +114,15 @@ def load_appservices( return appservices +def _proxy_prefixes_overlap(prefix_a: str, prefix_b: str) -> bool: + """Returns whether two proxy prefixes overlap by sharing a common path prefix.""" + return ( + prefix_a == prefix_b + or prefix_a.startswith(prefix_b + "/") + or prefix_b.startswith(prefix_a + "/") + ) + + def _load_appservice( hostname: str, as_info: JsonDict, config_filename: str ) -> ApplicationService: @@ -199,6 +220,30 @@ def _load_appservice( "The `io.element.msc4190` option should be true or false if specified." ) + # Opt-in list of scopes granted to this appservice for restricted C-S API + # functionality. + scopes = as_info.get("io.element.msc4502.scopes", []) + if not isinstance(scopes, list) or not all(isinstance(s, str) for s in scopes): + raise ValueError( + "The `io.element.msc4502.scopes` option should be a list of strings if specified." + ) + + # Opt-in setting to enable proxying C-S and S-S API endpoints. + # When set, Synapse will reverse-proxy requests under the prefix to the appservice: + proxy_prefix = as_info.get("io.element.msc4512.proxy_prefix") + if proxy_prefix is not None: + if not isinstance(proxy_prefix, str) or not proxy_prefix: + raise ValueError( + "The `io.element.msc4512.proxy_prefix` option should be a non-empty string." + ) + + proxy_url = as_info.get("io.element.msc4512.proxy_url") + if proxy_url is not None: + if not isinstance(proxy_url, str) or not proxy_url: + raise ValueError( + "The `io.element.msc4512.proxy_url` option should be a non-empty string." + ) + return ApplicationService( token=as_info["as_token"], url=as_info["url"], @@ -213,4 +258,7 @@ def _load_appservice( supports_ephemeral=supports_ephemeral, msc3202_transaction_extensions=msc3202_transaction_extensions, msc4190_device_management=msc4190_enabled, + scopes=scopes, + proxy_prefix=proxy_prefix, + proxy_url=proxy_url, ) diff --git a/synapse/config/experimental.py b/synapse/config/experimental.py index f99f7b139e..1c2f021322 100644 --- a/synapse/config/experimental.py +++ b/synapse/config/experimental.py @@ -203,6 +203,9 @@ class ExperimentalConfig(Config): # See https://github.com/element-hq/synapse/issues/19524 self.msc4370_enabled = experimental.get("msc4370_enabled", False) + # MSC4502: Targeted and unrestricted room member queries + self.msc4502_enabled: bool = experimental.get("msc4502_enabled", False) + auth_delegated = (config.get("matrix_authentication_service") or {}).get( "enabled", False ) @@ -287,6 +290,10 @@ class ExperimentalConfig(Config): # (and MSC4308: Thread Subscriptions extension to Sliding Sync) self.msc4306_enabled: bool = experimental.get("msc4306_enabled", False) + # MSC4446: Allow moving the fully read marker backwards. + # Tracked in: https://github.com/element-hq/synapse/issues/19940 + self.msc4446_enabled: bool = experimental.get("msc4446_enabled", False) + # MSC4354: Sticky Events # Tracked in: https://github.com/element-hq/synapse/issues/19409 # Note that sticky events persisted before this feature is enabled will not be @@ -305,3 +312,6 @@ class ExperimentalConfig(Config): # MSC4491: Invite reasons in room creation self.msc4491_enabled: bool = experimental.get("msc4491_enabled", False) + + # MSC4512: Delegating parts of the C-S and S-S API to application services + self.msc4512_enabled: bool = experimental.get("msc4512_enabled", False) diff --git a/synapse/config/homeserver.py b/synapse/config/homeserver.py index 94ebe583a4..e2ae182fc3 100644 --- a/synapse/config/homeserver.py +++ b/synapse/config/homeserver.py @@ -19,6 +19,8 @@ # # +from synapse.config.push_rules import PushRulesConfig + from ._base import ConfigError, RootConfig from .account_validity import AccountValidityConfig from .api import ApiConfig @@ -102,6 +104,7 @@ class HomeServerConfig(RootConfig): EmailConfig, PasswordAuthProviderConfig, PushConfig, + PushRulesConfig, SpamCheckerConfig, RoomConfig, UserDirectoryConfig, diff --git a/synapse/config/matrixrtc.py b/synapse/config/matrixrtc.py index 84c245e286..896317fca8 100644 --- a/synapse/config/matrixrtc.py +++ b/synapse/config/matrixrtc.py @@ -17,7 +17,7 @@ from typing import Any -from pydantic import Field, StrictStr, ValidationError, model_validator +from pydantic import Field, StrictStr, ValidationError, field_validator, model_validator from typing_extensions import Self from synapse.types import JsonDict @@ -29,20 +29,34 @@ from ._base import Config, ConfigError class TransportConfigModel(ParseModel): type: StrictStr + url: StrictStr | None = Field(default=None) + """An optional WebSocket URL pointing to the LiveKit SFU. If type is "livekit", either this or livekit_service_url is required.""" + livekit_service_url: StrictStr | None = Field(default=None) - """An optional livekit service URL. Only required if type is "livekit".""" + """Deprecated. An optional HTTP URL pointing to the LiveKit authorization service. If type is "livekit", either this or url is required.""" @model_validator(mode="after") - def validate_livekit_service_url(self) -> Self: - if self.type == "livekit" and not self.livekit_service_url: + def validate_livekit_transport(self) -> Self: + if self.type == "livekit" and not self.url and not self.livekit_service_url: raise ValueError( - "You must set a `livekit_service_url` when using the 'livekit' transport." + "You must set either `url` or `livekit_service_url` when using the 'livekit' transport." ) return self class MatrixRtcConfigModel(ParseModel): - transports: list = [] + transports: list[dict[str, Any]] = [] + + @field_validator("transports") + @classmethod + def validate_transports( + cls, transports: list[dict[str, Any]] + ) -> list[dict[str, Any]]: + """Validate each transport by attempting to construct a `TransportConfigModel` + from it, raising a `ValidationError` if construction fails.""" + for transport in transports: + TransportConfigModel(**transport) + return transports class MatrixRtcConfig(Config): diff --git a/synapse/config/push_rules.py b/synapse/config/push_rules.py new file mode 100644 index 0000000000..deb7e8dde1 --- /dev/null +++ b/synapse/config/push_rules.py @@ -0,0 +1,63 @@ +# +# This file is licensed under the Affero General Public License (AGPL) version 3. +# +# Copyright (C) 2026 Element Creations Ltd +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# See the GNU Affero General Public License for more details: +# . + +from typing import Annotated, Any + +from pydantic import ( + Field, + StrictInt, + ValidationError, +) + +from synapse.config._util import ConfigByteSize +from synapse.types import JsonDict +from synapse.util.pydantic_models import ParseModel + +from ._base import Config, ConfigError + + +class PushRulesLimitsConfig(ParseModel): + # Chosen arbitrarily, but with the rough rationale that a user + # might have on the order of 10k rooms and want to set a push rule override for each one. + rule_count: Annotated[StrictInt, Field(ge=0)] = 10_000 + + # Chosen arbitrarily, but with the rationale that room IDs are allowed to be up to 255 bytes + # and they are often used in rule IDs. + rule_id_length: Annotated[StrictInt, Field(ge=1)] = 300 + + # Chosen arbitrarily, but with the rationale that real-world push rules don't get + # nearly this big in practice. + # Even 512 bytes would probably have been fine, but we should leave space for the use cases + # of push rules to grow in the future. + rule_size: Annotated[ConfigByteSize, Field(ge=1)] = 1024 + + +class PushRulesConfigModel(ParseModel): + limits: PushRulesLimitsConfig = Field(default_factory=PushRulesLimitsConfig) + + +class PushRulesConfig(Config): + section = "push_rules" + + def read_config(self, config: JsonDict, **kwargs: Any) -> None: + raw_config = config.get("push_rules", {}) + + try: + parsed = PushRulesConfigModel(**raw_config) + except ValidationError as e: + raise ConfigError( + f"Could not validate configuration: {e}", + path=("push_rules",), + ) from e + + self.limits = parsed.limits diff --git a/synapse/config/repository.py b/synapse/config/repository.py index 373e518ddc..6d06c910d7 100644 --- a/synapse/config/repository.py +++ b/synapse/config/repository.py @@ -3,6 +3,7 @@ # # Copyright 2014, 2015 OpenMarket Ltd # Copyright (C) 2023 New Vector, Ltd +# Copyright (C) 2026 Element Creations Ltd # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License as @@ -21,14 +22,16 @@ import logging import os -from typing import Any +from typing import Annotated, Any import attr +from pydantic import AnyUrl, BeforeValidator, ValidationError from synapse.config.server import generate_ip_set, parse_proxy_config from synapse.types import JsonDict from synapse.util.check_dependencies import check_requirements from synapse.util.module_loader import load_module +from synapse.util.pydantic_models import ParseModel from ._base import Config, ConfigError @@ -134,6 +137,27 @@ class MediaUploadLimit: time_period_ms: int """The time period in milliseconds.""" + info_uri: str | None = None + """The URI to return with the M_USER_LIMIT_EXCEEDED error. + + If left unset (`None`), Synapse falls back to a static page served by itself + (see `MEDIA_UPLOAD_LIMIT_EXCEEDED_PATH`), which explains that the limit has been + exceeded and can be customized by server administrators via a custom + template.""" + + can_upgrade: bool = False + """Whether the user can upgrade their plan to increase the limit. This is returned in the M_USER_LIMIT_EXCEEDED error.""" + + +class MediaUploadLimitConfigModel(ParseModel): + """Internal model for parsing a single media_upload_limits config entry.""" + + max_size: Annotated[int, BeforeValidator(Config.parse_size)] + time_period: Annotated[int, BeforeValidator(Config.parse_duration)] + info_uri: AnyUrl | None = None + """We accept AnyUrl as a subset of valid URIs. It could be widened in future if needed.""" + can_upgrade: bool = False + class ContentRepositoryConfig(Config): section = "media" @@ -143,6 +167,18 @@ class ContentRepositoryConfig(Config): # is not a media repo worker, as it's exposed in `/capabilities` self.url_preview_enabled = bool(config.get("url_preview_enabled", False)) + # Load the template used to render the fallback page. + # + # We set this up on all workers (not just the media repo) as the + # fallback page is served by whichever process handles + # `/_synapse/client/media_upload_limit_exceeded`, so every process must + # be able to render it. This must happen before the early return below, + # which is taken by workers that do not load the media repo. + self.media_upload_limit_exceeded_template = self.read_templates( + ["media_upload_limit_exceeded.html"], + (td for td in (self.root.server.custom_template_directory,) if td), + )[0] + # Only enable the media repo if either the media repo is enabled or the # current worker app is the media repo. if ( @@ -308,11 +344,40 @@ class ContentRepositoryConfig(Config): self.enable_authenticated_media = config.get("enable_authenticated_media", True) self.media_upload_limits: list[MediaUploadLimit] = [] - for limit_config in config.get("media_upload_limits", []): - time_period_ms = self.parse_duration(limit_config["time_period"]) - max_bytes = self.parse_size(limit_config["max_size"]) + for raw_entry in config.get("media_upload_limits", []): + try: + entry = MediaUploadLimitConfigModel.model_validate( + raw_entry, strict=True + ) + except ValidationError as e: + raise ConfigError( + "Could not validate media_upload_limits entry", + ("media_upload_limits",), + ) from e - self.media_upload_limits.append(MediaUploadLimit(max_bytes, time_period_ms)) + info_uri = str(entry.info_uri) if entry.info_uri is not None else None + self.media_upload_limits.append( + MediaUploadLimit( + max_bytes=entry.max_size, + time_period_ms=entry.time_period, + info_uri=info_uri, + can_upgrade=entry.can_upgrade, + ) + ) + + # The absolute URI of the static fallback page, used as the `info_uri` + # for any media upload limit (whether from config or a module callback) + # that doesn't specify one. Built from public_baseurl so that it is a + # usable absolute URL. We import here to avoid a circular import at + # module load time. + from synapse.rest.synapse.client.media_upload_limit_exceeded import ( + MEDIA_UPLOAD_LIMIT_EXCEEDED_PATH, + ) + + self.media_upload_limit_fallback_info_uri = ( + self.root.server.public_baseurl + + MEDIA_UPLOAD_LIMIT_EXCEEDED_PATH.lstrip("/") + ) def generate_config_section(self, data_dir_path: str, **kwargs: Any) -> str: assert data_dir_path is not None diff --git a/synapse/config/server.py b/synapse/config/server.py index 282679aabe..9c5a42fc52 100644 --- a/synapse/config/server.py +++ b/synapse/config/server.py @@ -37,6 +37,7 @@ from twisted.conch.ssh.keys import Key from synapse.api.room_versions import KNOWN_ROOM_VERSIONS from synapse.types import JsonDict, StrSequence +from synapse.util.duration import Duration from synapse.util.module_loader import load_module from synapse.util.stringutils import parse_and_validate_server_name @@ -175,7 +176,7 @@ DEFAULT_IP_RANGE_BLOCKLIST = [ "fec0::/10", ] -DEFAULT_ROOM_VERSION = "10" +DEFAULT_ROOM_VERSION = "11" # Defaults for the presence state machine timers, in milliseconds. Overridden # by the corresponding options in the `presence` config section. @@ -584,6 +585,12 @@ class ServerConfig(Config): " 'allow_public_rooms_over_federation' is set." ) + # Whether to support MSC4429 and MSC4262 Profile updates down sync + self.include_profile_updates_in_sync = config.get( + "include_profile_updates_in_sync", + False, + ) + # Check if the legacy "restrict_public_rooms_to_local_users" flag is set. This # flag is now obsolete but we need to check it for backward-compatibility. if config.get("restrict_public_rooms_to_local_users", False): @@ -652,6 +659,15 @@ class ServerConfig(Config): else: self.redaction_retention_period = None + # How long to allow event redactions for on `m.room.message` + redaction_allowed_period = config.get("redaction_allowed_period", None) + if redaction_allowed_period is not None: + self.redaction_allowed_period: int | None = self.parse_duration( + redaction_allowed_period + ) + else: + self.redaction_allowed_period = None + # How long to keep locally forgotten rooms before purging them from the DB. forgotten_room_retention_period = config.get( "forgotten_room_retention_period", None @@ -935,6 +951,10 @@ class ServerConfig(Config): config.get("exclude_rooms_from_sync") or [] ) + self.rooms_to_exclude_from_presence: list[str] = ( + config.get("exclude_rooms_from_presence") or [] + ) + delete_stale_devices_after: str | None = ( config.get("delete_stale_devices_after") or None ) @@ -949,13 +969,34 @@ class ServerConfig(Config): # The maximum allowed delay duration for delayed events (MSC4140). max_event_delay_duration = config.get("max_event_delay_duration") if max_event_delay_duration is not None: - self.max_event_delay_ms: int | None = self.parse_duration( - max_event_delay_duration - ) - if self.max_event_delay_ms <= 0: - raise ConfigError("max_event_delay_duration must be a positive value") + max_event_delay_ms = self.parse_duration(max_event_delay_duration) + if max_event_delay_ms <= 0: + raise ConfigError( + "'max_event_delay_duration' must be a positive value if set", + ("max_event_delay_duration",), + ) + self.max_event_delay_duration = Duration(milliseconds=max_event_delay_ms) else: - self.max_event_delay_ms = None + self.max_event_delay_duration = Duration() + + # The maximum number of delayed events a user may have scheduled at a time. + # (Defined here despite being experimental to be near the other MSC4140 config) + experimental = config.get("experimental_features") or {} + self.max_delayed_events_per_user: int = experimental.get( + "msc4140_max_delayed_events_per_user", 100 + ) + if ( + not isinstance(self.max_delayed_events_per_user, int) + or self.max_delayed_events_per_user < 0 + ): + raise ConfigError( + "'msc4140_max_delayed_events_per_user' must be a non-negative integer", + ("experimental", "msc4140_max_delayed_events_per_user"), + ) + + self.msc4140_enabled = bool( + self.max_delayed_events_per_user and self.max_event_delay_duration + ) def has_tls_listener(self) -> bool: return any(listener.is_tls() for listener in self.listeners) diff --git a/synapse/config/workers.py b/synapse/config/workers.py index fb7378bfc8..5fe1d1362e 100644 --- a/synapse/config/workers.py +++ b/synapse/config/workers.py @@ -127,9 +127,10 @@ class WriterLocations: """Specifies the instances that write various streams. Attributes: - events: The instances that write to the event, backfill and `sticky_events` streams. - (`sticky_events` is written to during event persistence so must be handled by the - same stream writers.) + events: The instances that write to the `event`, `backfill`, `sticky_events` and + `profile_updates` streams. + (`sticky_events` and `profile_updates` are written to during event + persistence so must be handled by the same stream writers.) typing: The instances that write to the typing stream. Currently can only be a single instance. to_device: The instances that write to the to_device stream. Currently @@ -142,6 +143,8 @@ class WriterLocations: push_rules: The instances that write to the push stream. Currently can only be a single instance. device_lists: The instances that write to the device list stream. + thread_subscriptions: The instances that write to the thread subscriptions + stream. quarantined_media_changes: The instances that write to the quarantined media changes stream. """ @@ -179,7 +182,7 @@ class WriterLocations: converter=_instance_to_list_converter, ) thread_subscriptions: list[str] = attr.ib( - default=["master"], + default=[MAIN_PROCESS_INSTANCE_NAME], converter=_instance_to_list_converter, ) quarantined_media_changes: list[str] = attr.ib( @@ -361,8 +364,7 @@ class WorkerConfig(Config): writers = config.get("stream_writers") or {} self.writers = WriterLocations(**writers) - # Check that the configured writers for events and typing also appears in - # `instance_map`. + # Check that the configured writers also appear in `instance_map`. for stream in ( "events", "typing", @@ -371,6 +373,9 @@ class WorkerConfig(Config): "receipts", "presence", "push_rules", + "device_lists", + "thread_subscriptions", + "quarantined_media_changes", ): instances = _instance_to_list_converter(getattr(self.writers, stream)) for instance in instances: @@ -421,6 +426,16 @@ class WorkerConfig(Config): "Must specify at least one instance to handle `device_lists` messages." ) + if len(self.writers.thread_subscriptions) == 0: + raise ConfigError( + "Must specify at least one instance to handle `thread_subscriptions` messages." + ) + + if len(self.writers.quarantined_media_changes) == 0: + raise ConfigError( + "Must specify at least one instance to handle `quarantined_media_changes` messages." + ) + self.events_shard_config = RoutableShardedWorkerHandlingConfig( self.writers.events ) diff --git a/synapse/federation/federation_client.py b/synapse/federation/federation_client.py index 56104ffacc..59f1b5b2d3 100644 --- a/synapse/federation/federation_client.py +++ b/synapse/federation/federation_client.py @@ -125,6 +125,9 @@ class SendJoinResult: # Always contains the server we joined off. servers_in_room: AbstractSet[str] + # Only valid for state DAG rooms (MSC4242) + state_dag: list[EventBase] | None + class FederationClient(FederationBase): def __init__(self, hs: "HomeServer"): @@ -1110,11 +1113,12 @@ class FederationClient(FederationBase): SynapseError: if the chosen remote server returns a 300/400 code, or no servers successfully handle the request. """ - # See related restriction in /createRoom requests in handlers/room.py - if room_version.msc4242_state_dags: - raise UnsupportedRoomVersionError( - "Homeserver does not support this room version over federation" - ) + + def find_create_event(events: list[EventBase]) -> EventBase | None: + for e in events: + if (e.type, e.state_key) == (EventTypes.Create, ""): + return e + return None async def send_request(destination: str) -> SendJoinResult: response = await self._do_send_join( @@ -1144,13 +1148,16 @@ class FederationClient(FederationBase): state = response.state auth_chain = response.auth_events + state_dag: list[EventBase] = [] + if room_version.msc4242_state_dags: + if not response.state_dag: + raise InvalidResponseError("No state_dag returned") + state_dag = response.state_dag - create_event = None - for e in state: - if (e.type, e.state_key) == (EventTypes.Create, ""): - create_event = e - break - + # Validate the create event and room version are what we expect to see. + create_event = find_create_event( + state_dag if room_version.msc4242_state_dags else state + ) if create_event is None: # If the state doesn't have a create event then the room is # invalid, and it would fail auth checks anyway. @@ -1168,57 +1175,7 @@ class FederationClient(FederationBase): % (create_room_version,) ) - logger.info( - "Processing from send_join %d events", len(state) + len(auth_chain) - ) - - # We now go and check the signatures and hashes for the event. Note - # that we limit how many events we process at a time to keep the - # memory overhead from exploding. - valid_pdus_map: dict[str, EventBase] = {} - - async def _execute(pdu: EventBase) -> None: - valid_pdu = await self._check_sigs_and_hash_and_fetch_one( - pdu=pdu, - origin=destination, - room_version=room_version, - ) - - if valid_pdu: - valid_pdus_map[valid_pdu.event_id] = valid_pdu - - await concurrently_execute( - _execute, itertools.chain(state, auth_chain), 10000 - ) - - # NB: We *need* to copy to ensure that we don't have multiple - # references being passed on, as that causes... issues. - signed_state = [ - valid_pdus_map[p.event_id].deep_copy() - for p in state - if p.event_id in valid_pdus_map - ] - - signed_auth = [ - valid_pdus_map[p.event_id] - for p in auth_chain - if p.event_id in valid_pdus_map - ] - - # double-check that the auth chain doesn't include a different create event - auth_chain_create_events = [ - e.event_id - for e in signed_auth - if (e.type, e.state_key) == (EventTypes.Create, "") - ] - if auth_chain_create_events and auth_chain_create_events != [ - create_event.event_id - ]: - raise InvalidResponseError( - "Unexpected create event(s) in auth chain: %s" - % (auth_chain_create_events,) - ) - + # Validate and set faster room joins fields servers_in_room = None if response.servers_in_room is not None: servers_in_room = set(response.servers_in_room) @@ -1238,15 +1195,107 @@ class FederationClient(FederationBase): # Fix things up in case the remote homeserver is badly behaved. servers_in_room.add(destination) - return SendJoinResult( - event=event, - state=signed_state, - auth_chain=signed_auth, - origin=destination, - partial_state=response.members_omitted, - servers_in_room=servers_in_room or frozenset(), + logger.info( + "Processing from send_join %d events", + len(state_dag) + if room_version.msc4242_state_dags + else (len(state) + len(auth_chain)), ) + # We now go and check the signatures and hashes for the event. Note + # that we limit how many events we process at a time to keep the + # memory overhead from exploding. + valid_pdus_map: dict[str, EventBase] = {} + + async def _execute(pdu: EventBase) -> None: + valid_pdu = await self._check_sigs_and_hash_and_fetch_one( + pdu=pdu, + origin=destination, + room_version=room_version, + ) + + if valid_pdu: + valid_pdus_map[valid_pdu.event_id] = valid_pdu + + # Verify signatures/hashes on events, and make sure they all refer to the same room. + if room_version.msc4242_state_dags: + if state or auth_chain or servers_in_room: + raise InvalidResponseError( + "State DAG rooms must not set servers_in_room, state or auth_chain fields" + ) + await concurrently_execute(_execute, itertools.chain(state_dag), 10000) + # NB: We *need* to copy to ensure that we don't have multiple + # references being passed on, as that causes... issues. + signed_state_dag = [ + valid_pdus_map[p.event_id].deep_copy() + for p in state_dag + if p.event_id in valid_pdus_map + ] + + # Verify each event is for this room (and thus has the same create event as it is v12+) + for state_event in signed_state_dag: + if state_event.room_id != pdu.room_id: + raise InvalidResponseError( + "%s in state_dag belongs to room %s, not %s which we are joining" + % (state_event.event_id, state_event.room_id, pdu.room_id) + ) + return SendJoinResult( + event=event, + state=[], + auth_chain=[], + state_dag=signed_state_dag, + origin=destination, + # The current Synapse implementation of MSC4242 does not support + # faster remote room joins, so always set partial_state=False. + partial_state=False, + servers_in_room=frozenset(), + ) + else: + if state_dag: + raise InvalidResponseError( + "Room does not support state DAGs but set state_dag field" + ) + await concurrently_execute( + _execute, itertools.chain(state, auth_chain), 10000 + ) + + # NB: We *need* to copy to ensure that we don't have multiple + # references being passed on, as that causes... issues. + signed_state = [ + valid_pdus_map[p.event_id].deep_copy() + for p in state + if p.event_id in valid_pdus_map + ] + + signed_auth = [ + valid_pdus_map[p.event_id] + for p in auth_chain + if p.event_id in valid_pdus_map + ] + + # double-check that the auth chain doesn't include a different create event + auth_chain_create_events = [ + e.event_id + for e in signed_auth + if (e.type, e.state_key) == (EventTypes.Create, "") + ] + if auth_chain_create_events and auth_chain_create_events != [ + create_event.event_id + ]: + raise InvalidResponseError( + "Unexpected create event(s) in auth chain: %s" + % (auth_chain_create_events,) + ) + return SendJoinResult( + event=event, + state=signed_state, + auth_chain=signed_auth, + origin=destination, + partial_state=response.members_omitted, + servers_in_room=servers_in_room or frozenset(), + state_dag=None, + ) + # MSC3083 defines additional error codes for room joins. failover_errcodes = None if room_version.restricted_join_rule: @@ -1550,6 +1599,7 @@ class FederationClient(FederationBase): limit: int, min_depth: int, timeout: int, + state_dag: bool = False, ) -> list[EventBase]: """Tries to fetch events we are missing. This is called when we receive an event without having received all of its ancestors. @@ -1565,6 +1615,7 @@ class FederationClient(FederationBase): limit: Maximum number of events to return. min_depth: Minimum depth of events to return. timeout: Max time to wait in ms + state_dag: True to walk the state DAG (MSC4242 rooms) """ try: content = await self.transport_layer.get_missing_events( @@ -1575,6 +1626,7 @@ class FederationClient(FederationBase): limit=limit, min_depth=min_depth, timeout=timeout, + state_dag=state_dag, ) received_time = self._clock.time_msec() diff --git a/synapse/federation/federation_server.py b/synapse/federation/federation_server.py index 7f03e26c97..bae5c38083 100644 --- a/synapse/federation/federation_server.py +++ b/synapse/federation/federation_server.py @@ -81,6 +81,7 @@ from synapse.logging.opentracing import ( ) from synapse.metrics import SERVER_NAME_LABEL from synapse.metrics.background_process_metrics import wrap_as_background_process +from synapse.module_api.callbacks.federation import FederatedEventDeliveryMethod from synapse.replication.http.federation import ( ReplicationFederationSendEduRestServlet, ) @@ -142,6 +143,7 @@ class FederationServer(FederationBase): self.server_name = hs.hostname self.handler = hs.get_federation_handler() self._spam_checker_module_callbacks = hs.get_module_api_callbacks().spam_checker + self._federation_callbacks = hs.get_module_api_callbacks().federation self._federation_event_handler = hs.get_federation_event_handler() self.state = hs.get_state_handler() self._event_auth_handler = hs.get_event_auth_handler() @@ -245,6 +247,10 @@ class FederationServer(FederationBase): res = self._transaction_dict_from_pdus(pdus) + await self._federation_callbacks.notify_on_event_delivered_over_federation( + origin, pdus, FederatedEventDeliveryMethod.BACKFILL + ) + return 200, res async def on_timestamp_to_event_request( @@ -266,6 +272,7 @@ class FederationServer(FederationBase): body including `event_id`. """ async with self._server_linearizer.queue((origin, room_id)): + await self._event_auth_handler.assert_host_in_room(room_id, origin) origin_host, _ = parse_server_name(origin) await self.check_server_matches_acl(origin_host, room_id) @@ -655,14 +662,27 @@ class FederationServer(FederationBase): # - but that's non-trivial to get right, and anyway somewhat defeats # the point of the linearizer. async with self._server_linearizer.queue((origin, room_id)): - resp = await self._state_resp_cache.wrap( - (room_id, event_id), - self._on_context_state_request_compute, - room_id, - event_id, - ) - - return 200, resp + if not self._federation_callbacks.interested_in_events_delivered_over_federation(): + # In the usual case where no module is interested in tracking event deliveries, + # use the response cache. + resp = await self._state_resp_cache.wrap( + (room_id, event_id), + self._on_context_state_request_compute, + room_id, + event_id, + ) + return 200, resp + else: + # When a module is interested in tracking event deliveries, + # we can't use the response cache that returns pre-serialised + # events, as we wouldn't have the raw events to track. + resp, events = await self._on_context_state_request_compute_with_events( + room_id, event_id + ) + await self._federation_callbacks.notify_on_event_delivered_over_federation( + origin, events, FederatedEventDeliveryMethod.STATE + ) + return 200, resp @trace @tag_args @@ -697,6 +717,28 @@ class FederationServer(FederationBase): async def _on_context_state_request_compute( self, room_id: str, event_id: str ) -> dict[str, list]: + """ + Respond to a `/state` request, returning just the response. + + This separation exists because we don't want to hold on to the underlying + events in the response cache, just the serialised JSON. + """ + resp, _ = await self._on_context_state_request_compute_with_events( + room_id, event_id + ) + return resp + + async def _on_context_state_request_compute_with_events( + self, room_id: str, event_id: str + ) -> tuple[dict[str, list], list[EventBase]]: + """ + Respond to a `/state` request. + + Returns: + Tuple of: + 1. the `/state` response + 2. list of the events used to build that response + """ pdus: Collection[EventBase] event_ids = await self.handler.get_state_ids_for_pdu(room_id, event_id) pdus = await self.store.get_events_as_list(event_ids) @@ -705,10 +747,13 @@ class FederationServer(FederationBase): room_id, [pdu.event_id for pdu in pdus] ) - return { - "pdus": serialize_and_filter_pdus(pdus), - "auth_chain": serialize_and_filter_pdus(auth_chain), - } + return ( + { + "pdus": serialize_and_filter_pdus(pdus), + "auth_chain": serialize_and_filter_pdus(auth_chain), + }, + [*pdus, *auth_chain], + ) async def on_pdu_request( self, origin: str, event_id: str @@ -716,6 +761,9 @@ class FederationServer(FederationBase): pdu = await self.handler.get_persisted_pdu(origin, event_id) if pdu: + await self._federation_callbacks.notify_on_event_delivered_over_federation( + origin, [pdu], FederatedEventDeliveryMethod.EVENT + ) return 200, self._transaction_dict_from_pdus([pdu]) else: return 404, "" @@ -899,6 +947,12 @@ class FederationServer(FederationBase): if servers_in_room is not None: resp["servers_in_room"] = list(servers_in_room) + await self._federation_callbacks.notify_on_event_delivered_over_federation( + origin, + [event, *state_events, *auth_chain_events], + FederatedEventDeliveryMethod.SEND_JOIN, + ) + return resp async def on_make_leave_request( @@ -1140,8 +1194,12 @@ class FederationServer(FederationBase): await self.check_server_matches_acl(origin_host, room_id) time_now = self._clock.time_msec() - auth_pdus = await self.handler.on_event_auth(event_id) + auth_pdus = await self.handler.on_event_auth(event_id, room_id) res = {"auth_chain": serialize_and_filter_pdus(auth_pdus, time_now)} + + await self._federation_callbacks.notify_on_event_delivered_over_federation( + origin, auth_pdus, FederatedEventDeliveryMethod.EVENT_AUTH + ) return 200, res async def on_query_client_keys( @@ -1224,6 +1282,10 @@ class FederationServer(FederationBase): else: logger.debug("Returning %d events", len(missing_events)) + await self._federation_callbacks.notify_on_event_delivered_over_federation( + origin, missing_events, FederatedEventDeliveryMethod.GET_MISSING_EVENTS + ) + time_now = self._clock.time_msec() return {"events": serialize_and_filter_pdus(missing_events, time_now)} diff --git a/synapse/federation/sender/transaction_manager.py b/synapse/federation/sender/transaction_manager.py index 99aa05ebd6..9a7fb8a3b1 100644 --- a/synapse/federation/sender/transaction_manager.py +++ b/synapse/federation/sender/transaction_manager.py @@ -18,7 +18,7 @@ # # import logging -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Mapping from prometheus_client import Gauge @@ -35,6 +35,7 @@ from synapse.logging.opentracing import ( whitelisted_homeserver, ) from synapse.metrics import SERVER_NAME_LABEL +from synapse.module_api.callbacks.federation import FederatedEventDeliveryMethod from synapse.types import JsonDict from synapse.util.json import json_decoder from synapse.util.metrics import measure_func @@ -64,6 +65,7 @@ class TransactionManager: self._store = hs.get_datastores().main self._transaction_actions = TransactionActions(self._store) self._transport_layer = hs.get_federation_transport_client() + self._federation_callbacks = hs.get_module_api_callbacks().federation self._federation_metrics_domains = ( hs.config.federation.federation_metrics_domains @@ -192,14 +194,39 @@ class TransactionManager: logger.info("TX [%s] {%s} got 200 response", destination, txn_id) - for e_id, r in response.get("pdus", {}).items(): - if "error" in r: - logger.warning( - "TX [%s] {%s} Remote returned error for %s: %s", + pdu_responses = response.get("pdus", {}) + if not isinstance(pdu_responses, Mapping): + logger.warning( + "TX [%s] {%s} Remote returned invalid type for `pdus`", + destination, + txn_id, + ) + else: + for event_id, pdu_response in pdu_responses.items(): + if not isinstance(pdu_response, Mapping) or "error" in pdu_response: + logger.warning( + "TX [%s] {%s} Remote returned error for %s: %s", + destination, + txn_id, + event_id, + pdu_response, + ) + + # If modules have requested to be notified about delivered events, + # build and send that notification. + if self._federation_callbacks.interested_in_events_delivered_over_federation(): + # A PDU is considered acknowledged when the remote echoes the event_id back to + # us, without an error in the PDU response dict. + acknowledged_pdu_ids = { + event_id + for event_id, pdu_response in response.get("pdus", {}).items() + if isinstance(pdu_response, Mapping) + and "error" not in pdu_response + } + await self._federation_callbacks.notify_on_event_delivered_over_federation( destination, - txn_id, - e_id, - r, + [p for p in pdus if p.event_id in acknowledged_pdu_ids], + FederatedEventDeliveryMethod.SEND, ) if pdus and destination in self._federation_metrics_domains: diff --git a/synapse/federation/transport/client.py b/synapse/federation/transport/client.py index 5d5212ef96..e4747016c4 100644 --- a/synapse/federation/transport/client.py +++ b/synapse/federation/transport/client.py @@ -776,18 +776,21 @@ class TransportLayerClient: limit: int, min_depth: int, timeout: int, + state_dag: bool, ) -> JsonDict: path = _create_v1_path("/get_missing_events/%s", room_id) - + request_body = { + "limit": int(limit), + "min_depth": int(min_depth), + "earliest_events": earliest_events, + "latest_events": latest_events, + } + if state_dag: + request_body["org.matrix.msc4242.state_dag"] = True return await self.client.post_json( destination=destination, path=path, - data={ - "limit": int(limit), - "min_depth": int(min_depth), - "earliest_events": earliest_events, - "latest_events": latest_events, - }, + data=request_body, timeout=timeout, ) @@ -986,6 +989,10 @@ class SendJoinResponse: # "event" is not included in the response. event: EventBase | None = None + # MSC4242: State DAGs. Always included for state dag rooms, else None. + # Replaces auth_events. + state_dag: list[EventBase] | None = None + # The room state is incomplete members_omitted: bool = False @@ -1068,7 +1075,7 @@ class SendJoinParser(ByteParser[SendJoinResponse]): MAX_RESPONSE_SIZE = 500 * 1024 * 1024 def __init__(self, room_version: RoomVersion, v1_api: bool): - self._response = SendJoinResponse([], [], event_dict={}) + self._response = SendJoinResponse([], [], event_dict={}, state_dag=[]) self._room_version = room_version self._coros: list[Generator[None, bytes, None]] = [] @@ -1112,6 +1119,15 @@ class SendJoinParser(ByteParser[SendJoinResponse]): ) ) + if room_version.msc4242_state_dags: + self._coros.append( + ijson.items_coro( + _event_list_parser(room_version, self._response.state_dag), + prefix + "state_dag.item", + use_float=True, + ) + ) + def write(self, data: bytes) -> int: for c in self._coros: c.send(data) diff --git a/synapse/federation/transport/server/__init__.py b/synapse/federation/transport/server/__init__.py index 0eff49cf73..70fc55123b 100644 --- a/synapse/federation/transport/server/__init__.py +++ b/synapse/federation/transport/server/__init__.py @@ -24,6 +24,7 @@ import logging from typing import TYPE_CHECKING, Iterable, Literal from synapse.api.errors import FederationDeniedError, SynapseError +from synapse.federation.transport.server import appservice_proxy from synapse.federation.transport.server._base import ( Authenticator, BaseFederationServlet, @@ -340,3 +341,6 @@ def register_servlets( ratelimiter=ratelimiter, server_name=hs.hostname, ).register(resource) + + if "federation" in servlet_groups: + appservice_proxy.register_servlets(hs, resource, authenticator, ratelimiter) diff --git a/synapse/federation/transport/server/appservice_proxy.py b/synapse/federation/transport/server/appservice_proxy.py new file mode 100644 index 0000000000..7ececd8fff --- /dev/null +++ b/synapse/federation/transport/server/appservice_proxy.py @@ -0,0 +1,115 @@ +# +# This file is licensed under the Affero General Public License (AGPL) version 3. +# +# Copyright (C) 2026 Element Creations Ltd +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# See the GNU Affero General Public License for more details: +# . +# + +import logging +import re +from http import HTTPStatus +from io import BytesIO +from typing import TYPE_CHECKING + +from synapse.api.errors import Codes, SynapseError +from synapse.appservice import ApplicationService +from synapse.federation.transport.server._base import Authenticator +from synapse.http import QuieterFileBodyProducer +from synapse.http.appservice_proxy import proxy_request_to_appservice +from synapse.http.server import HttpServer, ServletCallback +from synapse.http.site import SynapseRequest +from synapse.util.json import json_decoder +from synapse.util.ratelimitutils import FederationRateLimiter + +if TYPE_CHECKING: + from synapse.server import HomeServer + +logger = logging.getLogger(__name__) + + +def _make_proxy_callback( + hs: "HomeServer", + authenticator: Authenticator, + ratelimiter: FederationRateLimiter, + appservice: ApplicationService, +) -> ServletCallback: + """Builds a servlet callback that authenticates an incoming federation request, + rate-limits it by origin, and forwards it to the given application service's + proxy URL. + """ + + async def _proxy(request: SynapseRequest, **kwargs: str) -> None: + body_producer = None + content = None + + if request.method in (b"PUT", b"POST"): + raw_body = request.content.read() # type: ignore[union-attr] + try: + content = json_decoder.decode(raw_body.decode("utf-8")) + except Exception: + raise SynapseError( + HTTPStatus.BAD_REQUEST, "Content not JSON.", Codes.NOT_JSON + ) + body_producer = QuieterFileBodyProducer(BytesIO(raw_body)) + + origin = await authenticator.authenticate_request(request, content) + + # Apply the same per-origin rate limiting that every other federation endpoint gets. + with ratelimiter.ratelimit(origin) as d: + await d + if request._disconnected: + logger.warning( + "client disconnected before we started processing request" + ) + return + + await proxy_request_to_appservice( + request, + hs, + appservice, + body_producer, + extra_request_headers={b"X-Matrix-Origin": origin.encode("ascii")}, + ) + + return _proxy + + +def register_servlets( + hs: "HomeServer", + resource: HttpServer, + authenticator: Authenticator, + ratelimiter: FederationRateLimiter, +) -> None: + """Registers blanket reverse-proxy routes for each application service that has + configured a proxy prefix. This forwards requests under + /_matrix/federation///* (where is either "vN" or "unstable") + to the same path under the application service's proxy URL after verifying request + authentication. + """ + if not hs.config.experimental.msc4512_enabled: + return + + for appservice in hs.get_datastores().main.get_app_services(): + if appservice.proxy_prefix is None or appservice.proxy_url is None: + continue + + pattern = re.compile( + r"^/_matrix/federation/(?:unstable/[^/]+|v[^/]+)/%s(/.*)?$" + % (re.escape(appservice.proxy_prefix),) + ) + callback = _make_proxy_callback(hs, authenticator, ratelimiter, appservice) + + for method in ("GET", "POST", "PUT", "DELETE"): + resource.register_paths( + method, + (pattern,), + callback, + "ApplicationServiceFederationProxy", + ) diff --git a/synapse/federation/transport/server/federation.py b/synapse/federation/transport/server/federation.py index d783e6da51..8a8914bf4f 100644 --- a/synapse/federation/transport/server/federation.py +++ b/synapse/federation/transport/server/federation.py @@ -37,6 +37,7 @@ from synapse.federation.transport.server._base import ( BaseFederationServlet, ) from synapse.http.servlet import ( + parse_boolean, parse_boolean_from_args, parse_integer, parse_integer_from_args, @@ -46,7 +47,7 @@ from synapse.http.servlet import ( ) from synapse.http.site import SynapseRequest from synapse.media._base import DEFAULT_MAX_TIMEOUT_MS, MAXIMUM_ALLOWED_MAX_TIMEOUT_MS -from synapse.media.thumbnailer import ThumbnailProvider +from synapse.media.thumbnailer import ANIMATED_THUMBNAIL_TYPE, ThumbnailProvider from synapse.types import JsonDict from synapse.util import SYNAPSE_VERSION from synapse.util.ratelimitutils import FederationRateLimiter @@ -875,8 +876,9 @@ class FederationMediaThumbnailServlet(BaseFederationServerServlet): width = parse_integer(request, "width", required=True) height = parse_integer(request, "height", required=True) method = parse_string(request, "method", "scale") + animated = parse_boolean(request, "animated", default=False) # TODO Parse the Accept header to get an prioritised list of thumbnail types. - m_type = "image/png" + m_type = ANIMATED_THUMBNAIL_TYPE if animated else "image/png" max_timeout_ms = parse_integer( request, "timeout_ms", default=DEFAULT_MAX_TIMEOUT_MS ) @@ -884,7 +886,15 @@ class FederationMediaThumbnailServlet(BaseFederationServerServlet): if self.dynamic_thumbnails: await self.thumbnail_provider.select_or_generate_local_thumbnail( - request, media_id, width, height, method, m_type, max_timeout_ms, True + request, + media_id, + width, + height, + method, + m_type, + max_timeout_ms, + True, + animated=animated, ) else: await self.thumbnail_provider.respond_local_thumbnail( diff --git a/synapse/handlers/appservice.py b/synapse/handlers/appservice.py index 36b2f63e41..68b8aa71f1 100644 --- a/synapse/handlers/appservice.py +++ b/synapse/handlers/appservice.py @@ -313,7 +313,10 @@ class ApplicationServicesHandler: StreamKeyType.PRESENCE, StreamKeyType.TO_DEVICE, ) - and service.supports_ephemeral + # Honour both the stable `receive_ephemeral` registration flag and the + # legacy `de.sorunome.msc2409.push_ephemeral` one, matching the + # transaction body built in `ApplicationServiceApi.push_bulk`. + and (service.supports_ephemeral or service.supports_unstable_ephemeral) ) or ( stream_key == StreamKeyType.DEVICE_LIST diff --git a/synapse/handlers/deactivate_account.py b/synapse/handlers/deactivate_account.py index 9ec00d55ad..34596ade16 100644 --- a/synapse/handlers/deactivate_account.py +++ b/synapse/handlers/deactivate_account.py @@ -173,7 +173,9 @@ class DeactivateAccountHandler: # in rooms, but these cases behave like message history, following # https://spec.matrix.org/v1.17/client-server-api/#post_matrixclientv3accountdeactivate await self._profile_handler.delete_profile_upon_deactivation( - user, requester, by_admin + target_user=user, + requester=requester, + by_admin=by_admin, ) logger.info("Marking %s as erased", user_id) diff --git a/synapse/handlers/delayed_events.py b/synapse/handlers/delayed_events.py index f016d95e31..ac2da15112 100644 --- a/synapse/handlers/delayed_events.py +++ b/synapse/handlers/delayed_events.py @@ -13,12 +13,13 @@ # import logging +from http import HTTPStatus from typing import TYPE_CHECKING, Optional from twisted.internet.interfaces import IDelayedCall from synapse.api.constants import EventTypes, StickyEvent, StickyEventField -from synapse.api.errors import ShadowBanError, SynapseError +from synapse.api.errors import Codes, ShadowBanError, SynapseError from synapse.api.ratelimiting import Ratelimiter from synapse.config.workers import MAIN_PROCESS_INSTANCE_NAME from synapse.http.site import SynapseRequest @@ -30,6 +31,8 @@ from synapse.replication.http.delayed_events import ( ) from synapse.storage.databases.main.delayed_events import ( DelayedEventDetails, + DelayedEventResponse, + DelayedEventResponseLegacyCompat, EventType, StateKey, Timestamp, @@ -330,7 +333,7 @@ class DelayedEventsHandler: state_key: str | None, origin_server_ts: int | None, content: JsonDict, - delay: int, + delay: Duration, sticky_duration_ms: int | None, ) -> str: """ @@ -344,20 +347,37 @@ class DelayedEventsHandler: origin_server_ts: The custom timestamp to send the event with. If None, the timestamp will be the actual time when the event is sent. content: The content of the event to be sent. - delay: How long (in milliseconds) to wait before automatically sending the event. + delay: How long to wait before automatically sending the event. sticky_duration_ms: If an MSC4354 sticky event: the sticky duration (in milliseconds). The event will be attempted to be reliably delivered to clients and remote servers during its sticky period. Returns: The ID of the added delayed event. Raises: - SynapseError: if the delayed event fails validation checks. + SynapseError: if the delayed event fails validation checks, or + if the requested delay is longer than allowed, or + if sending delayed events has been disallowed entirely. """ # Use standard request limiter for scheduling new delayed events. # TODO: Instead apply ratelimiting based on the scheduled send time. # See https://github.com/element-hq/synapse/issues/18021 await self._request_ratelimiter.ratelimit(requester) + if not self._config.server.msc4140_enabled: + raise SynapseError( + HTTPStatus.FORBIDDEN, + "Sending delayed events has been disallowed", + Codes.FORBIDDEN, + ) + if delay > self._config.server.max_event_delay_duration: + requested_delay = delay.as_millis() + max_delay = self._config.server.max_event_delay_duration.as_millis() + raise SynapseError( + HTTPStatus.BAD_REQUEST, + f"The requested delay ({requested_delay}ms) exceeds the allowed maximum ({max_delay}ms)", + Codes.DELAY_TOO_LARGE, + ) + self._event_creation_handler.validator.validate_builder( self._event_creation_handler.event_builder_factory.for_room_version( await self._store.get_room_version(room_id), @@ -384,6 +404,7 @@ class DelayedEventsHandler: content=content, delay=delay, sticky_duration_ms=sticky_duration_ms, + limit=self._config.server.max_delayed_events_per_user, ) if self._repl_client is not None: @@ -530,8 +551,30 @@ class DelayedEventsHandler: else: self._next_delayed_event_call.reset(delay_duration.as_secs()) - async def get_all_for_user(self, requester: Requester) -> list[JsonDict]: - """Return all pending delayed events requested by the given user.""" + async def get_for_user( + self, requester: Requester, delay_id: str + ) -> DelayedEventResponse: + """ + Return the specified pending delayed event requested by the given user. + + Raises: + NotFoundError: if no matching delayed event could be found. + """ + await self._delayed_event_mgmt_ratelimiter.ratelimit(requester) + return await self._store.get_delayed_event_for_user( + delay_id, + requester.user.localpart, + ) + + async def get_all_for_user( + self, requester: Requester + ) -> list[DelayedEventResponseLegacyCompat]: + """ + Return all pending delayed events owned by the given user. + Includes fields from earlier revisions of MSC4140 for + compatibility with clients that still expect them. + """ + # TODO: Remove legacy fields once stable await self._delayed_event_mgmt_ratelimiter.ratelimit(requester) return await self._store.get_all_delayed_events_for_user( requester.user.localpart diff --git a/synapse/handlers/device.py b/synapse/handlers/device.py index 2225466648..ae61bc24bf 100644 --- a/synapse/handlers/device.py +++ b/synapse/handlers/device.py @@ -30,6 +30,8 @@ from typing import ( cast, ) +from prometheus_client import Gauge + from synapse.api import errors from synapse.api.constants import EduTypes, EventTypes, Membership from synapse.api.errors import ( @@ -41,6 +43,7 @@ from synapse.api.errors import ( SynapseError, ) from synapse.logging.opentracing import log_kv, set_tag, trace +from synapse.metrics import SERVER_NAME_LABEL from synapse.metrics.background_process_metrics import ( wrap_as_background_process, ) @@ -89,6 +92,21 @@ DELETE_DEVICE_MSGS_TASK_NAME = "delete_device_messages" MAX_DEVICE_DISPLAY_NAME_LEN = 100 DELETE_STALE_DEVICES_INTERVAL = Duration(days=1) +device_list_conversion_lag_gauge = Gauge( + "synapse_device_lists_changes_conversion_lag_seconds", + "Age of the oldest device list change that has yet to be converted to outbound federation pokes", + labelnames=[SERVER_NAME_LABEL], +) + +device_list_conversion_stream_lag_gauge = Gauge( + "synapse_device_lists_changes_conversion_stream_lag", + "Number of stream IDs between the current device lists stream position and the position converted to outbound federation pokes", + labelnames=[SERVER_NAME_LABEL], +) + +# How often to update the device list conversion lag gauges. +DEVICE_LIST_CONVERSION_LAG_GAUGE_METRIC_UPDATE_INTERVAL = Duration(seconds=30) + def _check_device_name_length(name: str | None) -> None: """ @@ -960,6 +978,13 @@ class DeviceWriterHandler(DeviceHandler): self.device_list_updater.incoming_device_list_update, ) + # Report how far behind we are at converting device list changes + # into outbound pokes. + self.clock.looping_call( + self._report_device_list_conversion_lag, + DEVICE_LIST_CONVERSION_LAG_GAUGE_METRIC_UPDATE_INTERVAL, + ) + @trace @measure_func("notify_device_update") async def notify_device_update( @@ -1033,6 +1058,35 @@ class DeviceWriterHandler(DeviceHandler): self._handle_new_device_update_async() return + @wrap_as_background_process("_report_device_list_conversion_lag") + async def _report_device_list_conversion_lag(self) -> None: + """Report how far behind we are at converting rows in + `device_lists_changes_in_room` to `device_lists_outbound_pokes`. + """ + ( + oldest_ts, + last_converted_pos, + ) = await self.store.get_device_list_conversion_lag() + + if oldest_ts is None: + device_list_conversion_lag_ms = 0 + else: + device_list_conversion_lag_ms = max(0, self.clock.time_msec() - oldest_ts) + + device_list_conversion_lag_gauge.labels( + **{SERVER_NAME_LABEL: self.server_name} + ).set(device_list_conversion_lag_ms / 1000.0) # convert to seconds + + # The stream ID lag is only an approximation of the conversion + # backlog: the converted position only advances when the conversion + # loop runs, and stream IDs in the gap may not have rows needing + # conversion at all. + current_pos = self.store.get_device_stream_token().stream + + device_list_conversion_stream_lag_gauge.labels( + **{SERVER_NAME_LABEL: self.server_name} + ).set(max(0, current_pos - last_converted_pos)) + @wrap_as_background_process("_handle_new_device_update_async") async def _handle_new_device_update_async(self) -> None: """Called when we have a new local device list update that we need to diff --git a/synapse/handlers/devicemessage.py b/synapse/handlers/devicemessage.py index 63f73bb832..6a64c1ccf7 100644 --- a/synapse/handlers/devicemessage.py +++ b/synapse/handlers/devicemessage.py @@ -121,6 +121,7 @@ class DeviceMessageHandler: origin, sender_user_id, ) + return message_type = content["type"] message_id = content["message_id"] for user_id, by_device in content["messages"].items(): diff --git a/synapse/handlers/e2e_keys.py b/synapse/handlers/e2e_keys.py index 64f705a3da..6284f75319 100644 --- a/synapse/handlers/e2e_keys.py +++ b/synapse/handlers/e2e_keys.py @@ -1272,21 +1272,27 @@ class E2eKeysHandler: master_key_signature_list = [] sigs = signed_master_key["signatures"] for signing_key_id, signature in sigs[user_id].items(): - _, signing_device_id = signing_key_id.split(":", 1) - if ( - signing_device_id not in devices - or signing_key_id not in devices[signing_device_id]["keys"] - ): - # signed by an unknown device, or the - # device does not have the key - raise SynapseError(400, "Invalid signature", Codes.INVALID_SIGNATURE) + algorithm, signing_device_id = signing_key_id.split(":", 1) + # we only check the signature for known algorithms + if algorithm == "ed25519": + if ( + signing_device_id not in devices + or signing_key_id not in devices[signing_device_id]["keys"] + ): + # signed by an unknown device, or the + # device does not have the key + raise SynapseError( + 400, "Invalid signature", Codes.INVALID_SIGNATURE + ) - # get the key and check the signature - pubkey = devices[signing_device_id]["keys"][signing_key_id] - verify_key = decode_verify_key_bytes(signing_key_id, decode_base64(pubkey)) - _check_device_signature( - user_id, verify_key, signed_master_key, stored_master_key - ) + # get the key and check the signature + pubkey = devices[signing_device_id]["keys"][signing_key_id] + verify_key = decode_verify_key_bytes( + signing_key_id, decode_base64(pubkey) + ) + _check_device_signature( + user_id, verify_key, signed_master_key, stored_master_key + ) master_key_signature_list.append( SignatureListItem(signing_key_id, user_id, master_key_id, signature) diff --git a/synapse/handlers/event_auth.py b/synapse/handlers/event_auth.py index a9a6990581..8c49d3e06f 100644 --- a/synapse/handlers/event_auth.py +++ b/synapse/handlers/event_auth.py @@ -234,11 +234,21 @@ class EventAuthHandler: # Get the rooms which allow access to this room and check if the user is # in any of them. - allowed_rooms = await self.get_rooms_that_allow_join(state_ids) + allowed_rooms, has_unknown_rules = await self.get_rooms_that_allow_join( + state_ids + ) if not await self.is_user_in_rooms(allowed_rooms, user_id): - # If this is a remote request, the user might be in an allowed room - # that we do not know about. - if not self._is_mine_id(user_id): + # If there are unknown allow rules, there could be other servers + # that are able to authorise the join. Alternatively, if this is + # a remote request, the user might be in an allowed room that we + # do not know about. + if has_unknown_rules: + raise SynapseError( + 400, + "Unrecognized restricted join rules found.", + Codes.UNABLE_AUTHORISE_JOIN, + ) + elif not self._is_mine_id(user_id): for room_id in allowed_rooms: if not await self._store.is_host_joined(room_id, self._server_name): raise SynapseError( @@ -289,7 +299,7 @@ class EventAuthHandler: async def get_rooms_that_allow_join( self, state_ids: StateMap[str] - ) -> StrCollection: + ) -> tuple[StrCollection, bool]: """ Generate a list of rooms in which membership allows access to a room. @@ -297,12 +307,16 @@ class EventAuthHandler: state_ids: The current state of the room the user wishes to join Returns: - A collection of room IDs. Membership in any of the rooms in the list grants the ability to join the target room. + A tuple of a collection of room IDs and a boolean indicating whether + there are any unknown allow rules. Membership in any of the rooms in + the list grants the ability to join the target room. If unknown rules + are present, failure to authorise the join should not be treated as + a permanent error. """ # If there's no join rule, then it defaults to invite (so this doesn't apply). join_rules_event_id = state_ids.get((EventTypes.JoinRules, ""), None) if not join_rules_event_id: - return () + return (), False # If the join rule is not restricted, this doesn't apply. join_rules_event = await self._store.get_event(join_rules_event_id) @@ -310,16 +324,18 @@ class EventAuthHandler: # If allowed is of the wrong form, then only allow invited users. allow_list = join_rules_event.content.get("allow", []) if not isinstance(allow_list, list): - return () + return (), False # Pull out the other room IDs, invalid data gets filtered. result = [] + has_unknown_rules = False for allow in allow_list: if not isinstance(allow, dict): continue # If the type is unexpected, skip it. if allow.get("type") != RestrictedJoinRuleTypes.ROOM_MEMBERSHIP: + has_unknown_rules = True continue room_id = allow.get("room_id") @@ -328,7 +344,7 @@ class EventAuthHandler: result.append(room_id) - return result + return result, has_unknown_rules async def is_user_in_rooms(self, room_ids: StrCollection, user_id: str) -> bool: """ diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index ba83d4fd26..c67974c575 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -53,6 +53,7 @@ from synapse.api.errors import ( PartialStateConflictError, RequestSendFailed, SynapseError, + UnsupportedRoomVersionError, ) from synapse.api.room_versions import KNOWN_ROOM_VERSIONS, RoomVersion from synapse.crypto.event_signing import compute_event_signature @@ -582,8 +583,8 @@ class FederationHandler: return pdu - async def on_event_auth(self, event_id: str) -> list[EventBase]: - event = await self.store.get_event(event_id) + async def on_event_auth(self, event_id: str, room_id: str) -> list[EventBase]: + event = await self.store.get_event(event_id, check_room_id=room_id) auth = await self.store.get_auth_chain( event.room_id, list(event.auth_event_ids()), include_given=True ) @@ -669,6 +670,12 @@ class FederationHandler: room_id ) + # See related restriction in /createRoom requests in handlers/room.py + if room_version_obj.msc4242_state_dags: + raise UnsupportedRoomVersionError( + "Homeserver does not support this room version over federation" + ) + ret = await self.federation_client.send_join( host_list, event, @@ -797,10 +804,55 @@ class FederationHandler: if not predecessor or not isinstance(predecessor.get("room_id"), str): return event.event_id, max_stream_id old_room_id = predecessor["room_id"] - logger.debug( - "Found predecessor for %s during remote join: %s", room_id, old_room_id + + # We can't take the new room's word for it. + # Check to see that the predecessor room consents to the + # room upgrade. + if not await self._event_auth_handler.is_host_in_room( + room_id=old_room_id, host=self.hs.hostname + ): + logger.info( + "Ignoring unverified predecessor for %s during remote join: %s (not in old room)", + room_id, + old_room_id, + ) + return event.event_id, max_stream_id + + tombstone = await self._state_storage_controller.get_current_state_event( + old_room_id, + event_type=EventTypes.Tombstone, + state_key="", ) + if tombstone is None: + logger.warning( + "Ignoring unverified predecessor for %s during remote join: %s (no tombstone in old room)", + room_id, + old_room_id, + ) + return event.event_id, max_stream_id + + intended_successor_room = tombstone.content.get( + EventContentFields.TOMBSTONE_SUCCESSOR_ROOM, None + ) + + if not isinstance(intended_successor_room, str): + logger.warning( + "Ignoring unverified predecessor for %s during remote join: %s (tombstone is invalid)", + room_id, + old_room_id, + ) + return event.event_id, max_stream_id + + if intended_successor_room != room_id: + logger.warning( + "Ignoring unverified predecessor for %s during remote join: predecessor defined as %s (the old room ID) but the old room's tombstone points to %r which doesn't match", + room_id, + old_room_id, + intended_successor_room, + ) + return event.event_id, max_stream_id + # We retrieve the room member handler here as to not cause a cyclic dependency member_handler = self.hs.get_room_member_handler() await member_handler.transfer_room_state_on_room_upgrade( @@ -1216,6 +1268,7 @@ class FederationHandler: assert event.sender == user_id assert event.state_key == user_id assert event.room_id == room_id + assert event.content.get("membership") == membership return origin, event, room_version async def on_make_leave_request( @@ -1539,7 +1592,14 @@ class FederationHandler: if i == max_retries - 1: raise e else: - destinations = {x.split(":", 1)[-1] for x in (sender_user_id, room_id)} + # The sender always tells us a server to try. Pre-v12 room IDs also + # encode the resident server's domain, but v12+ room IDs are a hash + # with no domain component, so we must not treat them as a server + # name -- doing so raises an invalid-destination error which can + # abort the whole exchange before the valid destination is tried. + destinations = {get_domain_from_id(sender_user_id)} + if ":" in room_id: + destinations.add(get_domain_from_id(room_id)) try: await self.federation_client.forward_third_party_invite( diff --git a/synapse/handlers/message.py b/synapse/handlers/message.py index b34ee9d50f..c61fccb5f2 100644 --- a/synapse/handlers/message.py +++ b/synapse/handlers/message.py @@ -704,6 +704,9 @@ class EventCreationHandler: Codes.USER_ACCOUNT_SUSPENDED, ) + if event_dict["type"] == EventTypes.Redaction: + await self._check_redaction_allowed_period(event_dict) + is_create_event = ( event_dict["type"] == EventTypes.Create and event_dict["state_key"] == "" ) @@ -2236,6 +2239,48 @@ class EventCreationHandler: return bool(original_event and sender != original_event.sender) + async def _check_redaction_allowed_period(self, event_dict: dict) -> None: + """Reject a redaction of an `m.room.message` older than the configured period. + + Only applies to `m.room.message` targets. Enforced for local users only + (federated redactions bypass `create_event`). When the target is an edit + (`m.replace`), the age and type of the original event are used, not the + edit. + """ + period = self.config.server.redaction_allowed_period + if period is None: + return + + redacts = event_dict["content"].get("redacts") or event_dict.get("redacts") + room_id = event_dict["room_id"] + + if redacts is None: + return + + target = await self.store.get_event( + redacts, check_room_id=room_id, allow_none=True + ) + if target is None: + return + + relation = relation_from_event(target) + if relation is not None and relation.rel_type == RelationTypes.REPLACE: + original = await self.store.get_event( + relation.parent_id, check_room_id=room_id, allow_none=True + ) + if original is not None: + target = original + + if target.type != EventTypes.Message: + return + + if target.origin_server_ts < self.clock.time_msec() - period: + raise SynapseError( + 403, + f"Events older than {period}ms cannot be redacted.", + Codes.FORBIDDEN, + ) + async def _maybe_kick_guest_users( self, event: EventBase, context: EventContext ) -> None: diff --git a/synapse/handlers/presence.py b/synapse/handlers/presence.py index 28fbbfc1e1..55dd5ffb59 100644 --- a/synapse/handlers/presence.py +++ b/synapse/handlers/presence.py @@ -122,6 +122,7 @@ from synapse.types import ( ) from synapse.util.async_helpers import Linearizer from synapse.util.duration import Duration +from synapse.util.iterutils import batch_iter from synapse.util.metrics import Measure from synapse.util.wheel_timer import WheelTimer @@ -222,6 +223,12 @@ class BasePresenceHandler(abc.ABC): self._presence_enabled = hs.config.server.presence_enabled self._track_presence = hs.config.server.track_presence + # Rooms which, on their own, should not cause presence to be routed + # between their members. See `exclude_rooms_from_presence` in the config. + self._rooms_to_exclude_from_presence = frozenset( + hs.config.server.rooms_to_exclude_from_presence + ) + # The (configurable) presence state machine timers. self._last_active_granularity = ( hs.config.server.presence_last_active_granularity @@ -435,6 +442,7 @@ class BasePresenceHandler(abc.ABC): self.store, self.presence_router, states, + self._rooms_to_exclude_from_presence, ) for destinations, host_states in hosts_to_states: @@ -688,7 +696,12 @@ class WorkerPresenceHandler(BasePresenceHandler): async def notify_from_replication( self, states: list[UserPresenceState], stream_id: int ) -> None: - parties = await get_interested_parties(self.store, self.presence_router, states) + parties = await get_interested_parties( + self.store, + self.presence_router, + states, + self._rooms_to_exclude_from_presence, + ) room_ids_to_states, users_to_states = parties self.notifier.on_new_event( @@ -972,6 +985,14 @@ class PresenceHandler(BasePresenceHandler): Duration(minutes=1), ) + if not self._presence_enabled and self.user_to_current_state: + # Presence is disabled but the database still contains non-offline + # presence states, i.e. presence used to be enabled. Nothing writes + # to the presence stream while presence is disabled, so without + # intervention clients would show the stale states forever. Send + # out one final round of updates marking everyone as offline. + self.clock.call_when_running(self._mark_stale_presence_as_offline) + presence_wheel_timer_size_gauge.register_hook( homeserver_instance_id=hs.get_instance_id(), hook=lambda: {(self.server_name,): len(self.wheel_timer)}, @@ -1029,6 +1050,36 @@ class PresenceHandler(BasePresenceHandler): [self.user_to_current_state[user_id] for user_id in unpersisted] ) + @wrap_as_background_process("PresenceHandler._mark_stale_presence_as_offline") + async def _mark_stale_presence_as_offline(self) -> None: + """One-off job, run at startup when presence is disabled, that marks + any non-offline presence states left over from when presence was + enabled as offline, and streams the changes out to clients. + """ + states = [ + state.copy_and_replace( + state=PresenceState.OFFLINE, + status_msg=None, + currently_active=False, + ) + for state in self.user_to_current_state.values() + if state.state != PresenceState.OFFLINE + ] + if not states: + return + + logger.info( + "Presence is disabled: marking %d stale presence states as offline", + len(states), + ) + + self.user_to_current_state.update({state.user_id: state for state in states}) + + # There may be a lot of stale states (e.g. everyone that was online + # when presence was disabled), so persist them in batches. + for batch in batch_iter(states, 500): + await self._persist_and_notify(list(batch)) + async def _update_states( self, new_states: Iterable[UserPresenceState], @@ -1141,6 +1192,7 @@ class PresenceHandler(BasePresenceHandler): self.store, self.presence_router, list(to_federation_ping.values()), + self._rooms_to_exclude_from_presence, ) for destinations, states in hosts_to_states: @@ -1414,7 +1466,12 @@ class PresenceHandler(BasePresenceHandler): """ stream_id, max_token = await self.store.update_presence(states) - parties = await get_interested_parties(self.store, self.presence_router, states) + parties = await get_interested_parties( + self.store, + self.presence_router, + states, + self._rooms_to_exclude_from_presence, + ) room_ids_to_states, users_to_states = parties self.notifier.on_new_event( @@ -1561,7 +1618,10 @@ class PresenceHandler(BasePresenceHandler): observed_user.to_string() ) - if observer_room_ids & observed_room_ids: + shared_room_ids = ( + observer_room_ids & observed_room_ids + ) - self._rooms_to_exclude_from_presence + if shared_room_ids: return True return False @@ -1672,6 +1732,12 @@ class PresenceHandler(BasePresenceHandler): to be handled. """ + # Excluded rooms should not, on their own, share presence between their + # members. This method is entirely per-room presence fan-out, so skip + # excluded rooms wholesale. + if room_id in self._rooms_to_exclude_from_presence: + return + # Sets of newly joined users. Note that if the local server is # joining a remote room for the first time we'll see both the joining # user and all remote users as newly joined. @@ -1929,6 +1995,9 @@ class PresenceEventSource(EventSource[int, UserPresenceState]): self.server_name = hs.hostname self.clock = hs.get_clock() self.store = hs.get_datastores().main + self._rooms_to_exclude_from_presence = frozenset( + hs.config.server.rooms_to_exclude_from_presence + ) async def get_new_events( self, @@ -2043,9 +2112,31 @@ class PresenceEventSource(EventSource[int, UserPresenceState]): **{SERVER_NAME_LABEL: self.server_name}, ).inc() - sharing_users = await self.store.do_users_share_a_room( - user_id, updated_users - ) + # An updated user is interesting if they share a + # (non-excluded) room with the syncing user. We check by + # intersecting the cached per-user room sets rather than via + # `do_users_share_a_room`: its per-pair cache has a + # quadratic working set and is cleared wholesale on every + # membership change, so on busy servers every check missed + # into SQL. + # + # For every presence update we need to run this code for + # every user that is currently syncing. The + # `get_rooms_for_user` will therefore be computed only once + # for each updated user regardless of the number of syncing + # users. + # + # The syncing user's rooms will also be cached as its needed + # during sync processing anyway. + my_rooms = await self.store.get_rooms_for_user(user_id) + if self._rooms_to_exclude_from_presence: + my_rooms = my_rooms - self._rooms_to_exclude_from_presence + rooms_by_user = await self.store.get_rooms_for_users(updated_users) + sharing_users = { + updated_user + for updated_user, rooms in rooms_by_user.items() + if not my_rooms.isdisjoint(rooms) + } interested_and_updated_users = ( sharing_users.union(additional_users_interested_in) @@ -2060,7 +2151,9 @@ class PresenceEventSource(EventSource[int, UserPresenceState]): ).inc() users_interested_in = ( - await self.store.get_users_who_share_room_with_user(user_id) + await self.store.get_users_who_share_room_with_user( + user_id, self._rooms_to_exclude_from_presence + ) ) users_interested_in.update(additional_users_interested_in) @@ -2073,7 +2166,9 @@ class PresenceEventSource(EventSource[int, UserPresenceState]): # No from_key has been specified. Return the presence for all users # this user is interested in interested_and_updated_users = ( - await self.store.get_users_who_share_room_with_user(user_id) + await self.store.get_users_who_share_room_with_user( + user_id, self._rooms_to_exclude_from_presence + ) ) interested_and_updated_users.update(additional_users_interested_in) @@ -2473,7 +2568,10 @@ def _combine_device_states( async def get_interested_parties( - store: DataStore, presence_router: PresenceRouter, states: list[UserPresenceState] + store: DataStore, + presence_router: PresenceRouter, + states: list[UserPresenceState], + excluded_rooms: AbstractSet[str] = frozenset(), ) -> tuple[dict[str, list[UserPresenceState]], dict[str, list[UserPresenceState]]]: """Given a list of states return which entities (rooms, users) are interested in the given states. @@ -2482,6 +2580,8 @@ async def get_interested_parties( store: The homeserver's data store. presence_router: A module for augmenting the destinations for presence updates. states: A list of incoming user presence updates. + excluded_rooms: Rooms which should not, on their own, cause presence to + be routed between their members. Returns: A 2-tuple of `(room_ids_to_states, users_to_states)`, @@ -2492,6 +2592,8 @@ async def get_interested_parties( for state in states: room_ids = await store.get_rooms_for_user(state.user_id) for room_id in room_ids: + if room_id in excluded_rooms: + continue room_ids_to_states.setdefault(room_id, []).append(state) # Always notify self @@ -2512,6 +2614,7 @@ async def get_interested_remotes( store: DataStore, presence_router: PresenceRouter, states: list[UserPresenceState], + excluded_rooms: AbstractSet[str] = frozenset(), ) -> list[tuple[StrCollection, Collection[UserPresenceState]]]: """Given a list of presence states figure out which remote servers should be sent which. @@ -2522,6 +2625,8 @@ async def get_interested_remotes( store: The homeserver's data store. presence_router: A module for augmenting the destinations for presence updates. states: A list of incoming user presence updates. + excluded_rooms: Rooms which should not, on their own, cause presence to + be routed to their remote members. Returns: A map from destinations to presence states to send to that destination. @@ -2535,6 +2640,8 @@ async def get_interested_remotes( room_ids = await store.get_rooms_for_user(state.user_id) hosts: set[str] = set() for room_id in room_ids: + if room_id in excluded_rooms: + continue room_hosts = await store.get_current_hosts_in_room(room_id) hosts.update(room_hosts) hosts_and_states.append((hosts, [state])) diff --git a/synapse/handlers/profile.py b/synapse/handlers/profile.py index c3886795b6..112ac4c8c5 100644 --- a/synapse/handlers/profile.py +++ b/synapse/handlers/profile.py @@ -34,6 +34,10 @@ from synapse.api.errors import ( StoreError, SynapseError, ) +from synapse.replication.http.profile import ( + ReplicationProfileDeleteField, + ReplicationProfileSetField, +) from synapse.storage.databases.main.media_repository import LocalMedia, RemoteMedia from synapse.storage.roommember import ProfileInfo from synapse.types import ( @@ -42,6 +46,7 @@ from synapse.types import ( JsonValue, Requester, ScheduledTask, + StreamKeyType, TaskStatus, UserID, create_requester, @@ -75,6 +80,7 @@ class ProfileHandler: self.clock = hs.get_clock() # nb must be called this for @cached self.store = hs.get_datastores().main self.hs = hs + self._notifier = hs.get_notifier() self.federation = hs.get_federation_client() hs.get_federation_registry().register_query_handler( @@ -99,6 +105,19 @@ class ProfileHandler: ) self._worker_locks = hs.get_worker_locks_handler() + # Profile updates stream + self._include_profile_updates_in_sync = ( + hs.config.server.include_profile_updates_in_sync + ) + self._is_events_writer = ( + hs.get_instance_name() in hs.config.worker.writers.events + ) + self._delete_profile_field_client = ReplicationProfileDeleteField.make_client( + self.hs + ) + self._set_profile_field_client = ReplicationProfileSetField.make_client(self.hs) + self._profile_updates_writer_instance = self.hs.config.worker.writers.events[0] + async def get_profile(self, user_id: str, ignore_backoff: bool = True) -> JsonDict: """ Get a user's profile as a JSON dictionary. @@ -191,13 +210,13 @@ class ProfileHandler: async def set_displayname( self, + *, target_user: UserID, requester: Requester, new_displayname: str, - *, by_admin: bool = False, propagate: bool = True, - ) -> None: + ) -> int | None: """Set the displayname of a user Preconditions: @@ -207,12 +226,17 @@ class ProfileHandler: updates into rooms, which could cause rooms to be accidentally joined after the deactivated user has left them. + FIXME: This precondition seems to lack a test. + Args: target_user: the user whose displayname is to be changed. requester: The user attempting to make this change. new_displayname: The displayname to give this user. by_admin: Whether this change was made by an administrator. propagate: Whether this change also applies to the user's membership events. + + Returns: + Stream ID of the profile updates stream row that was just inserted. """ if not self.hs.is_mine(target_user): raise SynapseError(400, "User is not hosted on this homeserver") @@ -223,8 +247,11 @@ class ProfileHandler: if not by_admin and not self.hs.config.registration.enable_set_displayname: profile = await self.store.get_profileinfo(target_user) if profile.display_name: + # The spec reserves 400 for malformed input; disabled profile + # modifications are covered by the 403 response of + # https://spec.matrix.org/v1.19/client-server-api/#put_matrixclientv3profileuseridkeyname raise SynapseError( - 400, + 403, "Changing display name is disabled on this server", Codes.FORBIDDEN, ) @@ -252,7 +279,11 @@ class ProfileHandler: authenticated_entity=requester.authenticated_entity, ) - await self.store.set_profile_displayname(target_user, displayname_to_set) + stream_id = await self.store.set_profile_field( + target_user, + ProfileFields.DISPLAYNAME, + displayname_to_set, + ) profile = await self.store.get_profileinfo(target_user) @@ -267,6 +298,8 @@ class ProfileHandler: if propagate: await self._update_join_states(requester, target_user) + return stream_id + async def get_avatar_url(self, target_user: UserID) -> str | None: """ Fetch a user's avatar URL from their profile. @@ -302,13 +335,13 @@ class ProfileHandler: async def set_avatar_url( self, + *, target_user: UserID, requester: Requester, new_avatar_url: str, - *, by_admin: bool = False, propagate: bool = True, - ) -> None: + ) -> int | None: """Set a new avatar URL for a user. Preconditions: @@ -318,12 +351,17 @@ class ProfileHandler: updates into rooms, which could cause rooms to be accidentally joined after the deactivated user has left them. + FIXME: This precondition seems to lack a test. + Args: target_user: the user whose avatar URL is to be changed. requester: The user attempting to make this change. new_avatar_url: The avatar URL to give this user. by_admin: Whether this change was made by an administrator. propagate: Whether this change also applies to the user's membership events. + + Returns: + Stream ID of the profile updates stream row that was just inserted. """ if not self.hs.is_mine(target_user): raise SynapseError(400, "User is not hosted on this homeserver") @@ -334,8 +372,11 @@ class ProfileHandler: if not by_admin and not self.hs.config.registration.enable_set_avatar_url: profile = await self.store.get_profileinfo(target_user) if profile.avatar_url: + # The spec reserves 400 for malformed input; disabled profile + # modifications are covered by the 403 response of + # https://spec.matrix.org/v1.19/client-server-api/#put_matrixclientv3profileuseridkeyname raise SynapseError( - 400, "Changing avatar is disabled on this server", Codes.FORBIDDEN + 403, "Changing avatar is disabled on this server", Codes.FORBIDDEN ) if not isinstance(new_avatar_url, str): @@ -361,7 +402,11 @@ class ProfileHandler: target_user, authenticated_entity=requester.authenticated_entity ) - await self.store.set_profile_avatar_url(target_user, avatar_url_to_set) + stream_id = await self.store.set_profile_field( + target_user, + ProfileFields.AVATAR_URL, + avatar_url_to_set, + ) profile = await self.store.get_profileinfo(target_user) @@ -376,6 +421,8 @@ class ProfileHandler: if propagate: await self._update_join_states(requester, target_user) + return stream_id + async def delete_profile_upon_deactivation( self, target_user: UserID, @@ -394,6 +441,9 @@ class ProfileHandler: **leave** the room on the user's behalf, so there's no point sending new join events into rooms to propagate the profile deletion. See the `users_pending_deactivation` table and the associated user parter loop. + + Profile update streams are NOT updated in any way; this happens when the + event persister processes the room leave events triggered elsewhere as above. """ if not self.hs.is_mine(target_user): raise SynapseError(400, "User is not hosted on this homeserver") @@ -406,7 +456,10 @@ class ProfileHandler: # have it. raise AuthError(400, "Cannot remove another user's profile") - await self.store.delete_profile(target_user) + # Record the profile delete + await self.store.delete_profile( + user_id=target_user, + ) await self._third_party_rules.on_profile_update( target_user.to_string(), @@ -415,6 +468,50 @@ class ProfileHandler: deactivation=True, ) + async def dispatch_set_profile_field( + self, + *, + target_user: UserID, + requester: Requester, + field_name: str, + new_value: JsonValue | dict[str, JsonValue], + by_admin: bool = False, + propagate: bool = True, + ) -> None: + """ + Dispatch setting a profile field value. This either happens in the same + instance, if configured for profile updates, or via replication in the + right instance. + + Args: + target_user: the user whose profile field is to be changed. + requester: The user attempting to make this change. + field_name: The field name to update. + new_value: New value for the profile field. + by_admin: Whether this change was made by an administrator. + propagate: Whether this change also applies to the user's membership events. + """ + if self._is_events_writer: + await self.set_field( + target_user=target_user, + requester=requester, + field_name=field_name, + new_value=new_value, + by_admin=by_admin, + propagate=propagate, + ) + else: + # Offload to the right worker via http replication + await self._set_profile_field_client( + instance_name=self._profile_updates_writer_instance, + user_id=target_user.to_string(), + requester=requester, + field_name=field_name, + new_value=new_value, + by_admin=by_admin, + propagate=propagate, + ) + @cached() async def check_avatar_size_and_mime_type(self, mxc: str) -> bool: """Check that the size and content type of the avatar at the given MXC URI are @@ -492,7 +589,7 @@ class ProfileHandler: async def get_profile_field( self, target_user: UserID, field_name: str - ) -> JsonValue: + ) -> JsonValue | dict[str, JsonValue]: """ Fetch a user's profile from the database for local users and over federation for remote users. @@ -530,15 +627,77 @@ class ProfileHandler: return result.get(field_name) - async def set_profile_field( + async def set_field( self, + *, target_user: UserID, requester: Requester, field_name: str, - new_value: JsonValue, - *, + new_value: JsonValue | dict[str, JsonValue], by_admin: bool = False, + propagate: bool = True, ) -> None: + """Wrapper function for setting any profile field for a user.""" + if field_name == ProfileFields.DISPLAYNAME: + if not isinstance(new_value, str): + raise SynapseError( + 400, "'displayname' must be a string", errcode=Codes.INVALID_PARAM + ) + stream_id = await self.set_displayname( + target_user=target_user, + requester=requester, + new_displayname=new_value, + by_admin=by_admin, + propagate=propagate, + ) + elif field_name == ProfileFields.AVATAR_URL: + if not isinstance(new_value, str): + raise SynapseError( + 400, "'avatar_url' must be a string", errcode=Codes.INVALID_PARAM + ) + stream_id = await self.set_avatar_url( + target_user=target_user, + requester=requester, + new_avatar_url=new_value, + by_admin=by_admin, + propagate=propagate, + ) + else: + stream_id = await self.set_profile_field( + target_user=target_user, + requester=requester, + field_name=field_name, + new_value=new_value, + by_admin=by_admin, + ) + + if stream_id is not None: + room_ids = await self.store.get_rooms_for_user(target_user.to_string()) + if room_ids: + # Wake up the stream for the rooms involved + self._notifier.on_new_event( + StreamKeyType.PROFILE_UPDATES, + stream_id, + rooms=room_ids, + ) + else: + # Wake up the stream for ourselves, as we might be updating our + # profile even if we don't have rooms + self._notifier.on_new_event( + StreamKeyType.PROFILE_UPDATES, + stream_id, + users=[target_user], + ) + + async def set_profile_field( + self, + *, + target_user: UserID, + requester: Requester, + field_name: str, + new_value: JsonValue | dict[str, JsonValue], + by_admin: bool = False, + ) -> int | None: """Set a new profile field for a user. Preconditions: @@ -546,6 +705,8 @@ class ProfileHandler: notify modules about the change whilst claiming it is not related to user deactivation. + FIXME: This precondition seems to lack a test. + Args: target_user: the user whose profile is to be changed. requester: The user attempting to make this change. @@ -559,7 +720,16 @@ class ProfileHandler: if not by_admin and target_user != requester.user: raise AuthError(403, "Cannot set another user's profile") - await self.store.set_profile_field(target_user, field_name, new_value) + # Don't recreate a profile row for a user that does not exist at all; + # deactivated (e.g. erased) users do exist, so are allowed through. + if await self.store.get_user_by_id(target_user.to_string()) is None: + raise SynapseError(404, "User not found", Codes.NOT_FOUND) + + stream_id = await self.store.set_profile_field( + target_user, + field_name, + new_value, + ) # Custom fields do not propagate into the user directory *or* rooms. profile = await self.store.get_profileinfo(target_user) @@ -567,6 +737,48 @@ class ProfileHandler: target_user.to_string(), profile, by_admin, deactivation=False ) + return stream_id + + async def dispatch_delete_profile_field( + self, + *, + target_user: UserID, + requester: Requester, + field_name: str, + by_admin: bool = False, + ) -> None: + """ + Dispatch deleting a profile field value. This either happens in the same + instance, if configured for profile updates, or via replication in the + right instance. + + To delete a displayname / avatar_uri, use the `dispatch_set_profile_field` + method, using an empty string as the value. + + Args: + target_user: the user whose profile field is to be changed. + requester: The user attempting to make this change. + field_name: The field name to update. + by_admin: Whether this change was made by an administrator. + """ + assert field_name not in (ProfileFields.DISPLAYNAME, ProfileFields.AVATAR_URL) + if self._is_events_writer: + await self.delete_profile_field( + target_user=target_user, + requester=requester, + field_name=field_name, + by_admin=by_admin, + ) + else: + # Offload to the right worker via http replication + await self._delete_profile_field_client( + instance_name=self._profile_updates_writer_instance, + user_id=target_user.to_string(), + requester=requester, + field_name=field_name, + by_admin=by_admin, + ) + async def delete_profile_field( self, target_user: UserID, @@ -577,24 +789,34 @@ class ProfileHandler: ) -> None: """Delete a field from a user's profile. + This should only be called for custom profile fields, + not displayname or avatar_url. + Preconditions: - This must NOT be called as part of deactivating the user, because we will notify modules about the change whilst claiming it is not related to user deactivation. + FIXME: This precondition seems to lack a test. + Args: target_user: the user whose profile is to be changed. requester: The user attempting to make this change. field_name: The name of the profile field to remove. by_admin: Whether this change was made by an administrator. """ + assert field_name not in (ProfileFields.DISPLAYNAME, ProfileFields.AVATAR_URL) + if not self.hs.is_mine(target_user): raise SynapseError(400, "User is not hosted on this homeserver") if not by_admin and target_user != requester.user: raise AuthError(400, "Cannot set another user's profile") - await self.store.delete_profile_field(target_user, field_name) + stream_id = await self.store.delete_profile_field( + target_user, + field_name, + ) # Custom fields do not propagate into the user directory *or* rooms. profile = await self.store.get_profileinfo(target_user) @@ -602,6 +824,24 @@ class ProfileHandler: target_user.to_string(), profile, by_admin, deactivation=False ) + if stream_id: + room_ids = await self.store.get_rooms_for_user(target_user.to_string()) + if room_ids: + # Wake up the stream for the rooms involved + self._notifier.on_new_event( + StreamKeyType.PROFILE_UPDATES, + stream_id, + rooms=room_ids, + ) + else: + # Wake up the stream for ourselves, as we might be updating our + # profile even if we don't have rooms + self._notifier.on_new_event( + StreamKeyType.PROFILE_UPDATES, + stream_id, + users=[target_user], + ) + async def on_profile_query(self, args: JsonDict) -> JsonDict: """Handles federation profile query requests.""" diff --git a/synapse/handlers/read_marker.py b/synapse/handlers/read_marker.py index 85d2dd62bb..3f3b9e6d8b 100644 --- a/synapse/handlers/read_marker.py +++ b/synapse/handlers/read_marker.py @@ -41,7 +41,11 @@ class ReadMarkerHandler: ) async def received_client_read_marker( - self, room_id: str, user_id: str, event_id: str + self, + room_id: str, + user_id: str, + event_id: str, + allow_backward: bool = False, ) -> None: """Updates the read marker for a given user in a given room if the event ID given is ahead in the stream relative to the current read marker. @@ -59,7 +63,7 @@ class ReadMarkerHandler: # Get event ordering, this also ensures we know about the event event_ordering = await self.store.get_event_ordering(event_id, room_id) - if existing_read_marker: + if existing_read_marker and not allow_backward: try: old_event_ordering = await self.store.get_event_ordering( existing_read_marker["event_id"], room_id diff --git a/synapse/handlers/receipts.py b/synapse/handlers/receipts.py index f6383baf0b..c208b4e91a 100644 --- a/synapse/handlers/receipts.py +++ b/synapse/handlers/receipts.py @@ -19,7 +19,7 @@ # # import logging -from typing import TYPE_CHECKING, Iterable, Sequence +from typing import TYPE_CHECKING, Callable, Iterable, Sequence from synapse.api.constants import EduTypes, ReceiptTypes from synapse.appservice import ApplicationService @@ -228,6 +228,31 @@ class ReceiptEventSource(EventSource[MultiWriterStreamToken, JsonMapping]): Args: rooms: A list of mappings, each mapping has a `content` field, which is a map of event ID -> receipt type -> user ID -> receipt information. + user_id: The user whose private read receipts should be kept. + + Returns: + The same as rooms, but filtered. + """ + return ReceiptEventSource._filter_private_receipts( + rooms, lambda receipt_user_id: receipt_user_id == user_id + ) + + @staticmethod + def _filter_private_receipts( + rooms: Sequence[JsonMapping], is_visible: Callable[[str], bool] + ) -> list[JsonMapping]: + """ + Filters a list of serialized receipts and removes private read receipts + of users for which `is_visible` returns False. + + This may operate on the return value of cached functions. Care must be + taken to ensure that the input values are not modified. + + Args: + rooms: A list of mappings, each mapping has a `content` field, which + is a map of event ID -> receipt type -> user ID -> receipt information. + is_visible: Called with each private read receipt's user ID; returns + whether that user's private read receipts may be included. Returns: The same as rooms, but filtered. @@ -237,7 +262,7 @@ class ReceiptEventSource(EventSource[MultiWriterStreamToken, JsonMapping]): # Iterate through each room's receipt content. for room in rooms: - # The receipt content with other user's private read receipts removed. + # The receipt content with hidden users' private read receipts removed. content = {} # Iterate over each event ID / receipts for that event. @@ -246,30 +271,34 @@ class ReceiptEventSource(EventSource[MultiWriterStreamToken, JsonMapping]): # If there are private read receipts, additional logic is necessary. if ReceiptTypes.READ_PRIVATE in event_content: # Make a copy without private read receipts to avoid leaking - # other user's private read receipts.. + # hidden users' private read receipts.. event_content = { receipt_type: receipt_value for receipt_type, receipt_value in event_content.items() if receipt_type != ReceiptTypes.READ_PRIVATE } - # Copy the current user's private read receipt from the - # original content, if it exists. - user_private_read_receipt = orig_event_content[ - ReceiptTypes.READ_PRIVATE - ].get(user_id, None) - if user_private_read_receipt: - event_content[ReceiptTypes.READ_PRIVATE] = { - user_id: user_private_read_receipt - } + # Copy the visible users' private read receipts from the + # original content, if there are any. + visible_private_read_receipts = { + receipt_user_id: receipt_value + for receipt_user_id, receipt_value in orig_event_content[ + ReceiptTypes.READ_PRIVATE + ].items() + if is_visible(receipt_user_id) + } + if visible_private_read_receipts: + event_content[ReceiptTypes.READ_PRIVATE] = ( + visible_private_read_receipts + ) # Include the event if there is at least one non-private read - # receipt or the current user has a private read receipt. + # receipt or a visible user has a private read receipt. if event_content: content[event_id] = event_content # Include the event if there is at least one non-private read receipt - # or the current user has a private read receipt. + # or a visible user has a private read receipt. if content: # Build a new event to avoid mutating the cache. new_room = {k: v for k, v in room.items() if k != "content"} @@ -345,6 +374,11 @@ class ReceiptEventSource(EventSource[MultiWriterStreamToken, JsonMapping]): events.append(event) + # Private read receipts must only be sent for users matching one of the + # appservice's namespaces (or its sender user). See + # https://spec.matrix.org/v1.19/application-service-api/#pushing-ephemeral-data + events = self._filter_private_receipts(events, service.is_interested_in_user) + return events, to_key def get_current_key(self) -> MultiWriterStreamToken: diff --git a/synapse/handlers/relations.py b/synapse/handlers/relations.py index a8db082feb..dee4746dcb 100644 --- a/synapse/handlers/relations.py +++ b/synapse/handlers/relations.py @@ -44,6 +44,7 @@ from synapse.synapse_rust.events import ( # noqa: F401 BundledAggregations, ThreadAggregation, ) +from synapse.synapse_rust.room_versions import RoomVersion from synapse.types import JsonDict, Requester, UserID from synapse.util.async_helpers import gather_results from synapse.visibility import filter_and_transform_events_for_client @@ -193,6 +194,7 @@ class RelationsHandler: event_id: str, initial_redaction_event: EventBase, relation_types: list[str], + room_version: RoomVersion, ) -> None: """Redacts all events related to the given event ID with one of the given relation types. @@ -210,6 +212,8 @@ class RelationsHandler: event_id. relation_types: The types of relations to look for. If "*" is in the list, all related events will be redacted regardless of the type. + room_version: The RoomVersion of the room. Used for deciding where the + 'redacts' key should go in the event dict. Raises: ShadowBanError if the requester is shadow-banned @@ -226,16 +230,28 @@ class RelationsHandler: ) for related_event_id in related_event_ids: + new_redaction_content = dict(initial_redaction_event.content) + event_dict: JsonDict = { + "type": EventTypes.Redaction, + "content": new_redaction_content, + "room_id": initial_redaction_event.room_id, + "sender": requester.user.to_string(), + } + # Depending on the room version involved, the "redacts" key can go in one of + # two places. + # + # The Matrix Spec page for changes in Room Version 11 asks that we maintain + # a backward and forwards compatibility for clients over that API. That + # compatibility fixup will be in the client event serialization code. Here + # we form and persist the event strictly by the version of the room. + if room_version.updated_redaction_rules: + event_dict["content"].update({"redacts": related_event_id}) + else: + event_dict["redacts"] = related_event_id try: await self._event_creation_handler.create_and_send_nonmember_event( requester, - { - "type": EventTypes.Redaction, - "content": initial_redaction_event.content, - "room_id": initial_redaction_event.room_id, - "sender": requester.user.to_string(), - "redacts": related_event_id, - }, + event_dict, ratelimit=False, ) except SynapseError as e: diff --git a/synapse/handlers/reports.py b/synapse/handlers/reports.py index a7b8a4bed7..6cf688e8cc 100644 --- a/synapse/handlers/reports.py +++ b/synapse/handlers/reports.py @@ -17,7 +17,7 @@ import logging from http import HTTPStatus from typing import TYPE_CHECKING -from synapse.api.errors import Codes, SynapseError +from synapse.api.errors import Codes, NotFoundError, SynapseError from synapse.api.ratelimiting import Ratelimiter from synapse.types import ( Requester, @@ -91,6 +91,38 @@ class ReportsHandler: received_ts=self._clock.time_msec(), ) + async def report_room( + self, requester: Requester, room_id: str, reason: str + ) -> None: + """Files a report against a room from a user. + + A rate limit is applied to the report. + + If the report is otherwise valid (for a room which exists on our + server), we append it to the database for later processing. + + Args: + requester - The user filing the report. + room_id - The room being reported. + reason - The user-supplied reason the room is being reported. + + Raises: + NotFoundError if the room does not exist. + """ + + await self._check_limits(requester) + + room = await self._store.get_room(room_id) + if room is None: + raise NotFoundError("Room does not exist") + + await self._store.add_room_report( + room_id=room_id, + user_id=requester.user.to_string(), + reason=reason, + received_ts=self._clock.time_msec(), + ) + async def _check_limits(self, requester: Requester) -> None: await self._reports_ratelimiter.ratelimit( requester, diff --git a/synapse/handlers/room.py b/synapse/handlers/room.py index c75fff9170..26ebd42574 100644 --- a/synapse/handlers/room.py +++ b/synapse/handlers/room.py @@ -21,6 +21,7 @@ """Contains functions for performing actions on rooms.""" +import copy import itertools import logging import math @@ -165,7 +166,7 @@ class RoomCreationHandler: "history_visibility": HistoryVisibility.SHARED, "original_invitees_have_ops": False, "guest_can_join": False, - "power_level_content_override": {EventTypes.CallInvite: 50}, + "power_level_content_override": {"events": {EventTypes.CallInvite: 50}}, }, } @@ -1409,30 +1410,77 @@ class RoomCreationHandler: is_public: bool, room_version: RoomVersion, ) -> tuple[EventBase, synapse.events.snapshot.EventContext]: - ( - creation_event, - new_unpersisted_context, - ) = await self.event_creation_handler.create_event( - creator, - { + """Create and store the create event for a v12+ room, retrying on room ID collision. + + In v12+ rooms the room ID is derived from the create event, so this + builds the create event, stores the room under the resulting ID, and + retries with a slightly older `origin_server_ts` if that ID is already taken. + + Args: + creator: The user creating the room. + creation_content: The content for the create event. + is_public: Whether the room is published to the room directory. + room_version: The version of the room being created. + + Returns: + A tuple of the create event and its event context. + + Raises: + StoreError: if a unique room ID could not be generated after several + attempts. + """ + # In v12+ rooms, the room ID is the reference hash of the create event, + # so two rooms whose create events have identical content collide on the + # same room ID. This happens in practice when the same user creates + # several rooms at once (e.g. concurrent /createRoom requests with the + # same config), as the only entropy in the create event is + # `origin_server_ts`, which has millisecond resolution. + # + # Retry a few times on collision, perturbing `origin_server_ts` + # so the create event hashes to a fresh room ID. This mirrors the + # collision handling in `_generate_and_create_room_id` used for + # older room versions. + attempts = 0 + while attempts < 5: + event_dict: JsonDict = { "content": creation_content, "sender": creator.user.to_string(), "type": EventTypes.Create, "state_key": "", - }, - prev_event_ids=[], - depth=1, - state_map={}, - for_batch=False, - ) - await self.store.store_room( - room_id=creation_event.room_id, - room_creator_user_id=creator.user.to_string(), - is_public=is_public, - room_version=room_version, - ) - creation_context = await new_unpersisted_context.persist(creation_event) - return (creation_event, creation_context) + } + if attempts > 0: + # Bump the timestamp to give the create event (and hence the + # room ID it hashes to) different content from the colliding one. + event_dict["origin_server_ts"] = ( + self.clock.time_msec() + random.randint(1, 10) + ) + + ( + creation_event, + new_unpersisted_context, + ) = await self.event_creation_handler.create_event( + creator, + event_dict, + prev_event_ids=[], + depth=1, + state_map={}, + for_batch=False, + ) + try: + await self.store.store_room( + room_id=creation_event.room_id, + room_creator_user_id=creator.user.to_string(), + is_public=is_public, + room_version=room_version, + ) + except StoreError: + attempts += 1 + continue + + creation_context = await new_unpersisted_context.persist(creation_event) + return (creation_event, creation_context) + + raise StoreError(500, "Couldn't generate a unique room ID.") async def _send_events_for_new_room( self, @@ -1482,6 +1530,8 @@ class RoomCreationHandler: alias for the room power_level_content_override: The power level content to override in the default power level event. + `power_level_content_override` doesn't apply when `initial_state` has + a power level state event content (i.e. `EventTypes.PowerLevels`). creator_join_profile: Set to override the displayname and avatar for the creating user in this room. @@ -1561,7 +1611,7 @@ class RoomCreationHandler: prev_state_events = [new_event.event_id] return new_event, new_unpersisted_context - preset_config, config = self._room_preset_config(room_config) + preset_name, preset_config = self._room_preset_config(room_config) if creation_event_with_context is None: # MSC2175 removes the creator field from the create event. @@ -1627,6 +1677,7 @@ class RoomCreationHandler: events_to_send = [] # We treat the power levels override specially as this needs to be one # of the first events that get sent into a room. + # If the `initial_state` has `EventTypes.PowerLevels` content, use it. pl_content = initial_state.pop((EventTypes.PowerLevels, ""), None) if pl_content is not None: power_event, power_context = await create_event( @@ -1634,6 +1685,8 @@ class RoomCreationHandler: ) events_to_send.append((power_event, power_context)) else: + # If the `initial_state` does not have `EventTypes.PowerLevels` content, + # use the default power level content. # Please update the docs for `default_power_level_content_override` when # updating the `events` dict below power_level_content: JsonDict = { @@ -1666,25 +1719,27 @@ class RoomCreationHandler: # have set these users as additional_creators, hence don't set the PL for creators as # that is invalid. if ( - config["original_invitees_have_ops"] + preset_config["original_invitees_have_ops"] and not room_version.msc4289_creator_power_enabled ): for invitee in invite_list: power_level_content["users"][invitee] = 100 - # If the user supplied a preset name e.g. "private_chat", - # we apply that preset - power_level_content.update(config["power_level_content_override"]) + # If the user supplied a preset name e.g. "private_chat", apply that + # preset's power level content. + power_level_content = self._deepmerge_power_level_content( + power_level_content, preset_config["power_level_content_override"] + ) # If the server config contains default_power_level_content_override, # and that contains information for this room preset, apply it. if self._default_power_level_content_override: - override = self._default_power_level_content_override.get(preset_config) + override = self._default_power_level_content_override.get(preset_name) if override is not None: power_level_content.update(override) # Finally, if the user supplied specific permissions for this room, - # apply those. + # apply those. Supplied room override wins over server config. if power_level_content_override: power_level_content.update(power_level_content_override) pl_event, pl_context = await create_event( @@ -1703,7 +1758,7 @@ class RoomCreationHandler: if (EventTypes.JoinRules, "") not in initial_state: join_rules_event, join_rules_context = await create_event( EventTypes.JoinRules, - {"join_rule": config["join_rules"]}, + {"join_rule": preset_config["join_rules"]}, True, ) events_to_send.append((join_rules_event, join_rules_context)) @@ -1711,12 +1766,12 @@ class RoomCreationHandler: if (EventTypes.RoomHistoryVisibility, "") not in initial_state: visibility_event, visibility_context = await create_event( EventTypes.RoomHistoryVisibility, - {"history_visibility": config["history_visibility"]}, + {"history_visibility": preset_config["history_visibility"]}, True, ) events_to_send.append((visibility_event, visibility_context)) - if config["guest_can_join"]: + if preset_config["guest_can_join"]: if (EventTypes.GuestAccess, "") not in initial_state: guest_access_event, guest_access_context = await create_event( EventTypes.GuestAccess, @@ -1731,7 +1786,21 @@ class RoomCreationHandler: ) events_to_send.append((event, context)) - if config["encrypted"] and not ignore_forced_encryption: + # If the client supplied its own `m.room.encryption` event in the + # initial state, only let it take precedence over the forced default + # if it is valid (i.e. specifies an `algorithm` as a string, as + # required by the spec). This prevents a client from bypassing forced + # encryption entirely by supplying an empty or malformed event. + supplied_encryption = initial_state.get((EventTypes.RoomEncryption, "")) + supplied_encryption_is_valid = isinstance(supplied_encryption, dict) and ( + isinstance(supplied_encryption.get("algorithm"), str) + ) + + if ( + preset_config["encrypted"] + and not ignore_forced_encryption + and not supplied_encryption_is_valid + ): encryption_event, encryption_context = await create_event( EventTypes.RoomEncryption, {"algorithm": RoomEncryptionAlgorithms.DEFAULT}, @@ -1821,6 +1890,32 @@ class RoomCreationHandler: f"You cannot create an encrypted room. user_level ({room_admin_level}) < send_level ({encryption_level})", ) + def _deepmerge_power_level_content( + self, power_level_content: JsonDict, override: JsonDict + ) -> JsonDict: + """Deep-merge `override` into `power_level_content`. + + Nested dicts (e.g. events, users) are merged recursively. All other values from + `override` replace those in `power_level_content`. + + Args: + power_level_content: The base power level content to update. + override: Values to merge on top of `power_level_content`. + + Returns: + The updated power level content. + """ + for key, value in override.items(): + existing = power_level_content.get(key) + if isinstance(existing, dict) and isinstance(value, dict): + power_level_content[key] = self._deepmerge_power_level_content( + dict(existing), value + ) + else: + # Copy so we don't accidentally modify the preset config. + power_level_content[key] = copy.deepcopy(value) + return power_level_content + def _room_preset_config(self, room_config: JsonDict) -> tuple[str, dict]: # The spec says rooms should default to private visibility if # `visibility` is not specified. diff --git a/synapse/handlers/room_member.py b/synapse/handlers/room_member.py index f61d39963f..2c0164c0e1 100644 --- a/synapse/handlers/room_member.py +++ b/synapse/handlers/room_member.py @@ -1390,9 +1390,15 @@ class RoomMemberHandler(metaclass=abc.ABCMeta): return True, list(servers_that_can_issue_invite) # Ensure the member should be allowed access via membership in a room. - await self.event_auth_handler.check_restricted_join_rules( - state_before_join, room_version, user_id, previous_membership - ) + try: + await self.event_auth_handler.check_restricted_join_rules( + state_before_join, room_version, user_id, previous_membership + ) + except SynapseError as e: + if e.errcode == Codes.UNABLE_AUTHORISE_JOIN: + servers_that_can_issue_invite.discard(self.hs.hostname) + return True, list(servers_that_can_issue_invite) + raise # If this is going to be a local join, additional information must # be included in the event content in order to efficiently validate @@ -1547,7 +1553,6 @@ class RoomMemberHandler(metaclass=abc.ABCMeta): prev_member_event_id = prev_state_ids.get( (EventTypes.Member, event.state_key), None ) - if prev_member_event_id: prev_member_event = await self.store.get_event(prev_member_event_id) if prev_member_event.membership == Membership.JOIN: diff --git a/synapse/handlers/room_summary.py b/synapse/handlers/room_summary.py index 0bc6021fa7..4afd449292 100644 --- a/synapse/handlers/room_summary.py +++ b/synapse/handlers/room_summary.py @@ -685,9 +685,10 @@ class RoomSummaryHandler: if await self._event_auth_handler.has_restricted_join_rules( state_ids, room_version ): - allowed_rooms = ( - await self._event_auth_handler.get_rooms_that_allow_join(state_ids) - ) + ( + allowed_rooms, + _, + ) = await self._event_auth_handler.get_rooms_that_allow_join(state_ids) if await self._event_auth_handler.is_user_in_rooms( allowed_rooms, requester ): @@ -707,9 +708,10 @@ class RoomSummaryHandler: if await self._event_auth_handler.has_restricted_join_rules( state_ids, room_version ): - allowed_rooms = ( - await self._event_auth_handler.get_rooms_that_allow_join(state_ids) - ) + ( + allowed_rooms, + _, + ) = await self._event_auth_handler.get_rooms_that_allow_join(state_ids) for space_id in allowed_rooms: if await self._event_auth_handler.is_host_in_room(space_id, origin): return True @@ -822,7 +824,7 @@ class RoomSummaryHandler: if room_version and await self._event_auth_handler.has_restricted_join_rules( join_rules_state_ids, room_version ): - allowed_rooms = await self._event_auth_handler.get_rooms_that_allow_join( + allowed_rooms, _ = await self._event_auth_handler.get_rooms_that_allow_join( join_rules_state_ids ) if allowed_rooms: diff --git a/synapse/handlers/sliding_sync/__init__.py b/synapse/handlers/sliding_sync/__init__.py index 10ca3ddea0..afb93e42b4 100644 --- a/synapse/handlers/sliding_sync/__init__.py +++ b/synapse/handlers/sliding_sync/__init__.py @@ -57,6 +57,7 @@ from synapse.types import ( StrCollection, StreamKeyType, StreamToken, + StrictJsonMapping, ) from synapse.types.handlers import SLIDING_SYNC_DEFAULT_BUMP_EVENT_TYPES from synapse.types.handlers.sliding_sync import ( @@ -953,7 +954,10 @@ class SlidingSyncHandler: ) name_event = name_states.get((EventTypes.Name, "")) if name_event is not None: - room_name = name_event.content.get("name") + name_event_content: StrictJsonMapping = name_event.content + unchecked_room_name = name_event_content.get("name") + if isinstance(unchecked_room_name, str): + room_name = unchecked_room_name # We only need the room summary for calculating heroes, however if we do # fetch it then we can use it to calculate `joined_count` and @@ -1356,18 +1360,28 @@ class SlidingSyncHandler: room_avatar: str | None = None avatar_event = room_state.get((EventTypes.RoomAvatar, "")) if avatar_event is not None: - room_avatar = avatar_event.content.get("url") + room_avatar_content: StrictJsonMapping = avatar_event.content + unchecked_room_avatar = room_avatar_content.get("url") + if isinstance(unchecked_room_avatar, str): + room_avatar = unchecked_room_avatar # Assemble heroes: extract the info from the state we just fetched heroes: list[SlidingSyncResult.RoomResult.StrippedHero] = [] for hero_user_id in hero_user_ids: member_event = hero_membership_state.get((EventTypes.Member, hero_user_id)) if member_event is not None: + member_event_content: StrictJsonMapping = member_event.content + unchecked_display_name = member_event_content.get("displayname") + unchecked_avatar_url = member_event_content.get("avatar_url") heroes.append( SlidingSyncResult.RoomResult.StrippedHero( user_id=hero_user_id, - display_name=member_event.content.get("displayname"), - avatar_url=member_event.content.get("avatar_url"), + display_name=unchecked_display_name + if isinstance(unchecked_display_name, str) + else None, + avatar_url=unchecked_avatar_url + if isinstance(unchecked_avatar_url, str) + else None, ) ) diff --git a/synapse/handlers/sliding_sync/extensions.py b/synapse/handlers/sliding_sync/extensions.py index b3342de778..7ee26079ed 100644 --- a/synapse/handlers/sliding_sync/extensions.py +++ b/synapse/handlers/sliding_sync/extensions.py @@ -25,20 +25,31 @@ from typing import ( from typing_extensions import TypeAlias, assert_never -from synapse.api.constants import AccountDataTypes, EduTypes, StickyEvent +from synapse.api.constants import ( + AccountDataTypes, + EduTypes, + EventTypes, + ProfileFields, + ProfileUpdateAction, + StickyEvent, +) from synapse.events.utils import FilteredEvent from synapse.handlers.receipts import ReceiptEventSource from synapse.logging.opentracing import trace from synapse.storage.databases.main.receipts import ReceiptInRoom from synapse.types import ( Absent, + AbsentType, DeviceListUpdates, + JsonDict, JsonMapping, + JsonValue, MultiWriterStreamToken, SlidingSyncStreamToken, StrCollection, StreamToken, ThreadSubscriptionsToken, + UserID, ) from synapse.types.handlers.sliding_sync import ( HaveSentRoomFlag, @@ -47,6 +58,7 @@ from synapse.types.handlers.sliding_sync import ( PerConnectionState, SlidingSyncConfig, SlidingSyncResult, + StateValues, ) from synapse.types.rest.client import SlidingSyncStickyEventsToken from synapse.util.async_helpers import ( @@ -80,6 +92,7 @@ class SlidingSyncExtensionHandler: self._storage_controllers = hs.get_storage_controllers() self._enable_thread_subscriptions = hs.config.experimental.msc4306_enabled self._enable_sticky_events = hs.config.experimental.msc4354_enabled + self._enable_profiles = hs.config.server.include_profile_updates_in_sync @trace async def get_extensions_response( @@ -197,6 +210,18 @@ class SlidingSyncExtensionHandler: from_token=from_token, ) + profiles_coro = None + if sync_config.extensions.profiles is not Absent and self._enable_profiles: + profiles_coro = self.get_profiles_extension_response( + sync_config=sync_config, + profiles_request=sync_config.extensions.profiles, + actual_room_ids=actual_room_ids, + to_token=to_token, + from_token=from_token, + actual_room_response_map=actual_room_response_map, + actual_lists=actual_lists, + ) + ( to_device_response, e2ee_response, @@ -205,6 +230,7 @@ class SlidingSyncExtensionHandler: typing_response, thread_subs_response, sticky_events_response, + profiles_response, ) = await gather_optional_coroutines( to_device_coro, e2ee_coro, @@ -213,6 +239,7 @@ class SlidingSyncExtensionHandler: typing_coro, thread_subs_coro, sticky_events_coro, + profiles_coro, ) return SlidingSyncResult.Extensions( @@ -223,6 +250,7 @@ class SlidingSyncExtensionHandler: typing=typing_response, thread_subscriptions=thread_subs_response, sticky_events=sticky_events_response, + profiles=profiles_response, ) def find_relevant_room_ids_for_extension( @@ -1055,3 +1083,379 @@ class SlidingSyncExtensionHandler: sticky_events_stream_id=sticky_events_to_id ), ) + + async def _get_profile_ids_for_profiles_extension( + self, + user_id: str, + actual_room_ids: set[str], + sync_config: SlidingSyncConfig, + actual_room_response_map: Mapping[str, SlidingSyncResult.RoomResult], + actual_lists: Mapping[str, SlidingSyncResult.SlidingWindowList], + ) -> tuple[set[str], set[str]]: + """ + Calculate target user profiles as candiates to include in the profile + extension sync response. + + This function looks at both the sync config and the already calculated + rooms response, and pieces together the full set of user IDs to include + profiles for, based on sync config rooms being lazy loading or not. + + For rooms with lazy loading, only profiles for those users who have sent events + into the timeline will be included, unless they would be included otherwise. + For other rooms, all members of the room will be included as candidates. + + Note, this does not collect user IDs from the profile updates stream. + + Args: + user_id: The full user ID syncing. + actual_room_ids: The actual room IDs in the the Sliding Sync response. + sync_config: The Sliding Sync config object. + actual_room_response_map: A calculated map of responses per room. + actual_lists: Sliding window API. A map of list key to list results in the + Sliding Sync response. + + Returns: + Tuple containing two sets: + - first including all found user IDs, + - second containing user IDs calculated via lazy configured rooms. + """ + lazy_profile_user_ids = set() + non_lazy_profile_user_ids = set() + + # Separate rooms into lazy and non-lazy based on sync config. + # Look at subscriptions first + lazy_rooms = ( + { + room_id + for room_id, room_config in sync_config.room_subscriptions.items() + if (EventTypes.Member, StateValues.LAZY) in room_config.required_state + } + if sync_config.room_subscriptions + else set() + ) + # Iterate lists to find lazy rooms + if sync_config.lists: + for list_name, list_data in sync_config.lists.items(): + if (EventTypes.Member, StateValues.LAZY) in list_data.required_state: + for op in actual_lists[list_name].ops: + lazy_rooms.update(op.room_ids) + + if lazy_rooms: + # For rooms configured as lazy, include users based on room response. + for room_id, room_data in actual_room_response_map.items(): + if room_id not in lazy_rooms: + continue + # Include users from timeline events + for timeline_event in room_data.timeline_events: + lazy_profile_user_ids.add(timeline_event.event.sender) + # Include users from required state + for state_event in room_data.required_state: + if state_event.type == EventTypes.Member: + lazy_profile_user_ids.add(state_event.state_key) + # Include heroes + if room_data.heroes: + for hero in room_data.heroes: + lazy_profile_user_ids.add(hero.user_id) + + non_lazy_rooms = actual_room_ids.difference(lazy_rooms) + # If we still have non-lazy rooms, get their members. + if non_lazy_rooms: + non_lazy_profile_user_ids = ( + # TODO we should consider adding a limit to how many profiles + # of room members we push down the line. However, this produces + # a problem for clients in that they won't know which users + # just don't have any profile information, and which users were limited + # out. If we had an endpoint to fetch a list of profiles at once, + # we could have a hard limit here and clients could fetch the missing + # profiles separately for non-lazy initial sync cases. + await self.store.get_local_users_who_share_room_with_user( + user_id, + limit_to_rooms=non_lazy_rooms, + ) + ) + + # Unify the two lists + profile_user_ids = lazy_profile_user_ids.union(non_lazy_profile_user_ids) + + # Return a tuple containing the full list of user IDs and the lazy subset. + return ( + profile_user_ids, + lazy_profile_user_ids, + ) + + async def _get_profiles_extension_initial_sync_response( + self, + user_id: UserID, + fields: set[str] | None, + profile_user_ids: set[str], + ) -> dict[str, JsonDict]: + """ + Build an initial sync response for the profiles extension. + + Args: + user_id: The syncing user UserID + fields: A set of fields to include in the response. + `None` means all fields. + profile_user_ids: Set of user IDs whose profiles are related to this sync response. + + Returns: + A dictionary (in API response format) mapping users to their + profile updates in an `updated` dictionary. + + { + "@user:example.org": { + "updated": { + "displayname": "Somebody", + "avatar_url": "mxc://example.org/123123123", + "org.example.field": "hiss", + ... + } + }, + ... + } + """ + response: dict[str, JsonDict] = {} + + # This doesn't return entries for the users with no profile data, + # which is good as we don't want to generate anything for users + # with no profile data in initial sync. + profile_data_by_user = await self.store.get_profile_data_for_users( + # Force our own user to be in the set, as we should + # always watch our own profile updates + profile_user_ids | {user_id.to_string()} + ) + + # Serialise the profile updates into the sync response format. + for profile_user_id, profile_data in profile_data_by_user.items(): + per_user_updates: dict[str, JsonValue | dict[str, JsonValue]] + # Include the fields the client asked for, or all, if not specified + if fields is not None: + per_user_updates = { + k: v for k, v in profile_data.items() if k in fields + } + else: + per_user_updates = profile_data + + if per_user_updates: + response[profile_user_id] = { + "updated": per_user_updates, + } + + return response + + async def get_profiles_extension_response( + self, + sync_config: SlidingSyncConfig, + profiles_request: SlidingSyncConfig.Extensions.ProfilesExtension, + actual_room_ids: set[str], + to_token: StreamToken, + from_token: SlidingSyncStreamToken | None, + actual_room_response_map: Mapping[str, SlidingSyncResult.RoomResult], + actual_lists: Mapping[str, SlidingSyncResult.SlidingWindowList], + ) -> SlidingSyncResult.Extensions.ProfilesExtension | None: + """ + Generate a response for the profiles extension. + + Args: + sync_config: The Sliding Sync config. + profiles_request: The profiles extension request. + actual_room_ids: The actual room IDs in the the Sliding Sync response. + to_token: The stream token to generate a response until. + from_token: The stream token to generate a response from. + actual_room_response_map: A calculated map of responses per room. + actual_lists: Sliding window API. A map of list key to list results in the + Sliding Sync response. + + Returns: + - A SlidingSyncResult.Extensions.ProfilesExtension object containing + all the users who have profile updates. + - None if the extension is disabled. + """ + if not profiles_request.enabled: + return None + + user_id = sync_config.user.to_string() + fields = ( + set(profiles_request.fields) + if profiles_request.fields is not Absent + else None + ) + + response: dict[str, JsonDict | None] = {} + + ( + profile_user_ids, + lazy_profile_user_ids, + ) = await self._get_profile_ids_for_profiles_extension( + user_id=user_id, + actual_room_ids=actual_room_ids, + sync_config=sync_config, + actual_room_response_map=actual_room_response_map, + actual_lists=actual_lists, + ) + + if from_token is None: + # Initial sync + return SlidingSyncResult.Extensions.ProfilesExtension( + users=await self._get_profiles_extension_initial_sync_response( + user_id=sync_config.user, + fields=fields, + profile_user_ids=profile_user_ids, + ), + ) + + # Incremental sync + updates = await self.store.get_profile_updates_for_user_and_fields( + from_id=from_token.stream_token.profile_updates_key, + to_id=to_token.profile_updates_key, + user_id=user_id, + field_names=fields, + ) + + # Set of users that just joined their first room that we share with them + joined_room_user_ids: set[str] = set() + # Set of tracked users that have updated their profile + updated_user_ids: set[str] = set() + # Set of tracked users that just left their last room that we share with them + left_room_user_ids: set[str] = set() + + # Process updates in stream order + # We need to be careful of users that have multiple types of updates + # within this sequence of stream rows. + for update in updates: + if update.action == ProfileUpdateAction.JOINED_ROOM: + joined_room_user_ids.add(update.user_id) + # If the user joins a shared room, that overrides + # the fact that they previously left the last shared room + left_room_user_ids.discard(update.user_id) + elif update.action == ProfileUpdateAction.UPDATE: + updated_user_ids.add(update.user_id) + elif update.action == ProfileUpdateAction.LEFT_ROOM: + left_room_user_ids.add(update.user_id) + # If the user leaves their last shared room, that overrides + # the fact that they previously joined a shared room + # and perhaps updated their profile whilst they were in it + joined_room_user_ids.discard(update.user_id) + updated_user_ids.discard(update.user_id) + + # Add the users who joined a shared room or updated their profile to the set of + # users we will serialise profiles for + profile_user_ids.update(joined_room_user_ids) + profile_user_ids.update(updated_user_ids) + + # Process left rooms + for other_user_id in left_room_user_ids: + # Return a null response to the client + # This tells the client that it will no longer receive updates for the user + response[other_user_id] = None + + updated_user_fields: dict[str, set[str]] = {} + # Set fields from updates + for update in updates: + if ( + update.action != ProfileUpdateAction.UPDATE + or not update.affected_fields + or update.user_id in left_room_user_ids + # Skip if not interested in this user + or update.user_id not in profile_user_ids + ): + continue + interesting_changed_fields: set[str] + if fields is not None: + interesting_changed_fields = set(update.affected_fields) & fields + else: + interesting_changed_fields = set(update.affected_fields) + + if not interesting_changed_fields: + # Skip the update as the client is not interested in these fields + continue + + updated_user_fields.setdefault(update.user_id, set()).update( + interesting_changed_fields + ) + + profile_data_by_user = await self.store.get_profile_data_for_users( + profile_user_ids, + ) + + # Serialise the profile updates into the sync response format. + for profile_user_id in profile_user_ids: + if profile_user_id in left_room_user_ids: + continue + profile_data = profile_data_by_user.get(profile_user_id) + if profile_data is None: + # We don't have profile data for this user + # (This is different from having an empty profile) + # Return a null in incremental sync, telling the client to + # remove all profile information for this user. + response[profile_user_id] = None + continue + + # Calculate which fields had updates + updated_fields: set[str] = updated_user_fields.get(profile_user_id, set()) + # Calculate the full available field list + user_fields = set(profile_data.keys()).union(updated_fields) + + # If the user joined the room or is included via lazy loading events, + # include all fields the client wants. This happens because when lazy + # a room, clients will not necessarily have the profile for the user that + # sent an event in the room, and thus we deliver all the fields. The same + # is true if another user joins the room - we need to deliver an initial + # state for clients to work on. + # For non-lazy-loaded users, include only updated fields. We assume clients + # with non-lazy loaded rooms have received the profiles for all the members + # in the room, and thus only need updates. + user_fields = ( + user_fields + if profile_user_id in joined_room_user_ids + or profile_user_id in lazy_profile_user_ids + else updated_fields + ) + # Filter down if the client only wants a subset + if fields: + user_fields = user_fields.intersection(fields) + + if not user_fields: + continue + + per_user_updates: dict[str, JsonValue | dict[str, JsonValue]] = {} + per_user_removals: set[str] = set() + for field_name in user_fields: + # For custom fields the lack of a field means it will be `Absent`, + # for displayname/avatar_url it will be `None`, due to way we store + # things differently. + # FIXME: I intend to simplify this by pushing the special-case logic + # for these 'original' profile fields into the storage layer instead. + absent_type = ( + Absent + if field_name + not in (ProfileFields.DISPLAYNAME, ProfileFields.AVATAR_URL) + else None + ) + field_value: JsonValue | dict[str, JsonValue] | AbsentType = ( + profile_data.get(field_name, absent_type) + ) + if ( + # If the field isn't found on the profile and it is present in + # `updated_fields`, that means an existing field has been removed. + # We need the check against `updated_fields` as some profile fields + # are `None` by default, for example each and every user created + # by Synapse will have `avatar_url: None`, and we don't want to + # constantly send that to the clients. + field_value is absent_type and field_name in updated_fields + ): + per_user_removals.add(field_name) + else: + per_user_updates[field_name] = cast(JsonValue, field_value) + + if per_user_updates or per_user_removals: + entry: dict[str, JsonValue | JsonDict] = {} + response[profile_user_id] = entry + if per_user_updates: + entry["updated"] = per_user_updates + if per_user_removals: + entry["removed"] = list(per_user_removals) + + return SlidingSyncResult.Extensions.ProfilesExtension( + users=response, + ) diff --git a/synapse/handlers/sso.py b/synapse/handlers/sso.py index bb5ca329e0..f9d9475711 100644 --- a/synapse/handlers/sso.py +++ b/synapse/handlers/sso.py @@ -530,10 +530,11 @@ class SsoHandler: user_id, authenticated_entity=user_id, ) - await self._profile_handler.set_displayname( - user_id_obj, - requester, - attributes.display_name, + await self._profile_handler.dispatch_set_profile_field( + target_user=user_id_obj, + requester=requester, + field_name=ProfileFields.DISPLAYNAME, + new_value=attributes.display_name, by_admin=True, ) if attributes.picture: @@ -842,10 +843,11 @@ class SsoHandler: ) # save it as user avatar - await self._profile_handler.set_avatar_url( - uid, - create_requester(uid), - str(avatar_mxc_url), + await self._profile_handler.dispatch_set_profile_field( + target_user=uid, + requester=create_requester(uid), + field_name=ProfileFields.AVATAR_URL, + new_value=str(avatar_mxc_url), ) logger.info("successfully saved the user avatar") diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py index 9ecfe0da0f..049726a97e 100644 --- a/synapse/handlers/sync.py +++ b/synapse/handlers/sync.py @@ -18,8 +18,11 @@ # [This file includes modifications made by New Vector Limited] # # +import hashlib import itertools +import json import logging +import os from typing import ( TYPE_CHECKING, AbstractSet, @@ -37,6 +40,7 @@ from synapse.api.constants import ( EventContentFields, EventTypes, Membership, + ProfileUpdateAction, StickyEvent, ) from synapse.api.filtering import FilterCollection @@ -64,6 +68,7 @@ from synapse.types import ( DeviceListUpdates, JsonDict, JsonMapping, + JsonValue, MultiWriterStreamToken, MutableStateMap, Requester, @@ -104,10 +109,25 @@ non_empty_sync_counter = Counter( # client for no more than 30 minutes. LAZY_LOADED_MEMBERS_CACHE_MAX_AGE = 30 * 60 * 1000 +# Store the cache that tracks which lazy-loaded profile fields have been sent to a given +# client for no more than 30 minutes. +LAZY_LOADED_PROFILE_FIELDS_CACHE_MAX_AGE = 30 * 60 * 1000 + # Remember the last 100 members we sent to a client for the purposes of # avoiding redundantly sending the same lazy-loaded members to the client LAZY_LOADED_MEMBERS_CACHE_MAX_SIZE = 100 +# Remember the last 100 profile field updates we sent to a client for the purposes of +# avoiding redundantly sending the same lazy-loaded full profiles to the client +LAZY_LOADED_PROFILE_FIELDS_CACHE_MAX_SIZE = 100 + +# The digest size for the lazy loaded profile fields cache. +LAZY_LOADED_PROFILE_FIELDS_CACHE_DIGEST_SIZE = 16 + +# A random key generated on server startup, for the lazy loaded profile fields cache. +# Since this is a per-process cache, we don't care if the key is different per process. +LAZY_LOADED_PROFILE_FIELDS_CACHE_DIGEST_KEY = os.urandom(32) + SyncRequestKey = tuple[Any, ...] @@ -224,6 +244,7 @@ class SyncResult: next_batch: Token for the next sync presence: List of presence events for the user. account_data: List of account_data events for the user. + profile_updates: Map of user_id to profile field updates for that user. joined: JoinedSyncResult for each joined room. invited: InvitedSyncResult for each invited room. knocked: KnockedSyncResult for each knocked on room. @@ -239,6 +260,8 @@ class SyncResult: next_batch: StreamToken presence: list[UserPresenceState] account_data: list[JsonDict] + # user ID -> {profile field -> value | null if unset } + profile_updates: dict[str, dict[str, JsonValue | dict[str, JsonValue]] | None] joined: list[JoinedSyncResult] invited: list[InvitedSyncResult] knocked: list[KnockedSyncResult] @@ -260,6 +283,7 @@ class SyncResult: or self.knocked or self.archived or self.account_data + or self.profile_updates or self.to_device or self.device_lists ) @@ -275,6 +299,7 @@ class SyncResult: next_batch=next_batch, presence=[], account_data=[], + profile_updates={}, joined=[], invited=[], knocked=[], @@ -291,6 +316,7 @@ class SyncHandler: self.server_name = hs.hostname self.hs_config = hs.config self.store = hs.get_datastores().main + self._is_mine_id = hs.is_mine_id self.notifier = hs.get_notifier() self.presence_handler = hs.get_presence_handler() self._relations_handler = hs.get_relations_handler() @@ -329,6 +355,29 @@ class SyncHandler: max_len=0, expiry_ms=LAZY_LOADED_MEMBERS_CACHE_MAX_AGE, ) + # ExpiringCache((User, Device)) + # -> LruCache( + # blake2b(Other User ID + Field Name) -> blake2b(Field value) + # ) + self.lazy_loaded_profile_fields_cache: ExpiringCache[ + tuple[str, str | None], LruCache[bytes, bytes] + ] = ExpiringCache( + cache_name="lazy_loaded_profile_fields_cache", + server_name=self.server_name, + hs=hs, + clock=self.clock, + max_len=0, + expiry_ms=LAZY_LOADED_PROFILE_FIELDS_CACHE_MAX_AGE, + ) + """This cache contains fields and values we have sent to clients as profile + updates, for a particular user + device combo. The cache entry is a blake2b hash + of the user + field name, with the value being a blake2b hash of the field value. + If the field value changes for a particular user, the hash will change + and the cache will be missed. + + We don't manually remove entries from this cache, though it may be ignored + in cases where the sync must send the field down to the client. + """ self.rooms_to_exclude_globally = hs.config.server.rooms_to_exclude_from_sync @@ -1023,6 +1072,8 @@ class SyncHandler: def get_lazy_loaded_members_cache( self, cache_key: tuple[str, str | None] ) -> LruCache[str, str]: + # FIXME: This cache may be subject to losing members in the case that + # a sync is interrupted and retried, see https://github.com/element-hq/synapse/issues/19978 cache: LruCache[str, str] | None = self.lazy_loaded_members_cache.get(cache_key) if cache is None: logger.debug("creating LruCache for %r", cache_key) @@ -1036,6 +1087,35 @@ class SyncHandler: logger.debug("found LruCache for %r", cache_key) return cache + def get_lazy_loaded_profile_fields_cache( + self, cache_key: tuple[str, str | None] + ) -> LruCache[bytes, bytes]: + """This cache contains fields and values we have sent to clients as profile + updates, for a particular user + device combo. The cache entry is a blake2b hash + of the user + field name, with the value being a blake2b hash of the field value. + If the field value changes for a particular user, the hash will change + and the cache will be missed. + + We don't manually remove entries from this cache, though it may be ignored + in cases where the sync must send the field down to the client. + """ + # FIXME: This cache may be subject to losing field updates in the case that + # a sync is interrupted and retried, see https://github.com/element-hq/synapse/issues/19978 + cache: LruCache[bytes, bytes] | None = ( + self.lazy_loaded_profile_fields_cache.get(cache_key) + ) + if cache is None: + logger.debug("creating LruCache for %r", cache_key) + cache = LruCache( + max_size=LAZY_LOADED_PROFILE_FIELDS_CACHE_MAX_SIZE, + clock=self.clock, + server_name=self.server_name, + ) + self.lazy_loaded_profile_fields_cache[cache_key] = cache + else: + logger.debug("found LruCache for %r", cache_key) + return cache + async def compute_state_delta( self, room_id: str, @@ -1169,6 +1249,7 @@ class SyncHandler: end_token, members_to_fetch, timeline_state, + joined, ) # If we only have partial state for the room, `state_ids` may be missing the @@ -1391,6 +1472,7 @@ class SyncHandler: end_token: StreamToken, members_to_fetch: set[str] | None, timeline_state: StateMap[str], + joined: bool, ) -> StateMap[str]: """Calculate the state events to be included in an incremental sync response. @@ -1415,6 +1497,7 @@ class SyncHandler: events in the timeline. Otherwise, `None`. timeline_state: The contribution to the room state from state events in `batch`. Only contains the last event for any given state key. + joined: whether the user is currently joined to the room Returns: A map from (type, state_key) to event_id, for each event that we believe @@ -1440,13 +1523,28 @@ class SyncHandler: # events to understand the events in this timeline. So we always # fish out all the member events corresponding to the timeline # here. The caller will then dedupe any redundant ones. - member_ids = await self._state_storage_controller.get_current_state_ids( - room_id=room_id, - state_filter=StateFilter.from_types( - (EventTypes.Member, member) for member in members_to_fetch - ), - await_full_state=await_full_state, + member_filter = StateFilter.from_types( + (EventTypes.Member, member) for member in members_to_fetch ) + if joined: + member_ids = ( + await self._state_storage_controller.get_current_state_ids( + room_id=room_id, + state_filter=member_filter, + await_full_state=await_full_state, + ) + ) + else: + # The user is no longer in the room, so `end_token` points + # at the user's leave/etc event, and the current state may + # include state from after that point. Use state groups to + # get the memberships as of `end_token` instead. + member_ids = await self._state_storage_controller.get_state_ids_at( + room_id, + stream_position=end_token, + state_filter=member_filter, + await_full_state=await_full_state, + ) delta_state_ids.update(member_ids) # We don't do LL filtering for incremental syncs - see @@ -1759,9 +1857,19 @@ class SyncHandler: await self._generate_sync_entry_for_account_data(sync_result_builder) # Presence data is included if the server has it enabled and not filtered out. - include_presence_data = bool( - self.hs_config.server.presence_enabled - and not sync_config.filter_collection.blocks_all_presence() + presence_enabled = bool(self.hs_config.server.presence_enabled) + if not presence_enabled and since_token is not None: + # Even with presence disabled we send down any presence updates the + # client hasn't yet seen, so that the "mark everyone as offline" + # updates written when presence was disabled reach clients that + # would otherwise show the old presence states forever. The stream + # doesn't advance while presence is disabled, so once clients have + # caught up this check stops any further presence work. + presence_enabled = ( + since_token.presence_key < sync_result_builder.now_token.presence_key + ) + include_presence_data = ( + presence_enabled and not sync_config.filter_collection.blocks_all_presence() ) # Device list updates are sent if a since token is provided. include_device_list_updates = bool(since_token and since_token.device_list_key) @@ -1853,10 +1961,18 @@ class SyncHandler: } ) + # Note, this needs to be after we collect `joined`, `invited`, `knocked` and + # `archived` sync results since we want to utilize the work we did to collect + # events in those responses as a basis for which users to include profiles + # for when lazy loading. + if self.hs_config.server.include_profile_updates_in_sync: + await self._generate_sync_entry_for_profile_updates(sync_result_builder) + logger.debug("Sync response calculation complete") return SyncResult( presence=sync_result_builder.presence, account_data=sync_result_builder.account_data, + profile_updates=sync_result_builder.profile_updates, joined=sync_result_builder.joined, invited=sync_result_builder.invited, knocked=sync_result_builder.knocked, @@ -2121,6 +2237,301 @@ class SyncHandler: sync_result_builder.account_data = account_data_for_user + async def _generate_initial_sync_entry_for_profile_updates( + self, + *, + user_id: str, + sync_result_builder: "SyncResultBuilder", + profile_fields: set[str], + include_users: set[str] | None, + ) -> None: + """ + Build an initial sync entry for profile updates and attach it to the + given `sync_result_builder`. + + Note: Currently, only profile updates of local users are generated. + + Args: + user_id: The Matrix ID of the user to generate the sync entry for. + sync_result_builder: + profile_fields: The list of field IDs to filter for. + include_users: List of users profiles to include in the sync response, + for when we have calculated a list of users in our lazy loading + sync and want to only return those. + """ + # Currently, limited to only local profiles, so filter remote servers out + user_ids = await self.store.get_local_users_who_share_room_with_user(user_id) + # Ensure we're in the list even if we don't belong to any rooms + user_ids.add(user_id) + if include_users: + # Filter down to selected included users + user_ids = {user_id for user_id in user_ids if user_id in include_users} + + if not user_ids: + return + + profile_data_by_user = await self.store.get_profile_data_for_users(user_ids) + + # Serialise the profile updates into the sync response format. + profile_updates: dict[ + str, dict[str, JsonValue | dict[str, JsonValue]] | None + ] = {} + for other_user_id in user_ids: + profile_data = profile_data_by_user.get(other_user_id) + if profile_data is None: + # Don't generate anything for users with no profile data + # in initial sync. + continue + + per_user_updates: dict[str, JsonValue | dict[str, JsonValue]] = {} + for field_name in profile_fields: + if field_name in profile_data.keys(): + per_user_updates[field_name] = profile_data[field_name] + + if per_user_updates: + profile_updates[other_user_id] = per_user_updates + + if profile_updates: + sync_result_builder.profile_updates = profile_updates + + async def _generate_sync_entry_for_profile_updates( + self, sync_result_builder: "SyncResultBuilder" + ) -> None: + """ + Build a sync entry for profile updates and attach it to the given + `sync_result_builder`. + + Currently only local profiles updates will be included in the sync response. + + Args: + sync_result_builder: + """ + sync_config = sync_result_builder.sync_config + profile_fields = sync_config.filter_collection.profile_fields + if not profile_fields: + return + + user_id = sync_config.user.to_string() + since_token = sync_result_builder.since_token + now_token = sync_result_builder.now_token + + sync_config = sync_result_builder.sync_config + lazy_load_members = sync_config.filter_collection.lazy_load_members() + include_users = None + if lazy_load_members: + # Collect members from the existing `sync_result_builder` data. + # Ensure we filter out any remove users until we support profile + # updates for federated users. + include_users = set() + # invited + for invited in sync_result_builder.invited: + if self._is_mine_id(invited.invite.sender): + include_users.add(invited.invite.sender) + # joined + for joined in sync_result_builder.joined: + for timeline_event in joined.timeline.events: + if self._is_mine_id(timeline_event.event.sender): + include_users.add(timeline_event.event.sender) + # knocked + for knocked in sync_result_builder.knocked: + if self._is_mine_id(knocked.knock.sender): + include_users.add(knocked.knock.sender) + # archived + for archived in sync_result_builder.archived: + for timeline_event in archived.timeline.events: + if self._is_mine_id(timeline_event.event.sender): + include_users.add(timeline_event.event.sender) + + if since_token is None: + await self._generate_initial_sync_entry_for_profile_updates( + user_id=user_id, + sync_result_builder=sync_result_builder, + profile_fields=profile_fields, + include_users=include_users, + ) + return + + updates = await self.store.get_profile_updates_for_user_and_fields( + from_id=since_token.profile_updates_key, + to_id=now_token.profile_updates_key, + user_id=user_id, + field_names=profile_fields, + ) + + left_room_user_ids = { + update.user_id + for update in updates + if update.action == ProfileUpdateAction.LEFT_ROOM.value + } + joined_room_user_ids = { + update.user_id + for update in updates + if update.action == ProfileUpdateAction.JOINED_ROOM.value + } + users = set() + updated_users = { + update.user_id + for update in updates + if update.action == ProfileUpdateAction.UPDATE.value + } + # Add any users in the timeline, if we collected them due to lazy loading + if include_users: + users.update(include_users) + # Add users with updates + users.update(updated_users) + # Add any newly joined users + users.update(joined_room_user_ids) + + if not users and not left_room_user_ids: + return + + # Serialise the profile updates into the sync response format. + # user ID -> {profile field -> value | null if unset } + profile_updates: dict[ + str, dict[str, JsonValue | dict[str, JsonValue]] | None + ] = {} + + # Process field updates and users who have events in the sync response + if users: + updated_user_fields: dict[str, set[str]] = {} + # Set fields from updates + for update in updates: + if ( + # Skip the update if there is no field update (a joined or left room action), + update.action != ProfileUpdateAction.UPDATE + or update.affected_fields is None + # or if the client isn't interested in any of the fields + or update.affected_fields.isdisjoint(profile_fields) + # or we're not interested in this user. + or update.user_id not in users + ): + continue + updated_user_fields.setdefault(update.user_id, set()).update( + # Add any fields that were affected and that we're interested in + update.affected_fields & profile_fields + ) + + # Note: there's a small race condition here where a profile update may + # occur between fetching `now_token` above and reaching this step. In + # that case, the profile information will be newer than `now_token`. + # This is fine, as users will generally always want the latest profile + # information. However, it does mean that on the next sync, the same + # profile update will come down a second time. + # + # Hopefully clients can just filter these out. + profile_data_by_user = await self.store.get_profile_data_for_users(users) + + # Note, we've already collected field updates above via `updates`, + # outside of events in the timeline when lazy loading. When lazy loading, + # we're already always sending the fields that have changed, regardless + # of the lazy loading cache. + for other_user_id in users: + profile_data = profile_data_by_user.get(other_user_id) + if profile_data is None: + # No profile data for this user, just return a blank dictionary + # in incremental sync, telling the clients to remove all profile + # information for this user. + profile_updates[other_user_id] = None + continue + + per_user_updates: dict[str, JsonValue | dict[str, JsonValue]] = {} + if include_users and other_user_id in include_users: + # Include all the fields the client asked for, as this user + # has events in a lazy loaded sync response, except for + # fields we've recently sent in a previous lazy loaded sync response. + # We must include _updated_ fields even if the profile doesn't have + # this field. The value will be sent down as `None`. We must do + # this as currently legacy sync delivers field removals by + # delivering a null value to clients, and if a field is completely + # deleted, we can't otherwise do that. The fact this field has + # a `ProfileUpdateAction.UPDATE` is enough to tell us it should + # be sent down. + # TODO once removals are sent down in a dedicated key instead of + # null values, the `.union(updated_user_fields.get(other_user_id, []))` + # part here can be removed. + fields = ( + set(profile_data.keys()) + .union(updated_user_fields.get(other_user_id, [])) + .intersection(profile_fields) + ) + for field_name in fields: + cache_key = ( + sync_config.user.to_string(), + sync_config.device_id, + ) + cache = self.get_lazy_loaded_profile_fields_cache(cache_key) + # Only send this users field if we haven't recently sent it. + # Our cache contains previously set values as pairs of + # blake2b(other_used_id + field_name) -> blake2b(value), + # which ensures if the value changes, we'll miss the cache, + # thus sending the field update to the syncing user. + cache_value = hashlib.blake2b( + f"{other_user_id}-{field_name}".encode("utf8"), + key=LAZY_LOADED_PROFILE_FIELDS_CACHE_DIGEST_KEY, + digest_size=LAZY_LOADED_PROFILE_FIELDS_CACHE_DIGEST_SIZE, + ).digest() + value_hash = hashlib.blake2b( + json.dumps( + [ + profile_data.get(field_name), + ], + sort_keys=True, + separators=(",", ":"), + ensure_ascii=False, + ).encode("utf8"), + key=LAZY_LOADED_PROFILE_FIELDS_CACHE_DIGEST_KEY, + digest_size=LAZY_LOADED_PROFILE_FIELDS_CACHE_DIGEST_SIZE, + ).digest() + if cache.get(cache_value) != value_hash: + per_user_updates[field_name] = profile_data.get(field_name) + # Update our cache to indicate this user/field combo + # has been recently sent. + cache.set( + cache_value, + value_hash, + ) + else: + # Include only the diff, unless the user recently joined, + # then send all the fields the client asked for. + # We don't use a cache here as for non-lazy sync we always + # send changes and/or fields the client asked for, if relevant + # as above joined condition. + fields = ( + profile_fields + if other_user_id in joined_room_user_ids + else set(updated_user_fields.get(other_user_id, [])) + ) + # We must include _updated_ fields even if the profile doesn't have + # this field. The value will be sent down as `None`. We must do + # this as currently legacy sync delivers field removals by + # delivering a null value to clients, and if a field is completely + # deleted, we can't otherwise do that. The fact this field has + # a `ProfileUpdateAction.UPDATE` is enough to tell us it should + # be sent down. + # TODO once removals are sent down in a dedicated key instead of + # null values, the `.union(updated_user_fields.get(other_user_id, []))` + # part here can be removed. + fields = ( + set(profile_data.keys()) + .union(updated_user_fields.get(other_user_id, [])) + .intersection(fields) + ) + # fields.update(set(updated_user_fields.get(other_user_id, []))) + for field_name in fields: + per_user_updates[field_name] = profile_data.get(field_name) + + if per_user_updates: + profile_updates[other_user_id] = per_user_updates + + # Process left rooms + if left_room_user_ids: + for other_user_id in left_room_user_ids: + # Return an empty dictionary to the client + profile_updates[other_user_id] = None + + if profile_updates: + sync_result_builder.profile_updates = profile_updates + async def _generate_sync_entry_for_presence( self, sync_result_builder: "SyncResultBuilder", @@ -3137,6 +3548,7 @@ class SyncResultBuilder: # The following mirror the fields in a sync response presence account_data + profile_updates joined invited knocked @@ -3155,6 +3567,9 @@ class SyncResultBuilder: presence: list[UserPresenceState] = attr.Factory(list) account_data: list[JsonDict] = attr.Factory(list) + profile_updates: dict[str, dict[str, JsonValue | dict[str, JsonValue]] | None] = ( + attr.Factory(dict) + ) joined: list[JoinedSyncResult] = attr.Factory(list) invited: list[InvitedSyncResult] = attr.Factory(list) knocked: list[KnockedSyncResult] = attr.Factory(list) diff --git a/synapse/http/appservice_proxy.py b/synapse/http/appservice_proxy.py new file mode 100644 index 0000000000..714dfcc0c1 --- /dev/null +++ b/synapse/http/appservice_proxy.py @@ -0,0 +1,194 @@ +# +# This file is licensed under the Affero General Public License (AGPL) version 3. +# +# Copyright (C) 2026 Element Creations Ltd +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# See the GNU Affero General Public License for more details: +# . +# + +import logging +from http import HTTPStatus +from typing import TYPE_CHECKING, Optional, cast +from urllib.parse import parse_qs, unquote_to_bytes, urlencode, urlsplit + +from twisted.python import failure +from twisted.web.http_headers import Headers +from twisted.web.iweb import IBodyProducer, IResponse + +from synapse.api.errors import Codes, SynapseError +from synapse.appservice import ApplicationService +from synapse.http.proxy import ( + HOP_BY_HOP_HEADERS_LOWERCASE, + _ProxyResponseBody, + parse_connection_header_value, +) +from synapse.http.server import return_json_error, set_cors_headers +from synapse.http.site import SynapseRequest +from synapse.logging.context import make_deferred_yieldable, run_in_background +from synapse.util.async_helpers import timeout_deferred + +if TYPE_CHECKING: + from synapse.server import HomeServer + +logger = logging.getLogger(__name__) + + +def has_dot_segments(path: bytes) -> bool: + """Whether the given request path contains any "." or ".." segments. + + The path is percent-decoded before it is split, since `%2e%2e` and `..` are + equivalent to anything that resolves the path, and `%2f` hides a separator that + would otherwise not be seen. Note that a single decode is deliberate: it matches + the single decode that route arguments get in `JsonResource._async_render`, so + `%252e%252e` is left alone rather than being treated as a dot segment. + + The caller is expected to reject such a path rather than rewrite it. Synapse + routes on the raw path, and federation request signatures cover the raw URI, so + normalising a path in place would break both. + """ + return any( + segment in (b".", b"..") for segment in unquote_to_bytes(path).split(b"/") + ) + + +def strip_access_token_from_uri(uri: bytes) -> bytes: + """Remove any `access_token` query parameter from the given request URI. + + Clients are not supposed to authenticate with an `access_token` query + parameter, but some might do so anyway. Since the URI is forwarded to the + application service verbatim, strip it out just in case, so that it isn't + inadvertently leaked to the application service. + """ + split_uri = urlsplit(uri) + if not split_uri.query: + return uri + + args = parse_qs(split_uri.query, keep_blank_values=True) + for key in list(args.keys()): + if key.lower() == b"access_token": + del args[key] + + if not args: + return split_uri.path + + return split_uri.path + b"?" + urlencode(args, doseq=True).encode("ascii") + + +async def proxy_request_to_appservice( + request: SynapseRequest, + hs: "HomeServer", + appservice: ApplicationService, + body_producer: Optional[IBodyProducer], + extra_request_headers: dict[bytes, bytes] | None = None, +) -> None: + """Forward the given request to an application service's proxy URL and stream + the response back to the original caller unchanged. + + Args: + request: The inbound request to forward. + hs: The homeserver. + appservice: The application service to forward the request to. Must have + `proxy_url` and `hs_token` set. + body_producer: A producer for the request body to forward, or None if the + request has no body to forward. + extra_request_headers: Additional headers to set on the outbound request, + beyond those copied from the original request. + """ + assert appservice.proxy_url is not None + assert appservice.hs_token is not None + + request_path = request.uri.split(b"?", 1)[0] + if has_dot_segments(request_path): + return_json_error( + failure.Failure( + SynapseError( + HTTPStatus.BAD_REQUEST, + "Request path is not allowed", + Codes.INVALID_PARAM, + ) + ), + request, + None, + ) + return + + target_uri = appservice.proxy_url.encode("ascii") + strip_access_token_from_uri( + request.uri + ) + + # Only forward the bare minimum of request headers an application service could + # plausibly need. + headers = Headers() + for header_name, header_values in request.requestHeaders.getAllRawHeaders(): + if header_name.decode("ascii").lower() in { + "content-type", + "accept", + "accept-language", + }: + headers.setRawHeaders(header_name, header_values) + + headers.setRawHeaders( + b"Authorization", [b"Bearer " + appservice.hs_token.encode("ascii")] + ) + + if extra_request_headers: + for header_name, header_value in extra_request_headers.items(): + headers.setRawHeaders(header_name, [header_value]) + + agent = hs.get_proxied_http_client().agent + request_deferred = run_in_background( + agent.request, + request.method, + target_uri, + headers=headers, + bodyProducer=body_producer, + ) + request_deferred = timeout_deferred( + deferred=request_deferred, + timeout=30, # Give the application service at most 30s to respond. + clock=hs.get_clock(), + ) + + try: + response = await make_deferred_yieldable(request_deferred) + except Exception: + logger.warning( + "Error proxying request to application service %s at %s", + appservice.id, + target_uri, + exc_info=True, + ) + return_json_error(failure.Failure(), request, None) + return + + _send_response(request, response) + + +def _send_response(request: SynapseRequest, response: IResponse) -> None: + response_headers = cast(Headers, response.headers) + + request.setResponseCode(response.code) + set_cors_headers(request) + + # We strip the "hop-by-hop" headers as defined by RFC2616. + headers_to_strip = set(HOP_BY_HOP_HEADERS_LOWERCASE) + + # The `Connection` header can define additional headers that should not be + # copied over. + connection_header = response_headers.getRawHeaders(b"connection") + headers_to_strip |= parse_connection_header_value( + connection_header[0] if connection_header else None + ) + + for header_name, header_values in response_headers.getAllRawHeaders(): + if header_name.decode("ascii").lower() in headers_to_strip: + continue + request.responseHeaders.setRawHeaders(header_name, header_values) + + response.deliverBody(_ProxyResponseBody(request)) diff --git a/synapse/http/proxy.py b/synapse/http/proxy.py index b3a2f84f29..ad5c6d5d91 100644 --- a/synapse/http/proxy.py +++ b/synapse/http/proxy.py @@ -51,16 +51,18 @@ logger = logging.getLogger(__name__) # "Hop-by-hop" headers (as opposed to "end-to-end" headers) as defined by RFC2616 # section 13.5.1 and referenced in RFC9110 section 7.6.1. These are meant to only be # consumed by the immediate recipient and not be forwarded on. -HOP_BY_HOP_HEADERS_LOWERCASE = { - "connection", - "keep-alive", - "proxy-authenticate", - "proxy-authorization", - "te", - "trailers", - "transfer-encoding", - "upgrade", -} +HOP_BY_HOP_HEADERS_LOWERCASE = frozenset( + { + "connection", + "keep-alive", + "proxy-authenticate", + "proxy-authorization", + "te", + "trailers", + "transfer-encoding", + "upgrade", + } +) assert all(header.lower() == header for header in HOP_BY_HOP_HEADERS_LOWERCASE) diff --git a/synapse/http/server.py b/synapse/http/server.py index 2c235e04f4..a0ae20be16 100644 --- a/synapse/http/server.py +++ b/synapse/http/server.py @@ -33,6 +33,7 @@ from typing import ( Any, Awaitable, Callable, + Final, Iterable, Iterator, Pattern, @@ -673,8 +674,33 @@ class UnrecognizedRequestResource(resource.Resource): # or the response bytes as a return value. return NOT_DONE_YET - def getChild(self, name: str, request: Request) -> resource.Resource: - return self + def getChild(self, path: str, request: Request) -> resource.Resource: + # The child of a catch-all unrecognised request handler + # is itself another unrecognised request handler. + # We can return any UnrecognizedRequestResource that doesn't + # have children. + assert len(_BLANK_LEAF_UNRECOGNISED_REQUEST_RESOURCE.children) == 0 + return _BLANK_LEAF_UNRECOGNISED_REQUEST_RESOURCE + + +class _LeafUnrecognisedRequestResource(UnrecognizedRequestResource): + """ + UnrecognizedRequestResource, but with the added caveat that it can't have any children. + This makes it safe for it to return itself as a dynamic child. + + Constructed as a singleton; use `_BLANK_LEAF_UNRECOGNISED_REQUEST_RESOURCE` + """ + + def putChild(self, path: bytes, child: IResource) -> None: + raise RuntimeError("_LeafUnrecognisedRequestResource does not accept children") + + +_BLANK_LEAF_UNRECOGNISED_REQUEST_RESOURCE: Final[_LeafUnrecognisedRequestResource] = ( + _LeafUnrecognisedRequestResource() +) +""" +An UnrecognizedRequestResource that is guaranteed not to have children. +""" class RootRedirect(resource.Resource): diff --git a/synapse/http/site.py b/synapse/http/site.py index 9b7fd5c936..4b841a21e2 100644 --- a/synapse/http/site.py +++ b/synapse/http/site.py @@ -278,7 +278,7 @@ class SynapseRequest(Request): # See: https://github.com/element-hq/synapse/security/advisories/GHSA-rfq8-j7rh-8hf2 if command == b"POST": ctype = self.requestHeaders.getRawHeaders(b"content-type") - if ctype and b"multipart/form-data" in ctype[0]: + if ctype and b"multipart/form-data" in ctype[0].lower(): logger.warning( "Aborting connection from %s because `content-type: multipart/form-data` is unsupported: %s %s", self.client, diff --git a/synapse/media/media_repository.py b/synapse/media/media_repository.py index 3344e4c7be..31bb401e65 100644 --- a/synapse/media/media_repository.py +++ b/synapse/media/media_repository.py @@ -40,6 +40,7 @@ from synapse.api.errors import ( NotFoundError, RequestSendFailed, SynapseError, + UserLimitExceededError, cs_error, ) from synapse.api.ratelimiting import Ratelimiter @@ -68,7 +69,11 @@ from synapse.media.storage_provider import ( FileStorageProviderBackend, StorageProviderWrapper, ) -from synapse.media.thumbnailer import Thumbnailer, ThumbnailError +from synapse.media.thumbnailer import ( + ANIMATED_THUMBNAIL_TYPE, + Thumbnailer, + ThumbnailError, +) from synapse.media.url_previewer import UrlPreviewer from synapse.storage.databases.main.media_repository import LocalMedia, RemoteMedia from synapse.types import UserID @@ -396,8 +401,20 @@ class MediaRepository: sent_bytes=uploaded_media_size, attempted_bytes=content_length, ) - raise SynapseError( - 400, "Media upload limit exceeded", Codes.RESOURCE_LIMIT_EXCEEDED + + # Fall back to the static page served by Synapse when the limit + # doesn't specify its own `info_uri` (e.g. limits returned by a + # module callback without one). + info_uri = ( + limit.info_uri + or self.hs.config.media.media_upload_limit_fallback_info_uri + ) + + raise UserLimitExceededError( + 403, + "Media upload limit exceeded", + info_uri=info_uri, + can_upgrade=limit.can_upgrade, ) if is_new_media: @@ -1127,6 +1144,7 @@ class MediaRepository: t_height: int, t_method: str, t_type: str, + animated: bool = False, ) -> BytesIO | None: m_width = thumbnailer.width m_height = thumbnailer.height @@ -1144,12 +1162,12 @@ class MediaRepository: m_width, m_height = thumbnailer.transpose() if t_method == "crop": - return thumbnailer.crop(t_width, t_height, t_type) + return thumbnailer.crop(t_width, t_height, t_type, animated=animated) elif t_method == "scale": t_width, t_height = thumbnailer.aspect(t_width, t_height) t_width = min(m_width, t_width) t_height = min(m_height, t_height) - return thumbnailer.scale(t_width, t_height, t_type) + return thumbnailer.scale(t_width, t_height, t_type, animated=animated) return None @@ -1161,6 +1179,7 @@ class MediaRepository: t_method: str, t_type: str, url_cache: bool, + animated: bool = False, ) -> tuple[str, FileInfo] | None: async with self.media_storage.ensure_media_is_in_local_cache( FileInfo(None, media_id, url_cache=url_cache) @@ -1186,6 +1205,7 @@ class MediaRepository: t_height, t_method, t_type, + animated, ) if t_byte_source: @@ -1199,7 +1219,7 @@ class MediaRepository: height=t_height, method=t_method, type=t_type, - length=t_byte_source.tell(), + length=t_byte_source.getbuffer().nbytes, ), ) @@ -1236,6 +1256,7 @@ class MediaRepository: t_height: int, t_method: str, t_type: str, + animated: bool = False, ) -> str | None: async with self.media_storage.ensure_media_is_in_local_cache( FileInfo(server_name, file_id) @@ -1262,6 +1283,7 @@ class MediaRepository: t_height, t_method, t_type, + animated, ) if t_byte_source: @@ -1274,7 +1296,7 @@ class MediaRepository: height=t_height, method=t_method, type=t_type, - length=t_byte_source.tell(), + length=t_byte_source.getbuffer().nbytes, ), ) @@ -1365,17 +1387,23 @@ class MediaRepository: self.hs.get_reactor(), thumbnailer.transpose ) + # JPEG has no alpha channel, so it would flatten a transparent + # image onto a solid color background. + needs_alpha = await defer_to_thread( + self.hs.get_reactor(), lambda: thumbnailer.has_transparency + ) + # We deduplicate the thumbnail sizes by ignoring the cropped versions if # they have the same dimensions of a scaled one. thumbnails: dict[tuple[int, int, str], str] = {} for requirement in requirements: + t_type = requirement.media_type + if needs_alpha and t_type == "image/jpeg": + t_type = "image/png" + if requirement.method == "crop": thumbnails.setdefault( - ( - requirement.width, - requirement.height, - requirement.media_type, - ), + (requirement.width, requirement.height, t_type), requirement.method, ) elif requirement.method == "scale": @@ -1384,11 +1412,15 @@ class MediaRepository: ) t_width = min(m_width, t_width) t_height = min(m_height, t_height) - thumbnails[(t_width, t_height, requirement.media_type)] = ( - requirement.method - ) + thumbnails[(t_width, t_height, t_type)] = requirement.method # Now we generate the thumbnails for each dimension, store it + # + # For animated source images we also generate and cache an + # animated WebP thumbnail per (size, method), served only when a + # client requests `?animated=true`. These are deduplicated since + # the animated thumbnail is always WebP regardless of `t_type`. + animated_done: set[tuple[int, int, str]] = set() for (t_width, t_height, t_type), t_method in thumbnails.items(): # Generate the thumbnail if t_method == "crop": @@ -1411,81 +1443,127 @@ class MediaRepository: logger.error("Unrecognized method: %r", t_method) continue - if not t_byte_source: - continue + if t_byte_source: + await self._store_thumbnail( + server_name, + media_id, + file_id, + url_cache, + t_width, + t_height, + t_method, + t_type, + t_byte_source, + ) - file_info = FileInfo( - server_name=server_name, - file_id=file_id, - url_cache=url_cache, - thumbnail=ThumbnailInfo( - width=t_width, - height=t_height, - method=t_method, - type=t_type, - length=t_byte_source.tell(), - ), - ) - - async with self.media_storage.store_into_file(file_info) as ( - f, - fname, + if ( + thumbnailer.is_animated + and (t_width, t_height, t_method) not in animated_done ): - try: - await self.media_storage.write_to_file(t_byte_source, f) - finally: - t_byte_source.close() - - # We flush and close the file to ensure that the bytes have - # been written before getting the size. - f.flush() - f.close() - - t_len = os.path.getsize(fname) - - # Write to database - if server_name: - # Multiple remote media download requests can race (when - # using multiple media repos), so this may throw a violation - # constraint exception. If it does we'll delete the newly - # generated thumbnail from disk (as we're in the ctx - # manager). - # - # However: we've already called `finish()` so we may have - # also written to the storage providers. This is preferable - # to the alternative where we call `finish()` *after* this, - # where we could end up having an entry in the DB but fail - # to write the files to the storage providers. - try: - await self.store.store_remote_media_thumbnail( - server_name, - media_id, - file_id, - t_width, - t_height, - t_type, - t_method, - t_len, - ) - except Exception as e: - thumbnail_exists = ( - await self.store.get_remote_media_thumbnail( - server_name, - media_id, - t_width, - t_height, - t_type, - ) - ) - if not thumbnail_exists: - raise e - else: - await self.store.store_local_thumbnail( - media_id, t_width, t_height, t_type, t_method, t_len + animated_done.add((t_width, t_height, t_method)) + a_byte_source = await defer_to_thread( + self.hs.get_reactor(), + thumbnailer.crop + if t_method == "crop" + else thumbnailer.scale, + t_width, + t_height, + ANIMATED_THUMBNAIL_TYPE, + True, + ) + if a_byte_source: + await self._store_thumbnail( + server_name, + media_id, + file_id, + url_cache, + t_width, + t_height, + t_method, + ANIMATED_THUMBNAIL_TYPE, + a_byte_source, ) return {"width": m_width, "height": m_height} + async def _store_thumbnail( + self, + server_name: str | None, + media_id: str, + file_id: str, + url_cache: bool, + t_width: int, + t_height: int, + t_method: str, + t_type: str, + t_byte_source: BytesIO, + ) -> None: + """Store a generated thumbnail to the configured storage and database.""" + file_info = FileInfo( + server_name=server_name, + file_id=file_id, + url_cache=url_cache, + thumbnail=ThumbnailInfo( + width=t_width, + height=t_height, + method=t_method, + type=t_type, + length=t_byte_source.getbuffer().nbytes, + ), + ) + + async with self.media_storage.store_into_file(file_info) as (f, fname): + try: + await self.media_storage.write_to_file(t_byte_source, f) + finally: + t_byte_source.close() + + # We flush and close the file to ensure that the bytes have + # been written before getting the size. + f.flush() + f.close() + + t_len = os.path.getsize(fname) + + # Write to database + if server_name: + # Multiple remote media download requests can race (when + # using multiple media repos), so this may throw a violation + # constraint exception. If it does we'll delete the newly + # generated thumbnail from disk (as we're in the ctx + # manager). + # + # However: we've already called `finish()` so we may have + # also written to the storage providers. This is preferable + # to the alternative where we call `finish()` *after* this, + # where we could end up having an entry in the DB but fail + # to write the files to the storage providers. + try: + await self.store.store_remote_media_thumbnail( + server_name, + media_id, + file_id, + t_width, + t_height, + t_type, + t_method, + t_len, + ) + except Exception as e: + thumbnail_exists = await self.store.get_remote_media_thumbnail( + server_name, + media_id, + t_width, + t_height, + t_type, + ) + if not thumbnail_exists: + raise e + else: + await self.store.store_local_thumbnail( + media_id, t_width, t_height, t_type, t_method, t_len + ) + async def _apply_media_retention_rules(self) -> None: """ Purge old local and remote media according to the media retention rules diff --git a/synapse/media/thumbnailer.py b/synapse/media/thumbnailer.py index cb86a0ff66..a75a420750 100644 --- a/synapse/media/thumbnailer.py +++ b/synapse/media/thumbnailer.py @@ -20,11 +20,12 @@ # # import logging +from collections.abc import Callable from io import BytesIO from types import TracebackType -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Any, cast -from PIL import Image +from PIL import Image, ImageSequence from synapse.api.errors import Codes, NotFoundError, SynapseError, cs_error from synapse.config.repository import THUMBNAIL_SUPPORTED_MEDIA_FORMAT_MAP @@ -49,6 +50,11 @@ if TYPE_CHECKING: logger = logging.getLogger(__name__) +# Content type used when generating animated thumbnails. The spec recommends servers +# prefer WebP when supporting animation. See +# https://spec.matrix.org/v1.19/client-server-api/#get_matrixclientv1mediathumbnailservernamemediaid +ANIMATED_THUMBNAIL_TYPE = "image/webp" + EXIF_ORIENTATION_TAG = 0x0112 EXIF_TRANSPOSE_MAPPINGS = { 2: Image.FLIP_LEFT_RIGHT, @@ -66,13 +72,17 @@ class ThumbnailError(Exception): class Thumbnailer: - FORMATS = {"image/jpeg": "JPEG", "image/png": "PNG"} + FORMATS = {"image/jpeg": "JPEG", "image/png": "PNG", "image/webp": "WEBP"} # Which image formats we allow Pillow to open. # This should intentionally be kept restrictive, because the decoder of any # format in this list becomes part of our trusted computing base. PILLOW_FORMATS = ("jpeg", "png", "webp", "gif") + # Pillow reports MPO (a JPEG holding a stereo pair) as multi-frame, so + # frame count alone doesn't tell us whether something is an animation. + ANIMATED_FORMATS = frozenset({"GIF", "PNG", "WEBP"}) + @staticmethod def set_limits(max_image_pixels: int) -> None: Image.MAX_IMAGE_PIXELS = max_image_pixels @@ -80,6 +90,12 @@ class Thumbnailer: def __init__(self, input_path: str): # Have we closed the image? self._closed = False + # Whether attempting to thumbnail the image failed for some reason. The + # thumbnailing code should fallback to treating the image as static in + # this case. + # + # Cached so a broken animation isn't re-decoded for every thumbnail size. + self._animation_broken = False try: self.image = Image.open(input_path, formats=self.PILLOW_FORMATS) @@ -143,33 +159,124 @@ class Thumbnailer: else: return max((max_height * self.width) // self.height, 1), max_height - def _resize(self, width: int, height: int) -> Image.Image: + def _resize_image(self, image: Image.Image, width: int, height: int) -> Image.Image: # 1-bit or 8-bit color palette images need converting to RGB # otherwise they will be scaled using nearest neighbour which # looks awful. # # If the image has transparency, use RGBA instead. - if self.image.mode in ["1", "L", "P"]: - if self.image.info.get("transparency", None) is not None: - with self.image: - self.image = self.image.convert("RGBA") + if image.mode in ["1", "L", "P"]: + if image.info.get("transparency", None) is not None: + converted = image.convert("RGBA") else: - with self.image: - self.image = self.image.convert("RGB") - return self.image.resize((width, height), Image.LANCZOS) + converted = image.convert("RGB") + else: + converted = image + return converted.resize((width, height), Image.LANCZOS) + + @property + def has_transparency(self) -> bool: + """Whether the current frame actually makes use of transparency. + + Having an alpha channel isn't enough: fully opaque RGBA is common. + + Note that this can block for a while on large images, as `getextrema()` + scans the entire alpha plane. Consider calling it via `defer_to_thread`. + """ + image = self.image + + if image.mode == "P" and "transparency" in image.info: + return True + + if "A" not in image.getbands(): + return False + + try: + with image.getchannel("A") as alpha: + # Single band, so `getextrema` returns a plain (min, max) pair. + min_alpha = cast(float, alpha.getextrema()[0]) + except Exception: + # Assume transparency, since dropping it is the destructive option. + logger.exception("Error inspecting image alpha channel") + return True + + return min_alpha < 255 + + @property + def is_animated(self) -> bool: + if self._animation_broken: + return False + if self.image.format not in self.ANIMATED_FORMATS: + return False + return getattr(self.image, "is_animated", False) + + def _encode_animated_from( + self, transform: Callable[[Image.Image], Image.Image] + ) -> BytesIO | None: + """Apply `transform` to every frame of the source image and encode the + result as an animated thumbnail, or None if the source could not be + decoded as an animation and the caller should fall back to a static one. + """ + frames = [] + durations = [] + loop = self.image.info.get("loop", 0) + try: + for frame in ImageSequence.Iterator(self.image): + # Copy the frame to avoid referencing the original image memory. + f = frame.copy() + if f.mode != "RGBA": + f = f.convert("RGBA") + frames.append(transform(f)) + # A duration of 0 is valid (interpretation is implementation-defined, + # see RFC 9649 section 2.7.1.1), so only fall back when unset. The + # 100ms default matches libwebp's animation tools. + duration = frame.info.get("duration") + if duration is None: + duration = self.image.info.get("duration", 100) + durations.append(duration) + return self._encode_animated(frames, durations, loop) + except Exception as e: + logger.warning( + "Failed to generate an animated thumbnail, falling back to a " + "static one: %s", + e, + ) + self._animation_broken = True + + try: + # Leave the source on its first frame for the static fallback. + self.image.seek(0) + except Exception as e: + logger.warning("Failed to rewind image to its first frame: %s", e) + + return None @trace - def scale(self, width: int, height: int, output_type: str) -> BytesIO: + def scale( + self, width: int, height: int, output_type: str, animated: bool = False + ) -> BytesIO: """Rescales the image to the given dimensions. + If `animated` is set and the source image is animated, the animation is + preserved. Otherwise a static thumbnail of the first frame is produced. + Returns: The bytes of the encoded image ready to be written to disk """ - with self._resize(width, height) as scaled: + if animated and self.is_animated: + output = self._encode_animated_from( + lambda f: self._resize_image(f, width, height) + ) + if output is not None: + return output + + with self._resize_image(self.image, width, height) as scaled: return self._encode_image(scaled, output_type) @trace - def crop(self, width: int, height: int, output_type: str) -> BytesIO: + def crop( + self, width: int, height: int, output_type: str, animated: bool = False + ) -> BytesIO: """Rescales and crops the image to the given dimensions preserving aspect:: (w_in / h_in) = (w_scaled / h_scaled) @@ -196,7 +303,16 @@ class Thumbnailer: crop_right = width + crop_left crop = (crop_left, 0, crop_right, height) - with self._resize(scaled_width, scaled_height) as scaled_image: + if animated and self.is_animated: + output = self._encode_animated_from( + lambda f: self._resize_image(f, scaled_width, scaled_height).crop(crop) + ) + if output is not None: + return output + + with self._resize_image( + self.image, scaled_width, scaled_height + ) as scaled_image: with scaled_image.crop(crop) as cropped: return self._encode_image(cropped, output_type) @@ -206,6 +322,35 @@ class Thumbnailer: if fmt == "JPEG" or fmt == "PNG" and output_image.mode == "CMYK": output_image = output_image.convert("RGB") output_image.save(output_bytes_io, fmt, quality=80) + output_bytes_io.seek(0) + return output_bytes_io + + def _encode_animated( + self, + frames: list[Image.Image], + durations: list[int], + loop: int, + ) -> BytesIO: + """ + Encode a list of RGBA frames into an animated WebP, preserving per-frame + durations and the loop count. + """ + output_bytes_io = BytesIO() + if not frames: + raise ThumbnailError("No frames to encode for animated thumbnail") + + save_kwargs: dict[str, Any] = { + "format": "WEBP", + "save_all": True, + "append_images": frames[1:], + "loop": loop, + "duration": durations, + "minimize_size": True, + "quality": 80, + } + + frames[0].save(output_bytes_io, **save_kwargs) + output_bytes_io.seek(0) return output_bytes_io def close(self) -> None: @@ -318,6 +463,7 @@ class ThumbnailProvider: max_timeout_ms: int, for_federation: bool, allow_authenticated: bool = True, + animated: bool = False, ) -> None: media_info = await self.media_repo.get_local_media_info( request, media_id, max_timeout_ms @@ -379,6 +525,7 @@ class ThumbnailProvider: desired_method, desired_type, url_cache=bool(media_info.url_cache), + animated=animated, ) if thumbnail_result: @@ -413,6 +560,7 @@ class ThumbnailProvider: ip_address: str, use_federation: bool, allow_authenticated: bool = True, + animated: bool = False, ) -> None: media_info = await self.media_repo.get_remote_media_info( server_name, @@ -474,6 +622,7 @@ class ThumbnailProvider: desired_height, desired_method, desired_type, + animated=animated, ) if file_path: @@ -633,6 +782,9 @@ class ThumbnailProvider: # width/height/method so we can just call the "generate exact" # methods. + # A stored animated thumbnail is always of the animated type, so + # regenerate it as animated. + regen_animated = file_info.thumbnail.type == ANIMATED_THUMBNAIL_TYPE if server_name: await self.media_repo.generate_remote_exact_thumbnail( server_name, @@ -642,6 +794,7 @@ class ThumbnailProvider: t_height=file_info.thumbnail.height, t_method=file_info.thumbnail.method, t_type=file_info.thumbnail.type, + animated=regen_animated, ) else: await self.media_repo.generate_local_exact_thumbnail( @@ -651,6 +804,7 @@ class ThumbnailProvider: t_method=file_info.thumbnail.method, t_type=file_info.thumbnail.type, url_cache=url_cache, + animated=regen_animated, ) responder = await self.media_storage.fetch_media(file_info) diff --git a/synapse/module_api/__init__.py b/synapse/module_api/__init__.py index 1131253028..58ccddc792 100644 --- a/synapse/module_api/__init__.py +++ b/synapse/module_api/__init__.py @@ -90,6 +90,9 @@ from synapse.module_api.callbacks.account_validity_callbacks import ( ON_USER_LOGIN_CALLBACK, ON_USER_REGISTRATION_CALLBACK, ) +from synapse.module_api.callbacks.federation import ( + ON_EVENT_DELIVERED_OVER_FEDERATION_CALLBACK, +) from synapse.module_api.callbacks.media_repository_callbacks import ( GET_MEDIA_CONFIG_FOR_USER_CALLBACK, GET_MEDIA_UPLOAD_LIMITS_FOR_USER_CALLBACK, @@ -191,14 +194,21 @@ __all__ = [ "run_in_background", "run_as_background_process", "cached", + "CachedFunction", "NOT_SPAM", "UserID", + "DomainSpecificString", "DatabasePool", "LoggingTransaction", "DirectServeHtmlResource", "DirectServeJsonResource", + "SimpleHttpClient", + "SynapseRequest", "ModuleApi", + "AccountDataManager", + "PublicRoomListManager", "PRESENCE_ALL_USERS", + "UserPresenceState", "LoginResponse", "JsonDict", "JsonMapping", @@ -206,7 +216,13 @@ __all__ = [ "StateMap", "ProfileInfo", "RoomAlias", + "RoomID", + "Requester", + "UserInfo", + "UserIpAndAgent", "UserProfile", + "Absent", + "AbsentType", "RatelimitOverride", "MediaUploadLimit", ] @@ -631,6 +647,20 @@ class ModuleApi: add_field_to_unsigned_callback ) + def register_federation_callbacks( + self, + *, + on_event_delivered_over_federation: ON_EVENT_DELIVERED_OVER_FEDERATION_CALLBACK + | None = None, + ) -> None: + """Registers callbacks for federation. + + Added in Synapse v1.158.0.""" + if on_event_delivered_over_federation is not None: + self._callbacks.federation.register_callbacks( + on_event_delivered_over_federation=on_event_delivered_over_federation + ) + ######################################################################### # The following methods can be called by the module at any point in time. @@ -2025,10 +2055,11 @@ class ModuleApi: deactivation, ) - await self._hs.get_profile_handler().set_displayname( + await self._hs.get_profile_handler().dispatch_set_profile_field( target_user=user_id, requester=requester, - new_displayname=new_displayname, + field_name=ProfileFields.DISPLAYNAME, + new_value=new_displayname, by_admin=True, ) diff --git a/synapse/module_api/callbacks/__init__.py b/synapse/module_api/callbacks/__init__.py index 16ef7a4b47..6ce429dd1c 100644 --- a/synapse/module_api/callbacks/__init__.py +++ b/synapse/module_api/callbacks/__init__.py @@ -21,6 +21,8 @@ from typing import TYPE_CHECKING +from synapse.module_api.callbacks.federation import FederationModuleApiCallbacks + if TYPE_CHECKING: from synapse.server import HomeServer @@ -44,6 +46,7 @@ from synapse.module_api.callbacks.third_party_event_rules_callbacks import ( class ModuleApiCallbacks: def __init__(self, hs: "HomeServer") -> None: self.account_validity = AccountValidityModuleApiCallbacks() + self.federation = FederationModuleApiCallbacks() self.media_repository = MediaRepositoryModuleApiCallbacks(hs) self.ratelimit = RatelimitModuleApiCallbacks(hs) self.spam_checker = SpamCheckerModuleApiCallbacks(hs) diff --git a/synapse/module_api/callbacks/federation.py b/synapse/module_api/callbacks/federation.py new file mode 100644 index 0000000000..ce879b764b --- /dev/null +++ b/synapse/module_api/callbacks/federation.py @@ -0,0 +1,207 @@ +# +# This file is licensed under the Affero General Public License (AGPL) version 3. +# +# Copyright (C) 2026 Element Creations Ltd +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# See the GNU Affero General Public License for more details: +# . +# +import logging +from enum import Enum +from typing import Awaitable, Callable, Collection + +import attr + +from synapse.events import EventBase + +logger = logging.getLogger(__name__) + + +class FederatedEventDeliveryMethod(str, Enum): + """ + Method by which an event was 'delivered' to another server. + + Note that depending on the specific method, delivery may not have + actually been acknowledged by the other homeserver. + + Modules should anticipate more methods being added to this enum + over time (it is non-exhaustive). + + Only methods that deliver full, signed PDUs are included in this mechanism. + Some notable examples of excluded endpoints: + - `/send_knock` is excluded as it only returns unsigned 'stripped state'. + - `/timestamp_to_event` is excluded as it only returns event IDs, not events themselves. + """ + + SEND = "/send" + """ + The events were pushed over [`/send`](https://spec.matrix.org/v1.19/server-server-api/#put_matrixfederationv1sendtxnid). + + When a callback is triggered with this method, the events have been acknowledged + without error. + """ + + BACKFILL = "/backfill" + """ + The events were pulled over [`/backfill`](https://spec.matrix.org/v1.19/server-server-api/#get_matrixfederationv1backfillroomid). + + When a callback is triggered with this method, the events have _not_ been + acknowledged by the remote. + Actual delivery depends on network conditions and other factors influencing + the successful processing of the response at the remote homeserver. + """ + + GET_MISSING_EVENTS = "/get_missing_events" + """ + The events were pulled over [`/get_missing_events`](https://spec.matrix.org/v1.19/server-server-api/#post_matrixfederationv1get_missing_eventsroomid). + + When a callback is triggered with this method, the events have _not_ been + acknowledged by the remote. + Actual delivery depends on network conditions and other factors influencing + the successful processing of the response at the remote homeserver. + """ + + EVENT = "/event" + """ + The event was pulled over [`/event`](https://spec.matrix.org/v1.19/server-server-api/#get_matrixfederationv1eventeventid). + + When a callback is triggered with this method, the events have _not_ been + acknowledged by the remote. + Actual delivery depends on network conditions and other factors influencing + the successful processing of the response at the remote homeserver. + """ + + EVENT_AUTH = "/event_auth" + """ + The events were pulled over [`/event_auth`](https://spec.matrix.org/v1.19/server-server-api/#get_matrixfederationv1event_authroomideventid). + + When a callback is triggered with this method, the events have _not_ been + acknowledged by the remote. + Actual delivery depends on network conditions and other factors influencing + the successful processing of the response at the remote homeserver. + """ + + STATE = "/state" + """ + The events were pulled over [`/state`](https://spec.matrix.org/v1.19/server-server-api/#get_matrixfederationv1stateroomid). + + When a callback is triggered with this method, the events have _not_ been + acknowledged by the remote. + Actual delivery depends on network conditions and other factors influencing + the successful processing of the response at the remote homeserver. + """ + + SEND_JOIN = "/send_join" + """ + The events were pulled over [`/send_join`](https://spec.matrix.org/v1.19/server-server-api/#put_matrixfederationv2send_joinroomideventid). + + When a callback is triggered with this method, the events have _not_ been + acknowledged by the remote. + Actual delivery depends on network conditions and other factors influencing + the successful processing of the response at the remote homeserver. + """ + + +@attr.s(frozen=True, slots=True, auto_attribs=True) +class FederationEventDeliveryEvent: + """ + Represents the delivery of some events. + + Note that depending on `method`, + delivery may not be acknowledged. + """ + + server_name: str + """ + The server name of the destination the events were delivered to. + """ + + events: Collection[EventBase] + """ + The events that were delivered. + + Modules should not rely on this being the exhaustive list of all events that + were delivered in a single request; + delivery hooks may be triggered in multiple batches. + """ + + method: FederatedEventDeliveryMethod + """ + How the events were delivered to the server. + """ + + +ON_EVENT_DELIVERED_OVER_FEDERATION_CALLBACK = Callable[ + [FederationEventDeliveryEvent], Awaitable[None] +] + + +class FederationModuleApiCallbacks: + """ + Module API callbacks for generic federation events. + """ + + def __init__(self) -> None: + self._on_event_delivered_over_federation_callbacks: list[ + ON_EVENT_DELIVERED_OVER_FEDERATION_CALLBACK + ] = [] + + def interested_in_events_delivered_over_federation(self) -> bool: + """ + Whether any `on_event_delivered_over_federation` callbacks are registered. + """ + return len(self._on_event_delivered_over_federation_callbacks) > 0 + + def register_callbacks( + self, + on_event_delivered_over_federation: ON_EVENT_DELIVERED_OVER_FEDERATION_CALLBACK + | None = None, + ) -> None: + """ + Register callbacks from module for each hook. + + on_event_delivered_over_federation: + Callback fired when an event is delivered over federation. + See `FederationEventDeliveryEvent` for details. + + Performance note: + Registering this hook causes a performance (caching) optimisation on the + Federation `/state` endpoint to be bypassed. + """ + if on_event_delivered_over_federation is not None: + self._on_event_delivered_over_federation_callbacks.append( + on_event_delivered_over_federation + ) + + async def notify_on_event_delivered_over_federation( + self, + server_name: str, + events: Collection[EventBase], + method: FederatedEventDeliveryMethod, + ) -> None: + """Fire the registered callbacks to notify modules that some events were + delivered to another homeserver over federation. + + Does nothing if no callbacks are registered or if there are no events to + report. A callback that raises is logged and does not interrupt the others. + """ + if not events or not self._on_event_delivered_over_federation_callbacks: + return + + delivery = FederationEventDeliveryEvent( + server_name=server_name, + events=events, + method=method, + ) + for callback in self._on_event_delivered_over_federation_callbacks: + try: + await callback(delivery) + except Exception: + logger.exception( + "Error running on_event_delivered_over_federation callback" + ) diff --git a/synapse/module_api/callbacks/media_repository_callbacks.py b/synapse/module_api/callbacks/media_repository_callbacks.py index f1e6ea4c38..5b58229b77 100644 --- a/synapse/module_api/callbacks/media_repository_callbacks.py +++ b/synapse/module_api/callbacks/media_repository_callbacks.py @@ -150,7 +150,10 @@ class MediaRepositoryModuleApiCallbacks: ): # Use a copy of the data in case the module modifies it limit_copy = MediaUploadLimit( - max_bytes=limit.max_bytes, time_period_ms=limit.time_period_ms + max_bytes=limit.max_bytes, + time_period_ms=limit.time_period_ms, + info_uri=limit.info_uri, + can_upgrade=limit.can_upgrade, ) await delay_cancellation( callback(user_id, limit_copy, sent_bytes, attempted_bytes) diff --git a/synapse/notifier.py b/synapse/notifier.py index 6a057ac09f..e24d0ef5a2 100644 --- a/synapse/notifier.py +++ b/synapse/notifier.py @@ -528,6 +528,7 @@ class Notifier: StreamKeyType.UN_PARTIAL_STATED_ROOMS, StreamKeyType.THREAD_SUBSCRIPTIONS, StreamKeyType.STICKY_EVENTS, + StreamKeyType.PROFILE_UPDATES, ], new_token: int, users: Collection[str | UserID] | None = None, diff --git a/synapse/replication/http/__init__.py b/synapse/replication/http/__init__.py index 68cc6ce1fc..d934ef8067 100644 --- a/synapse/replication/http/__init__.py +++ b/synapse/replication/http/__init__.py @@ -30,6 +30,7 @@ from synapse.replication.http import ( login, membership, presence, + profile, push, register, send_events, @@ -59,6 +60,7 @@ class ReplicationRestResource(JsonResource): push.register_servlets(hs, self) state.register_servlets(hs, self) devices.register_servlets(hs, self) + profile.register_servlets(hs, self) # The following can't currently be instantiated on workers. if hs.config.worker.worker_app is None: diff --git a/synapse/replication/http/_base.py b/synapse/replication/http/_base.py index 87d6e80898..686c2351d2 100644 --- a/synapse/replication/http/_base.py +++ b/synapse/replication/http/_base.py @@ -344,7 +344,12 @@ class ReplicationEndpoint(metaclass=abc.ABCMeta): code=e.code, **{SERVER_NAME_LABEL: server_name}, ).inc() - raise e.to_synapse_error() + # This error is coming from another worker, so we trust it to be safe + # to relay to clients directly. + # In fact, we rely relaying verbatim at the very least to tell + # clients when they are rate-limited, + # but most likely other things too. + raise e.unsafe_to_verbatim_synapse_error() except Exception as e: _outgoing_request_counter.labels( name=cls.NAME, diff --git a/synapse/replication/http/profile.py b/synapse/replication/http/profile.py new file mode 100644 index 0000000000..dddc36477f --- /dev/null +++ b/synapse/replication/http/profile.py @@ -0,0 +1,150 @@ +# +# This file is licensed under the Affero General Public License (AGPL) version 3. +# +# Copyright (C) 2026 Element Creations, Ltd +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# See the GNU Affero General Public License for more details: +# . +# + + +import logging +from typing import TYPE_CHECKING + +from twisted.web.server import Request + +from synapse.http.server import HttpServer +from synapse.replication.http._base import ReplicationEndpoint +from synapse.synapse_rust.types import Requester +from synapse.types import JsonDict, JsonValue, UserID, create_requester + +if TYPE_CHECKING: + from synapse.server import HomeServer + +logger = logging.getLogger(__name__) + + +class ReplicationProfileSetField(ReplicationEndpoint): + """Update a profile field for a user. + + The POST looks like: + + POST /_synapse/replication/profile_set_field/ + + { + "requester": "@admin:hs", + "field_name": "displayname", + "new_value": "Alice", + "by_admin": true, + "propagate": false + } + + 200 OK + + {} + """ + + NAME = "profile_set_field" + PATH_ARGS = ("user_id",) + METHOD = "POST" + CACHE = False + + def __init__(self, hs: "HomeServer"): + super().__init__(hs) + + self._profile_handler = hs.get_profile_handler() + + @staticmethod + async def _serialize_payload( # type: ignore[override] + user_id: str, + requester: Requester, + field_name: str, + new_value: JsonValue | dict[str, JsonValue], + by_admin: bool, + propagate: bool, + ) -> JsonDict: + return { + "requester": requester.user.to_string(), + "field_name": field_name, + "new_value": new_value, + "by_admin": by_admin, + "propagate": propagate, + } + + async def _handle_request( # type: ignore[override] + self, request: Request, content: JsonDict, user_id: str + ) -> tuple[int, JsonDict]: + await self._profile_handler.set_field( + target_user=UserID.from_string(user_id), + requester=create_requester(content["requester"]), + field_name=content["field_name"], + new_value=content["new_value"], + by_admin=content["by_admin"], + propagate=content["propagate"], + ) + + return (200, {}) + + +class ReplicationProfileDeleteField(ReplicationEndpoint): + """Delete a profile field for a user. + + The POST looks like: + + POST /_synapse/replication/profile_delete_field/ + + { + "requester": "@admin:hs", + "field_name": "displayname", + "by_admin": true + } + + 200 OK + + {} + """ + + NAME = "profile_delete_field" + PATH_ARGS = ("user_id",) + METHOD = "POST" + CACHE = False + + def __init__(self, hs: "HomeServer"): + super().__init__(hs) + + self._profile_handler = hs.get_profile_handler() + + @staticmethod + async def _serialize_payload( # type: ignore[override] + user_id: str, + requester: Requester, + field_name: str, + by_admin: bool, + ) -> JsonDict: + return { + "requester": requester.user.to_string(), + "field_name": field_name, + "by_admin": by_admin, + } + + async def _handle_request( # type: ignore[override] + self, request: Request, content: JsonDict, user_id: str + ) -> tuple[int, JsonDict]: + await self._profile_handler.delete_profile_field( + target_user=UserID.from_string(user_id), + requester=create_requester(content["requester"]), + field_name=content["field_name"], + by_admin=content["by_admin"], + ) + + return (200, {}) + + +def register_servlets(hs: "HomeServer", http_server: HttpServer) -> None: + ReplicationProfileSetField(hs).register(http_server) + ReplicationProfileDeleteField(hs).register(http_server) diff --git a/synapse/replication/tcp/client.py b/synapse/replication/tcp/client.py index bc7e46d4c9..c0896b83e7 100644 --- a/synapse/replication/tcp/client.py +++ b/synapse/replication/tcp/client.py @@ -44,6 +44,7 @@ from synapse.replication.tcp.streams import ( UnPartialStatedRoomStream, ) from synapse.replication.tcp.streams._base import ( + ProfileUpdatesStream, StickyEventsStream, ThreadSubscriptionsStream, ) @@ -265,6 +266,23 @@ class ReplicationDataHandler: token, users=[row.user_id for row in rows], ) + elif stream_name == ProfileUpdatesStream.NAME: + updated_user_ids = {row.user_id for row in rows} + if updated_user_ids: + room_ids: set[str] = set() + # Get all the rooms of the updated users, dict of + # User ID -> [Room ID] + users_and_rooms = await self.store.get_rooms_for_users(updated_user_ids) + # Loop through each user's room IDs and add to our set of rooms + for user_room_ids in users_and_rooms.values(): + room_ids.update(user_room_ids) + + if room_ids: + self.notifier.on_new_event( + StreamKeyType.PROFILE_UPDATES, + token, + rooms=room_ids, + ) elif stream_name == StickyEventsStream.NAME: self.notifier.on_new_event( StreamKeyType.STICKY_EVENTS, diff --git a/synapse/replication/tcp/handler.py b/synapse/replication/tcp/handler.py index ad9fed72dd..b63902b3ec 100644 --- a/synapse/replication/tcp/handler.py +++ b/synapse/replication/tcp/handler.py @@ -67,6 +67,8 @@ from synapse.replication.tcp.streams import ( ) from synapse.replication.tcp.streams._base import ( DeviceListsStream, + ProfileUpdatesStream, + QuarantinedMediaStream, StickyEventsStream, ThreadSubscriptionsStream, ) @@ -218,6 +220,12 @@ class ReplicationCommandHandler: continue + if isinstance(stream, ProfileUpdatesStream): + if hs.get_instance_name() in hs.config.worker.writers.events: + self._streams_to_replicate.append(stream) + + continue + if isinstance(stream, StickyEventsStream): if hs.get_instance_name() in hs.config.worker.writers.events: self._streams_to_replicate.append(stream) @@ -230,6 +238,15 @@ class ReplicationCommandHandler: continue + if isinstance(stream, QuarantinedMediaStream): + if ( + hs.get_instance_name() + in hs.config.worker.writers.quarantined_media_changes + ): + self._streams_to_replicate.append(stream) + + continue + # Only add any other streams if we're on master. if hs.config.worker.worker_app is not None: continue diff --git a/synapse/replication/tcp/resource.py b/synapse/replication/tcp/resource.py index 95364778cc..7d6886e0b8 100644 --- a/synapse/replication/tcp/resource.py +++ b/synapse/replication/tcp/resource.py @@ -22,7 +22,7 @@ import logging import random -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Sequence from prometheus_client import Counter @@ -317,7 +317,7 @@ class ReplicationStreamer: def _batch_updates( - updates: list[tuple[Token, StreamRow]], + updates: Sequence[tuple[Token, StreamRow]], ) -> list[tuple[Token | None, StreamRow]]: """Takes a list of updates of form [(token, row)] and sets the token to None for all rows where the next row has the same token. This is used to diff --git a/synapse/replication/tcp/streams/__init__.py b/synapse/replication/tcp/streams/__init__.py index e41573cf68..e657822da7 100644 --- a/synapse/replication/tcp/streams/__init__.py +++ b/synapse/replication/tcp/streams/__init__.py @@ -37,6 +37,7 @@ from synapse.replication.tcp.streams._base import ( DeviceListsStream, PresenceFederationStream, PresenceStream, + ProfileUpdatesStream, PushersStream, PushRulesStream, QuarantinedMediaStream, @@ -70,6 +71,7 @@ STREAMS_MAP = { ToDeviceStream, FederationStream, AccountDataStream, + ProfileUpdatesStream, StickyEventsStream, ThreadSubscriptionsStream, UnPartialStatedRoomStream, @@ -94,6 +96,7 @@ __all__ = [ "ToDeviceStream", "FederationStream", "AccountDataStream", + "ProfileUpdatesStream", "StickyEventsStream", "ThreadSubscriptionsStream", "UnPartialStatedRoomStream", diff --git a/synapse/replication/tcp/streams/_base.py b/synapse/replication/tcp/streams/_base.py index a73f767add..f9c7821a39 100644 --- a/synapse/replication/tcp/streams/_base.py +++ b/synapse/replication/tcp/streams/_base.py @@ -26,13 +26,15 @@ from typing import ( Any, Awaitable, Callable, - TypeVar, + Sequence, + Union, ) import attr -from synapse.api.constants import AccountDataTypes +from synapse.api.constants import AccountDataTypes, ProfileUpdateAction from synapse.replication.http.streams import ReplicationGetStreamUpdates +from synapse.types import UserID if TYPE_CHECKING: from synapse.server import HomeServer @@ -49,11 +51,36 @@ _STREAM_UPDATE_TARGET_ROW_COUNT = 100 # A stream position token Token = int +RdataSafeValue = Union[ + str, + int, + bool, + float, + None, + # We could probably expand to support immutable types of these, but + # when they come off the wire they will deserialise to the mutable + # types again. + # Since nobody is using it right now, stick to `list` and `dict` + list["RdataSafeValue"], + dict[str, "RdataSafeValue"], +] +""" +Safe types that can be used in RDATA commands and thus used as +the wire format of stream rows. + +Prevents you from thinking you can push e.g. a `frozenset` over +the wire and get it back on the other end. + +At the moment, to be safe, a type has to roundtrip correctly with our JSON codec. +Consult the RdataCommand `from_line` and `to_line` for information. +""" + # The type of a stream update row, after JSON deserialisation, but before # parsing with Stream.parse_row (which turns it into a `ROW_TYPE`). Normally it's # just a row from a database query, though this is dependent on the stream in question. # -StreamRow = TypeVar("StreamRow", bound=tuple) +# NOTE: Prefer to use tuples, but since we have some streams still using list, support those for now. +StreamRow = Union[tuple[RdataSafeValue, ...], list[RdataSafeValue]] # The type returned by the update_function of a stream, as well as get_updates(), # get_updates_since, etc. @@ -63,7 +90,7 @@ StreamRow = TypeVar("StreamRow", bound=tuple) # * `new_last_token` is the new position in stream. # * `limited` is whether there are more updates to fetch. # -StreamUpdateResult = tuple[list[tuple[Token, StreamRow]], Token, bool] +StreamUpdateResult = tuple[Sequence[tuple[Token, StreamRow]], Token, bool] # The type of an update_function for a stream # @@ -406,9 +433,9 @@ class TypingStream(Stream): if hs.get_instance_name() in hs.config.worker.writers.typing: # On the writer, query the typing handler typing_writer_handler = hs.get_typing_writer_handler() - update_function: Callable[ - [str, int, int, int], Awaitable[tuple[list[tuple[int, Any]], int, bool]] - ] = typing_writer_handler.get_all_typing_updates + update_function: UpdateFunction = ( + typing_writer_handler.get_all_typing_updates + ) self.current_token_function = typing_writer_handler.get_current_token else: # Query the typing writer process @@ -765,6 +792,80 @@ class ThreadSubscriptionsStream(_StreamFromIdGen): return rows, rows[-1][0], len(updates) == limit +def _convert_affected_fields( + wire: list[str] | frozenset[str] | None, +) -> frozenset[str] | None: + return ( + frozenset(wire) + if wire is not None and not isinstance(wire, frozenset) + else None + ) + + +@attr.s(slots=True, auto_attribs=True) +class ProfileUpdatesStreamRow: + """Profile update stream row detailing what the profile update changes.""" + + user_id: UserID + """The full user ID with the profile update.""" + action: ProfileUpdateAction + """The action, either 'update' for a field update, 'left_room' if the user left + a room or `joined_room` if the user joined a room, see ProfileUpdateAction enum. + """ + affected_fields: frozenset[str] | None = attr.ib( + # Convert list back to frozenset from wire format + converter=_convert_affected_fields + ) + """Names of the profile fields that were added, updated or removed, see https://spec.matrix.org/unstable/client-server-api/#profiles. + This is None if `action` is not `update`. + """ + + +class ProfileUpdatesStream(_StreamFromIdGen): + """Stream to inform users about profile updates.""" + + # FIXME: See issue https://github.com/element-hq/synapse/issues/19981 + # for concerns around the current implementation of the profile + # updates stream. + + NAME = "profile_updates" + ROW_TYPE = ProfileUpdatesStreamRow + + def __init__(self, hs: "HomeServer"): + self.store = hs.get_datastores().main + super().__init__( + hs.get_instance_name(), + self._update_function, + self.store._profile_updates_id_gen, + ) + + async def _update_function( + self, instance_name: str, from_token: int, to_token: int, limit: int + ) -> StreamUpdateResult: + updates = await self.store.get_updated_profile_updates( + from_id=from_token, to_id=to_token, limit=limit + ) + rows: list[tuple[int, tuple[RdataSafeValue, ...]]] = [ + ( + stream_id, + # These are the args to `ProfileUpdatesStreamRow` + ( + user_id, + action, + # Must convert `field_names` to a list for transport over the wire + # It will be reconstructed as a frozenset on the other end + list(field_names) if field_names is not None else None, + ), + ) + for stream_id, user_id, action, field_names in updates + ] + + if not rows: + return [], to_token, False + + return rows, rows[-1][0], len(updates) == limit + + @attr.s(slots=True, auto_attribs=True) class StickyEventsStreamRow: """Stream to inform workers about changes to sticky events.""" diff --git a/synapse/replication/tcp/streams/federation.py b/synapse/replication/tcp/streams/federation.py index c99e720381..ac600441dd 100644 --- a/synapse/replication/tcp/streams/federation.py +++ b/synapse/replication/tcp/streams/federation.py @@ -18,13 +18,14 @@ # [This file includes modifications made by New Vector Limited] # # -from typing import TYPE_CHECKING, Any, Awaitable, Callable +from typing import TYPE_CHECKING import attr from synapse.replication.tcp.streams._base import ( Stream, Token, + UpdateFunction, current_token_without_instance, make_http_update_function, ) @@ -57,9 +58,7 @@ class FederationStream(Stream): self.current_token_func = current_token_without_instance( federation_sender.get_current_token ) - update_function: Callable[ - [str, int, int, int], Awaitable[tuple[list[tuple[int, Any]], int, bool]] - ] = federation_sender.get_replication_rows + update_function: UpdateFunction = federation_sender.get_replication_rows elif hs.should_send_federation(): # federation sender: Query master process diff --git a/synapse/res/templates/media_upload_limit_exceeded.html b/synapse/res/templates/media_upload_limit_exceeded.html new file mode 100644 index 0000000000..62be18a138 --- /dev/null +++ b/synapse/res/templates/media_upload_limit_exceeded.html @@ -0,0 +1,6 @@ +{% extends "_base.html" %} +{% block title %}Media upload limit exceeded{% endblock %} + +{% block body %} +

You have exceeded a media upload limit. Ask your server administrator for more information.

+{% endblock %} diff --git a/synapse/rest/__init__.py b/synapse/rest/__init__.py index a56a81a8e9..1f705aee23 100644 --- a/synapse/rest/__init__.py +++ b/synapse/rest/__init__.py @@ -28,6 +28,7 @@ from synapse.rest.client import ( account_data, account_validity, appservice_ping, + appservice_proxy, auth, auth_metadata, capabilities, @@ -60,6 +61,7 @@ from synapse.rest.client import ( retention, room, room_keys, + room_membership, room_upgrade_rest_servlet, sendtodevice, sync, @@ -128,6 +130,8 @@ CLIENT_SERVLET_FUNCTIONS: tuple[RegisterServletsFunc, ...] = ( rendezvous.register_servlets, auth_metadata.register_servlets, thread_subscriptions.register_servlets, + room_membership.register_servlets, + appservice_proxy.register_servlets, ) SERVLET_GROUPS: dict[str, Iterable[RegisterServletsFunc]] = { diff --git a/synapse/rest/admin/experimental_features.py b/synapse/rest/admin/experimental_features.py index abdb937793..c91c5b6a49 100644 --- a/synapse/rest/admin/experimental_features.py +++ b/synapse/rest/admin/experimental_features.py @@ -62,7 +62,7 @@ class ExperimentalFeaturesRestServlet(RestServlet): for a given user """ - PATTERNS = admin_patterns("/experimental_features/(?P[^/]*)") + PATTERNS = admin_patterns("/experimental_features/(?P[^/]*)$") def __init__(self, hs: "HomeServer"): super().__init__() diff --git a/synapse/rest/admin/scheduled_tasks.py b/synapse/rest/admin/scheduled_tasks.py index 5b3526c7e5..08c7bec783 100644 --- a/synapse/rest/admin/scheduled_tasks.py +++ b/synapse/rest/admin/scheduled_tasks.py @@ -15,7 +15,12 @@ # from typing import TYPE_CHECKING -from synapse.http.servlet import RestServlet, parse_integer, parse_string +from synapse.http.servlet import ( + RestServlet, + parse_integer, + parse_string, + parse_strings_from_args, +) from synapse.http.site import SynapseRequest from synapse.rest.admin import admin_patterns, assert_requester_is_admin from synapse.types import JsonDict, TaskStatus @@ -38,19 +43,33 @@ class ScheduledTasksRestServlet(RestServlet): async def on_GET(self, request: SynapseRequest) -> tuple[int, JsonDict]: await assert_requester_is_admin(self._auth, request) + # twisted.web.server.Request.args is incorrectly defined as Any | None + args: dict[bytes, list[bytes]] = request.args # type: ignore + # extract query params - action_name = parse_string(request, "action_name") + actions = parse_strings_from_args(args, "action_name") resource_id = parse_string(request, "resource_id") - status = parse_string(request, "status") + status_strings = parse_strings_from_args( + args, + "status", + allowed_values=[status.value for status in TaskStatus], + ) # This parameter was historically called `job_status`, while the Admin API docs # defined it as `status`. We now support both, as `status` is generally # a nicer name. A v2 of this endpoint should keep only `status`. - if status is None: - status = parse_string(request, "job_status") + if status_strings is None: + status_strings = parse_strings_from_args( + args, + "job_status", + allowed_values=[status.value for status in TaskStatus], + ) max_timestamp = parse_integer(request, "max_timestamp") - actions = [action_name] if action_name else None - statuses = [TaskStatus(status)] if status else None + statuses = ( + [TaskStatus(status) for status in status_strings] + if status_strings + else None + ) tasks = await self._store.get_scheduled_tasks( actions=actions, diff --git a/synapse/rest/admin/users.py b/synapse/rest/admin/users.py index 1bd48e18cc..43dab16598 100644 --- a/synapse/rest/admin/users.py +++ b/synapse/rest/admin/users.py @@ -28,7 +28,7 @@ from typing import TYPE_CHECKING import attr from pydantic import StrictBool, StrictInt, StrictStr -from synapse.api.constants import Direction +from synapse.api.constants import Direction, ProfileFields from synapse.api.errors import Codes, NotFoundError, SynapseError from synapse.http.servlet import ( RestServlet, @@ -49,6 +49,7 @@ from synapse.rest.admin._base import ( assert_user_is_admin, ) from synapse.rest.client._base import client_patterns +from synapse.storage.databases.main import UserPaginateResponse from synapse.storage.databases.main.registration import ExternalIDReuseException from synapse.storage.databases.main.stats import UserSortOrder from synapse.types import JsonDict, JsonMapping, TaskStatus, UserID @@ -181,13 +182,16 @@ class UsersRestServletV2(RestServlet): ) # If support for MSC3866 is not enabled, don't show the approval flag. - filter = None + users_filter = None if not self._msc3866_enabled: + users_filter = attr.filters.exclude( + attr.fields(UserPaginateResponse).approved + ) - def _filter(a: attr.Attribute) -> bool: - return a.name != "approved" - - ret = {"users": [attr.asdict(u, filter=filter) for u in users], "total": total} + ret = { + "users": [attr.asdict(u, filter=users_filter) for u in users], + "total": total, + } if (start + limit) < total: ret["next_token"] = str(start + len(users)) @@ -366,8 +370,12 @@ class UserRestServletV2(UserRestServletV2Get): if user: # modify user if "displayname" in body: - await self.profile_handler.set_displayname( - target_user, requester, body["displayname"], by_admin=True + await self.profile_handler.dispatch_set_profile_field( + target_user=target_user, + requester=requester, + field_name=ProfileFields.DISPLAYNAME, + new_value=body["displayname"], + by_admin=True, ) if threepids is not None: @@ -415,8 +423,12 @@ class UserRestServletV2(UserRestServletV2Get): ) if "avatar_url" in body: - await self.profile_handler.set_avatar_url( - target_user, requester, body["avatar_url"], by_admin=True + await self.profile_handler.dispatch_set_profile_field( + target_user=target_user, + requester=requester, + field_name=ProfileFields.AVATAR_URL, + new_value=body["avatar_url"], + by_admin=True, ) if "admin" in body: @@ -523,8 +535,12 @@ class UserRestServletV2(UserRestServletV2Get): ) if "avatar_url" in body and isinstance(body["avatar_url"], str): - await self.profile_handler.set_avatar_url( - target_user, requester, body["avatar_url"], by_admin=True + await self.profile_handler.dispatch_set_profile_field( + target_user=target_user, + requester=requester, + field_name=ProfileFields.AVATAR_URL, + new_value=body["avatar_url"], + by_admin=True, ) user_info_dict = await self.admin_handler.get_user(target_user) @@ -1371,7 +1387,7 @@ class RateLimitRestServlet(RestServlet): class AccountDataRestServlet(RestServlet): """Retrieve the given user's account data""" - PATTERNS = admin_patterns("/users/(?P[^/]*)/accountdata") + PATTERNS = admin_patterns("/users/(?P[^/]*)/accountdata$") def __init__(self, hs: "HomeServer"): self._auth = hs.get_auth() @@ -1409,7 +1425,7 @@ class UserReplaceMasterCrossSigningKeyRestServlet(RestServlet): """ PATTERNS = admin_patterns( - "/users/(?P[^/]*)/_allow_cross_signing_replacement_without_uia" + "/users/(?P[^/]*)/_allow_cross_signing_replacement_without_uia$" ) REPLACEMENT_PERIOD_MS = 10 * 60 * 1000 # 10 minutes @@ -1443,7 +1459,7 @@ class UserByExternalId(RestServlet): """Find a user based on an external ID from an auth provider""" PATTERNS = admin_patterns( - "/auth_providers/(?P[^/]*)/users/(?P[^/]*)" + "/auth_providers/(?P[^/]*)/users/(?P[^/]*)$" ) def __init__(self, hs: "HomeServer"): @@ -1469,7 +1485,7 @@ class UserByExternalId(RestServlet): class UserByThreePid(RestServlet): """Find a user based on 3PID of a particular medium""" - PATTERNS = admin_patterns("/threepid/(?P[^/]*)/users/(?P
[^/]*)") + PATTERNS = admin_patterns("/threepid/(?P[^/]*)/users/(?P
[^/]*)$") def __init__(self, hs: "HomeServer"): self._auth = hs.get_auth() @@ -1503,7 +1519,7 @@ class RedactUser(RestServlet): If only one parameter is sent, then all messages before or after given time will be redacted. """ - PATTERNS = admin_patterns("/user/(?P[^/]*)/redact") + PATTERNS = admin_patterns("/user/(?P[^/]*)/redact$") def __init__(self, hs: "HomeServer"): self._auth = hs.get_auth() @@ -1614,7 +1630,7 @@ class UserInvitesCount(RestServlet): Return the count of invites that the user has sent after the given timestamp """ - PATTERNS = admin_patterns("/users/(?P[^/]*)/sent_invite_count") + PATTERNS = admin_patterns("/users/(?P[^/]*)/sent_invite_count$") def __init__(self, hs: "HomeServer"): self._auth = hs.get_auth() @@ -1639,7 +1655,7 @@ class UserJoinedRoomCount(RestServlet): if they have subsequently left/been banned from those rooms. """ - PATTERNS = admin_patterns("/users/(?P[^/]*)/cumulative_joined_room_count") + PATTERNS = admin_patterns("/users/(?P[^/]*)/cumulative_joined_room_count$") def __init__(self, hs: "HomeServer"): self._auth = hs.get_auth() diff --git a/synapse/rest/client/account.py b/synapse/rest/client/account.py index 3b01e40121..13bc703628 100644 --- a/synapse/rest/client/account.py +++ b/synapse/rest/client/account.py @@ -423,6 +423,17 @@ class MsisdnThreepidRequestTokenRestServlet(RestServlet): self.identity_handler = hs.get_identity_handler() async def on_POST(self, request: SynapseRequest) -> tuple[int, JsonDict]: + if not self.hs.config.registration.account_threepid_delegate_msisdn: + logger.warning( + "No upstream msisdn account_threepid_delegate configured on the server to " + "handle this request" + ) + raise SynapseError( + 400, + "Adding phone numbers to user account is not supported by this homeserver", + Codes.THREEPID_MEDIUM_NOT_SUPPORTED, + ) + body = parse_and_validate_json_object_from_request( request, MsisdnRequestTokenBody ) @@ -462,17 +473,6 @@ class MsisdnThreepidRequestTokenRestServlet(RestServlet): logger.info("MSISDN %s is already in use by %s", msisdn, existing_user_id) raise SynapseError(400, "MSISDN is already in use", Codes.THREEPID_IN_USE) - if not self.hs.config.registration.account_threepid_delegate_msisdn: - logger.warning( - "No upstream msisdn account_threepid_delegate configured on the server to " - "handle this request" - ) - raise SynapseError( - 400, - "Adding phone numbers to user account is not supported by this homeserver", - Codes.THREEPID_MEDIUM_NOT_SUPPORTED, - ) - ret = await self.identity_handler.requestMsisdnToken( self.hs.config.registration.account_threepid_delegate_msisdn, body.country, diff --git a/synapse/rest/client/account_data.py b/synapse/rest/client/account_data.py index b18232fc56..cccc0473e8 100644 --- a/synapse/rest/client/account_data.py +++ b/synapse/rest/client/account_data.py @@ -61,7 +61,7 @@ class AccountDataServlet(RestServlet): """ PATTERNS = client_patterns( - "/user/(?P[^/]*)/account_data/(?P[^/]*)" + "/user/(?P[^/]*)/account_data/(?P[^/]*)$" ) CATEGORY = "Account data requests" @@ -136,7 +136,7 @@ class UnstableAccountDataServlet(RestServlet): PATTERNS = client_patterns( "/org.matrix.msc3391/user/(?P[^/]*)" - "/account_data/(?P[^/]*)", + "/account_data/(?P[^/]*)$", unstable=True, releases=(), ) @@ -174,7 +174,7 @@ class RoomAccountDataServlet(RestServlet): PATTERNS = client_patterns( "/user/(?P[^/]*)" "/rooms/(?P[^/]*)" - "/account_data/(?P[^/]*)" + "/account_data/(?P[^/]*)$" ) CATEGORY = "Account data requests" @@ -271,7 +271,7 @@ class UnstableRoomAccountDataServlet(RestServlet): PATTERNS = client_patterns( "/org.matrix.msc3391/user/(?P[^/]*)" "/rooms/(?P[^/]*)" - "/account_data/(?P[^/]*)", + "/account_data/(?P[^/]*)$", unstable=True, releases=(), ) diff --git a/synapse/rest/client/appservice_ping.py b/synapse/rest/client/appservice_ping.py index 2c6ad5bcf0..4cbf6935c5 100644 --- a/synapse/rest/client/appservice_ping.py +++ b/synapse/rest/client/appservice_ping.py @@ -46,7 +46,7 @@ logger = logging.getLogger(__name__) class AppservicePingRestServlet(RestServlet): PATTERNS = client_patterns( - "/appservice/(?P[^/]*)/ping", + "/appservice/(?P[^/]*)/ping$", releases=("v1",), ) diff --git a/synapse/rest/client/appservice_proxy.py b/synapse/rest/client/appservice_proxy.py new file mode 100644 index 0000000000..aac1f465d3 --- /dev/null +++ b/synapse/rest/client/appservice_proxy.py @@ -0,0 +1,82 @@ +# +# This file is licensed under the Affero General Public License (AGPL) version 3. +# +# Copyright (C) 2026 Element Creations Ltd +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# See the GNU Affero General Public License for more details: +# . +# + +import logging +import re +from typing import TYPE_CHECKING + +from synapse.api.ratelimiting import RequestRatelimiter +from synapse.appservice import ApplicationService +from synapse.http import QuieterFileBodyProducer +from synapse.http.appservice_proxy import proxy_request_to_appservice +from synapse.http.server import HttpServer, ServletCallback +from synapse.http.site import SynapseRequest + +if TYPE_CHECKING: + from synapse.server import HomeServer + +logger = logging.getLogger(__name__) + + +def _make_proxy_callback( + hs: "HomeServer", + ratelimiter: RequestRatelimiter, + appservice: ApplicationService, +) -> ServletCallback: + async def _proxy(request: SynapseRequest, **kwargs: str) -> None: + requester = await hs.get_auth().get_user_by_req(request) + + await ratelimiter.ratelimit(requester) + + await proxy_request_to_appservice( + request, + hs, + appservice, + QuieterFileBodyProducer(request.content), + extra_request_headers={ + b"X-Matrix-User-Identifier": requester.user.to_string().encode("ascii") + }, + ) + + return _proxy + + +def register_servlets(hs: "HomeServer", http_server: HttpServer) -> None: + """Registers blanket reverse-proxy routes for each application service that has + configured a proxy prefix. This forwards requests under + /_matrix/client///* (where is either "vN" or "unstable") + to the same path under the application service's proxy URL after verifying request + authentication. + """ + if not hs.config.experimental.msc4512_enabled: + return + + ratelimiter = hs.get_request_ratelimiter() + for appservice in hs.get_datastores().main.get_app_services(): + if appservice.proxy_prefix is None: + continue + + pattern = re.compile( + r"^/_matrix/client/(?:unstable/[^/]+|v[^/]+)/%s(/.*)?$" + % (re.escape(appservice.proxy_prefix),) + ) + callback = _make_proxy_callback(hs, ratelimiter, appservice) + + for method in ("GET", "POST", "PUT", "DELETE"): + http_server.register_paths( + method, + (pattern,), + callback, + "ApplicationServiceClientProxy", + ) diff --git a/synapse/rest/client/auth.py b/synapse/rest/client/auth.py index b1775346f6..215779a355 100644 --- a/synapse/rest/client/auth.py +++ b/synapse/rest/client/auth.py @@ -47,7 +47,7 @@ class AuthRestServlet(RestServlet): Current use is for web fallback auth. """ - PATTERNS = client_patterns(r"/auth/(?P[\w\.]*)/fallback/web") + PATTERNS = client_patterns(r"/auth/(?P[\w\.]*)/fallback/web$") def __init__(self, hs: "HomeServer"): super().__init__() diff --git a/synapse/rest/client/auth_metadata.py b/synapse/rest/client/auth_metadata.py index 42decfdd6a..c5b874b533 100644 --- a/synapse/rest/client/auth_metadata.py +++ b/synapse/rest/client/auth_metadata.py @@ -11,7 +11,6 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -import logging import typing from synapse.api.auth.mas import MasDelegatedAuth @@ -26,61 +25,6 @@ if typing.TYPE_CHECKING: from synapse.server import HomeServer -logger = logging.getLogger(__name__) - - -class AuthIssuerServlet(RestServlet): - """ - Advertises what OpenID Connect issuer clients should use to authorise users. - This endpoint was defined in a previous iteration of MSC2965, and is still - used by some clients. - """ - - PATTERNS = client_patterns( - "/org.matrix.msc2965/auth_issuer$", - unstable=True, - releases=(), - ) - - def __init__(self, hs: "HomeServer"): - super().__init__() - self._config = hs.config - self._auth = hs.get_auth() - - async def on_GET(self, request: SynapseRequest) -> tuple[int, JsonDict]: - # This endpoint is unauthenticated and the response only depends on - # the metadata we get from Matrix Authentication Service. Internally, - # MasDelegatedAuth.issuer() is already caching the - # response in memory anyway. Ideally we would follow any Cache-Control directive - # given by MAS, but this is fine for now. - # - # - `public` means it can be cached both in the browser and in caching proxies - # - `max-age` controls how long we cache on the browser side. 10m is sane enough - # - `s-maxage` controls how long we cache on the proxy side. Since caching - # proxies usually have a way to purge caches, it is fine to cache there for - # longer (1h), and issue cache invalidations in case we need it - # - `stale-while-revalidate` allows caching proxies to serve stale content while - # revalidating in the background. This is useful for making this request always - # 'snappy' to end users whilst still keeping it fresh - request.setHeader( - b"Cache-Control", - b"public, max-age=600, s-maxage=3600, stale-while-revalidate=600", - ) - - if self._config.mas.enabled: - assert isinstance(self._auth, MasDelegatedAuth) - return 200, {"issuer": await self._auth.issuer()} - - else: - # Wouldn't expect this to be reached: the servelet shouldn't have been - # registered. Still, fail gracefully if we are registered for some reason. - raise SynapseError( - 404, - "OIDC discovery has not been configured on this homeserver", - Codes.NOT_FOUND, - ) - - class AuthMetadataServlet(RestServlet): """ Advertises the OAuth 2.0 server metadata for the homeserver. @@ -139,5 +83,4 @@ class AuthMetadataServlet(RestServlet): def register_servlets(hs: "HomeServer", http_server: HttpServer) -> None: if hs.config.mas.enabled: - AuthIssuerServlet(hs).register(http_server) AuthMetadataServlet(hs).register(http_server) diff --git a/synapse/rest/client/capabilities.py b/synapse/rest/client/capabilities.py index 2be5f5849d..4ddaaeda74 100644 --- a/synapse/rest/client/capabilities.py +++ b/synapse/rest/client/capabilities.py @@ -109,6 +109,11 @@ class CapabilitiesRestServlet(RestServlet): "capabilities" ]["m.profile_fields"] + response["capabilities"]["org.matrix.msc4140.delayed_events"] = { + "max_delay_ms": self.config.server.max_event_delay_duration.as_millis(), + "max_scheduled": self.config.server.max_delayed_events_per_user, + } + if self.config.experimental.msc4267_enabled: response["capabilities"]["org.matrix.msc4267.forget_forced_upon_leave"] = { "enabled": self.config.room.forget_on_leave, diff --git a/synapse/rest/client/delayed_events.py b/synapse/rest/client/delayed_events.py index 7afecffe2d..07ca07b7b9 100644 --- a/synapse/rest/client/delayed_events.py +++ b/synapse/rest/client/delayed_events.py @@ -134,6 +134,28 @@ class SendDelayedEventServlet(RestServlet): return 200, {} +class DelayedEventServlet(RestServlet): + PATTERNS = client_patterns( + r"/org\.matrix\.msc4140/delayed_events/(?P[^/]+)$", + releases=(), + ) + CATEGORY = "Delayed event management requests" + + def __init__(self, hs: "HomeServer"): + super().__init__() + self.auth = hs.get_auth() + self.delayed_events_handler = hs.get_delayed_events_handler() + + async def on_GET( + self, request: SynapseRequest, delay_id: str + ) -> tuple[int, JsonDict]: + requester = await self.auth.get_user_by_req(request) + delayed_event = await self.delayed_events_handler.get_for_user( + requester, delay_id + ) + return 200, delayed_event.asdict() + + class DelayedEventsServlet(RestServlet): PATTERNS = client_patterns( r"/org\.matrix\.msc4140/delayed_events$", @@ -150,9 +172,11 @@ class DelayedEventsServlet(RestServlet): requester = await self.auth.get_user_by_req(request) # TODO: Support Pagination stream API ("from" query parameter) delayed_events = await self.delayed_events_handler.get_all_for_user(requester) - - ret = {"delayed_events": delayed_events} - return 200, ret + return 200, { + "delayed_events": [ + delayed_event.asdict() for delayed_event in delayed_events + ] + } def register_servlets(hs: "HomeServer", http_server: HttpServer) -> None: @@ -162,4 +186,5 @@ def register_servlets(hs: "HomeServer", http_server: HttpServer) -> None: CancelDelayedEventServlet(hs).register(http_server) SendDelayedEventServlet(hs).register(http_server) RestartDelayedEventServlet(hs).register(http_server) + DelayedEventServlet(hs).register(http_server) DelayedEventsServlet(hs).register(http_server) diff --git a/synapse/rest/client/devices.py b/synapse/rest/client/devices.py index 3d766cda3f..115ad543fc 100644 --- a/synapse/rest/client/devices.py +++ b/synapse/rest/client/devices.py @@ -77,7 +77,7 @@ class DeleteDevicesRestServlet(RestServlet): key which lists the device_ids to delete. Requires user interactive auth. """ - PATTERNS = client_patterns("/delete_devices") + PATTERNS = client_patterns("/delete_devices$") def __init__(self, hs: "HomeServer"): super().__init__() diff --git a/synapse/rest/client/filter.py b/synapse/rest/client/filter.py index cfe82e1473..6718daaf23 100644 --- a/synapse/rest/client/filter.py +++ b/synapse/rest/client/filter.py @@ -37,7 +37,7 @@ logger = logging.getLogger(__name__) class GetFilterRestServlet(RestServlet): - PATTERNS = client_patterns("/user/(?P[^/]*)/filter/(?P[^/]*)") + PATTERNS = client_patterns("/user/(?P[^/]*)/filter/(?P[^/]*)$") CATEGORY = "Encryption requests" def __init__(self, hs: "HomeServer"): @@ -76,7 +76,7 @@ class GetFilterRestServlet(RestServlet): class CreateFilterRestServlet(RestServlet): - PATTERNS = client_patterns("/user/(?P[^/]*)/filter") + PATTERNS = client_patterns("/user/(?P[^/]*)/filter$") CATEGORY = "Encryption requests" def __init__(self, hs: "HomeServer"): diff --git a/synapse/rest/client/knock.py b/synapse/rest/client/knock.py index cd3afda11e..cce856f0ef 100644 --- a/synapse/rest/client/knock.py +++ b/synapse/rest/client/knock.py @@ -46,7 +46,7 @@ class KnockRoomAliasServlet(RestServlet): POST /knock/{roomIdOrAlias} """ - PATTERNS = client_patterns("/knock/(?P[^/]*)") + PATTERNS = client_patterns("/knock/(?P[^/]*)$") CATEGORY = "Event sending requests" def __init__(self, hs: "HomeServer"): diff --git a/synapse/rest/client/login.py b/synapse/rest/client/login.py index aaf26bac6f..87730fc512 100644 --- a/synapse/rest/client/login.py +++ b/synapse/rest/client/login.py @@ -708,7 +708,7 @@ class SsoRedirectServlet(RestServlet): class CasTicketServlet(RestServlet): - PATTERNS = client_patterns("/login/cas/ticket", v1=True) + PATTERNS = client_patterns("/login/cas/ticket$", v1=True) def __init__(self, hs: "HomeServer"): super().__init__() diff --git a/synapse/rest/client/media.py b/synapse/rest/client/media.py index c740659cdd..c2e30b687b 100644 --- a/synapse/rest/client/media.py +++ b/synapse/rest/client/media.py @@ -36,7 +36,12 @@ from synapse.http.server import ( set_corp_headers, set_cors_headers, ) -from synapse.http.servlet import RestServlet, parse_integer, parse_string +from synapse.http.servlet import ( + RestServlet, + parse_boolean, + parse_integer, + parse_string, +) from synapse.http.site import SynapseRequest from synapse.media._base import ( DEFAULT_MAX_TIMEOUT_MS, @@ -45,7 +50,7 @@ from synapse.media._base import ( ) from synapse.media.media_repository import MediaRepository from synapse.media.media_storage import MediaStorage -from synapse.media.thumbnailer import ThumbnailProvider +from synapse.media.thumbnailer import ANIMATED_THUMBNAIL_TYPE, ThumbnailProvider from synapse.server import HomeServer from synapse.util.stringutils import parse_and_validate_server_name @@ -163,8 +168,9 @@ class ThumbnailResource(RestServlet): width = parse_integer(request, "width", required=True) height = parse_integer(request, "height", required=True) method = parse_string(request, "method", "scale") + animated = parse_boolean(request, "animated", default=False) # TODO Parse the Accept header to get an prioritised list of thumbnail types. - m_type = "image/png" + m_type = ANIMATED_THUMBNAIL_TYPE if animated else "image/png" max_timeout_ms = parse_integer( request, "timeout_ms", default=DEFAULT_MAX_TIMEOUT_MS ) @@ -181,6 +187,7 @@ class ThumbnailResource(RestServlet): m_type, max_timeout_ms, False, + animated=animated, ) else: await self.thumbnailer.respond_local_thumbnail( @@ -204,23 +211,33 @@ class ThumbnailResource(RestServlet): return ip_address = request.getClientAddress().host - remote_resp_function = ( - self.thumbnailer.select_or_generate_remote_thumbnail - if self.dynamic_thumbnails - else self.thumbnailer.respond_remote_thumbnail - ) - await remote_resp_function( - request, - server_name, - media_id, - width, - height, - method, - m_type, - max_timeout_ms, - ip_address, - True, - ) + if self.dynamic_thumbnails: + await self.thumbnailer.select_or_generate_remote_thumbnail( + request, + server_name, + media_id, + width, + height, + method, + m_type, + max_timeout_ms, + ip_address, + True, + animated=animated, + ) + else: + await self.thumbnailer.respond_remote_thumbnail( + request, + server_name, + media_id, + width, + height, + method, + m_type, + max_timeout_ms, + ip_address, + True, + ) self.media_repo.mark_recently_accessed(server_name, media_id) diff --git a/synapse/rest/client/openid.py b/synapse/rest/client/openid.py index e624a48ce7..b6e2c078d8 100644 --- a/synapse/rest/client/openid.py +++ b/synapse/rest/client/openid.py @@ -67,7 +67,7 @@ class IdTokenServlet(RestServlet): } """ - PATTERNS = client_patterns("/user/(?P[^/]*)/openid/request_token") + PATTERNS = client_patterns("/user/(?P[^/]*)/openid/request_token$") EXPIRES_MS = 3600 * 1000 diff --git a/synapse/rest/client/presence.py b/synapse/rest/client/presence.py index de3ffdaa0b..4565280113 100644 --- a/synapse/rest/client/presence.py +++ b/synapse/rest/client/presence.py @@ -40,7 +40,7 @@ logger = logging.getLogger(__name__) class PresenceStatusRestServlet(RestServlet): - PATTERNS = client_patterns("/presence/(?P[^/]*)/status", v1=True) + PATTERNS = client_patterns("/presence/(?P[^/]*)/status$", v1=True) CATEGORY = "Presence requests" def __init__(self, hs: "HomeServer"): diff --git a/synapse/rest/client/profile.py b/synapse/rest/client/profile.py index c2ec5b3611..4431aa2b2c 100644 --- a/synapse/rest/client/profile.py +++ b/synapse/rest/client/profile.py @@ -58,7 +58,7 @@ def _read_propagate(hs: "HomeServer", request: SynapseRequest) -> bool: class ProfileRestServlet(RestServlet): - PATTERNS = client_patterns("/profile/(?P[^/]*)", v1=True) + PATTERNS = client_patterns("/profile/(?P[^/]*)$", v1=True) CATEGORY = "Event sending requests" def __init__(self, hs: "HomeServer"): @@ -92,13 +92,13 @@ class ProfileRestServlet(RestServlet): class ProfileFieldRestServlet(RestServlet): PATTERNS = [ *client_patterns( - "/profile/(?P[^/]*)/(?Pdisplayname)", v1=True + "/profile/(?P[^/]*)/(?Pdisplayname)$", v1=True ), *client_patterns( - "/profile/(?P[^/]*)/(?Pavatar_url)", v1=True + "/profile/(?P[^/]*)/(?Pavatar_url)$", v1=True ), re.compile( - r"^/_matrix/client/v3/profile/(?P[^/]*)/(?P[^/]*)" + r"^/_matrix/client/v3/profile/(?P[^/]*)/(?P[^/]*)$", ), ] @@ -112,7 +112,7 @@ class ProfileFieldRestServlet(RestServlet): if hs.config.experimental.msc4133_enabled: self.PATTERNS.append( re.compile( - r"^/_matrix/client/unstable/uk\.tcpip\.msc4133/profile/(?P[^/]*)/(?P[^/]*)" + r"^/_matrix/client/unstable/uk\.tcpip\.msc4133/profile/(?P[^/]*)/(?P[^/]*)$" ) ) @@ -146,7 +146,9 @@ class ProfileFieldRestServlet(RestServlet): await self.profile_handler.check_profile_query_allowed(user, requester_user) if field_name == ProfileFields.DISPLAYNAME: - field_value: JsonValue = await self.profile_handler.get_displayname(user) + field_value: ( + JsonValue | dict[str, JsonValue] + ) = await self.profile_handler.get_displayname(user) elif field_name == ProfileFields.AVATAR_URL: field_value = await self.profile_handler.get_avatar_url(user) else: @@ -204,18 +206,14 @@ class ProfileFieldRestServlet(RestServlet): Codes.USER_ACCOUNT_SUSPENDED, ) - if field_name == ProfileFields.DISPLAYNAME: - await self.profile_handler.set_displayname( - user, requester, new_value, by_admin=is_admin, propagate=propagate - ) - elif field_name == ProfileFields.AVATAR_URL: - await self.profile_handler.set_avatar_url( - user, requester, new_value, by_admin=is_admin, propagate=propagate - ) - else: - await self.profile_handler.set_profile_field( - user, requester, field_name, new_value, by_admin=is_admin - ) + await self.profile_handler.dispatch_set_profile_field( + target_user=user, + requester=requester, + field_name=field_name, + new_value=new_value, + by_admin=is_admin, + propagate=propagate, + ) return 200, {} @@ -261,17 +259,21 @@ class ProfileFieldRestServlet(RestServlet): Codes.USER_ACCOUNT_SUSPENDED, ) - if field_name == ProfileFields.DISPLAYNAME: - await self.profile_handler.set_displayname( - user, requester, "", by_admin=is_admin, propagate=propagate - ) - elif field_name == ProfileFields.AVATAR_URL: - await self.profile_handler.set_avatar_url( - user, requester, "", by_admin=is_admin, propagate=propagate + if field_name in (ProfileFields.DISPLAYNAME, ProfileFields.AVATAR_URL): + await self.profile_handler.dispatch_set_profile_field( + target_user=user, + requester=requester, + field_name=field_name, + new_value="", + by_admin=is_admin, + propagate=propagate, ) else: - await self.profile_handler.delete_profile_field( - user, requester, field_name, by_admin=is_admin + await self.profile_handler.dispatch_delete_profile_field( + target_user=user, + requester=requester, + field_name=field_name, + by_admin=is_admin, ) return 200, {} @@ -284,8 +286,9 @@ class UnstableProfileFieldRestServlet(ProfileFieldRestServlet): def register_servlets(hs: "HomeServer", http_server: HttpServer) -> None: - # The specific field endpoint *must* appear before the generic profile endpoint. ProfileFieldRestServlet(hs).register(http_server) - ProfileRestServlet(hs).register(http_server) + if hs.config.experimental.msc4133_enabled: UnstableProfileFieldRestServlet(hs).register(http_server) + + ProfileRestServlet(hs).register(http_server) diff --git a/synapse/rest/client/read_marker.py b/synapse/rest/client/read_marker.py index 874e7487bf..8e0f2a2e7a 100644 --- a/synapse/rest/client/read_marker.py +++ b/synapse/rest/client/read_marker.py @@ -23,6 +23,7 @@ import logging from typing import TYPE_CHECKING from synapse.api.constants import ReceiptTypes +from synapse.api.errors import Codes, SynapseError from synapse.http.server import HttpServer from synapse.http.servlet import RestServlet, parse_json_object_from_request from synapse.http.site import SynapseRequest @@ -66,6 +67,21 @@ class ReadMarkerRestServlet(RestServlet): body = parse_json_object_from_request(request) unrecognized_types = set(body.keys()) - self._known_receipt_types + + if self.config.experimental.msc4446_enabled: + allow_backward = body.get("com.beeper.allow_backward", False) + if not isinstance(allow_backward, bool): + raise SynapseError( + 400, + "com.beeper.allow_backward must be a boolean.", + Codes.INVALID_PARAM, + ) + + # Prevent considering the `allow_backward` field as a receipt type. + unrecognized_types -= {"com.beeper.allow_backward"} + else: + allow_backward = False + if unrecognized_types: # It's fine if there are unrecognized receipt types, but let's log # it to help debug clients that have typoed the receipt type. @@ -86,6 +102,7 @@ class ReadMarkerRestServlet(RestServlet): room_id, user_id=requester.user.to_string(), event_id=event_id, + allow_backward=allow_backward, ) else: await self.receipts_handler.received_client_receipt( diff --git a/synapse/rest/client/receipts.py b/synapse/rest/client/receipts.py index d3a43537bb..949a1e64ad 100644 --- a/synapse/rest/client/receipts.py +++ b/synapse/rest/client/receipts.py @@ -20,6 +20,7 @@ # import logging +from http import HTTPStatus from typing import TYPE_CHECKING from synapse.api.constants import MAIN_TIMELINE, ReceiptTypes @@ -50,6 +51,7 @@ class ReceiptRestServlet(RestServlet): self.read_marker_handler = hs.get_read_marker_handler() self.presence_handler = hs.get_presence_handler() self._main_store = hs.get_datastores().main + self._msc4446_enabled = hs.config.experimental.msc4446_enabled self._known_receipt_types = { ReceiptTypes.READ, @@ -73,6 +75,25 @@ class ReceiptRestServlet(RestServlet): body = parse_json_object_from_request(request) + if self._msc4446_enabled: + allow_backward = body.get("com.beeper.allow_backward", False) + if not isinstance(allow_backward, bool): + raise SynapseError( + HTTPStatus.BAD_REQUEST, + "com.beeper.allow_backward must be a boolean.", + Codes.INVALID_PARAM, + ) + + if allow_backward and receipt_type != ReceiptTypes.FULLY_READ: + raise SynapseError( + HTTPStatus.BAD_REQUEST, + "com.beeper.allow_backward is only allowed to be true for " + f"{ReceiptTypes.FULLY_READ}.", + Codes.INVALID_PARAM, + ) + else: + allow_backward = False + # Pull the thread ID, if one exists. thread_id = None if "thread_id" in body: @@ -108,6 +129,7 @@ class ReceiptRestServlet(RestServlet): room_id, user_id=requester.user.to_string(), event_id=event_id, + allow_backward=allow_backward, ) else: await self.receipts_handler.received_client_receipt( diff --git a/synapse/rest/client/register.py b/synapse/rest/client/register.py index ae81e80654..8355d0c571 100644 --- a/synapse/rest/client/register.py +++ b/synapse/rest/client/register.py @@ -338,7 +338,7 @@ class RegistrationSubmitTokenServlet(RestServlet): class UsernameAvailabilityRestServlet(RestServlet): - PATTERNS = client_patterns("/register/available") + PATTERNS = client_patterns("/register/available$") def __init__(self, hs: "HomeServer"): super().__init__() @@ -401,7 +401,7 @@ class RegistrationTokenValidityRestServlet(RestServlet): """ PATTERNS = client_patterns( - f"/register/{LoginType.REGISTRATION_TOKEN}/validity", + f"/register/{LoginType.REGISTRATION_TOKEN}/validity$", releases=("v1",), ) CATEGORY = "Registration/login requests" diff --git a/synapse/rest/client/relations.py b/synapse/rest/client/relations.py index c913bc6970..daf220545e 100644 --- a/synapse/rest/client/relations.py +++ b/synapse/rest/client/relations.py @@ -94,7 +94,7 @@ class RelationPaginationServlet(RestServlet): class ThreadsServlet(RestServlet): - PATTERNS = (re.compile("^/_matrix/client/v1/rooms/(?P[^/]*)/threads"),) + PATTERNS = (re.compile("^/_matrix/client/v1/rooms/(?P[^/]*)/threads$"),) CATEGORY = "Client API requests" def __init__(self, hs: "HomeServer"): diff --git a/synapse/rest/client/reporting.py b/synapse/rest/client/reporting.py index 0c594b9f3f..c99d982acc 100644 --- a/synapse/rest/client/reporting.py +++ b/synapse/rest/client/reporting.py @@ -131,8 +131,7 @@ class ReportRoomRestServlet(RestServlet): super().__init__() self.hs = hs self.auth = hs.get_auth() - self.clock = hs.get_clock() - self.store = hs.get_datastores().main + self.reports_handler = hs.get_reports_handler() class PostBody(RequestBodyModel): reason: StrictStr @@ -141,25 +140,15 @@ class ReportRoomRestServlet(RestServlet): self, request: SynapseRequest, room_id: str ) -> tuple[int, JsonDict]: requester = await self.auth.get_user_by_req(request) - user_id = requester.user.to_string() - body = parse_and_validate_json_object_from_request(request, self.PostBody) - room = await self.store.get_room(room_id) - if room is None: - if self.hs.config.experimental.msc4277_enabled: - # Respond with 200 and no content regardless of whether the room - # exists to prevent enumeration attacks. - return 200, {} - else: - raise NotFoundError("Room does not exist") - - await self.store.add_room_report( - room_id=room_id, - user_id=user_id, - reason=body.reason, - received_ts=self.clock.time_msec(), - ) + try: + await self.reports_handler.report_room(requester, room_id, body.reason) + except NotFoundError: + if not self.hs.config.experimental.msc4277_enabled: + raise + # Respond with 200 and no content regardless of whether the room + # exists to prevent enumeration attacks. return 200, {} diff --git a/synapse/rest/client/room.py b/synapse/rest/client/room.py index 36f638e236..1b8774fde3 100644 --- a/synapse/rest/client/room.py +++ b/synapse/rest/client/room.py @@ -84,6 +84,7 @@ from synapse.types import JsonDict, Requester, StreamToken, ThirdPartyInstanceID from synapse.types.state import StateFilter from synapse.util.cancellation import cancellable from synapse.util.clock import Clock +from synapse.util.duration import Duration from synapse.util.events import generate_fake_event_id from synapse.util.stringutils import parse_and_validate_server_name @@ -215,7 +216,6 @@ class RoomStateEventRestServlet(RestServlet): self.auth = hs.get_auth() self.clock = hs.get_clock() self._event_serializer = hs.get_event_client_serializer() - self._max_event_delay_ms = hs.config.server.max_event_delay_ms self._spam_checker_module_callbacks = hs.get_module_api_callbacks().spam_checker self._msc4354_enabled = hs.config.experimental.msc4354_enabled @@ -343,7 +343,7 @@ class RoomStateEventRestServlet(RestServlet): if self._msc4354_enabled: sticky_duration_ms = parse_integer(request, StickyEvent.QUERY_PARAM_NAME) - delay = _parse_request_delay(request, self._max_event_delay_ms) + delay = _parse_request_for_delayed_event_delay(request) if delay is not None: delay_id = await self.delayed_events_handler.add( requester, @@ -416,7 +416,6 @@ class RoomSendEventRestServlet(TransactionRestServlet): self.event_creation_handler = hs.get_event_creation_handler() self.delayed_events_handler = hs.get_delayed_events_handler() self.auth = hs.get_auth() - self._max_event_delay_ms = hs.config.server.max_event_delay_ms self._msc4354_enabled = hs.config.experimental.msc4354_enabled def register(self, http_server: HttpServer) -> None: @@ -442,7 +441,7 @@ class RoomSendEventRestServlet(TransactionRestServlet): if self._msc4354_enabled: sticky_duration_ms = parse_integer(request, StickyEvent.QUERY_PARAM_NAME) - delay = _parse_request_delay(request, self._max_event_delay_ms) + delay = _parse_request_for_delayed_event_delay(request) if delay is not None: delay_id = await self.delayed_events_handler.add( requester, @@ -515,47 +514,20 @@ class RoomSendEventRestServlet(TransactionRestServlet): ) -def _parse_request_delay( - request: SynapseRequest, - max_delay: int | None, -) -> int | None: +def _parse_request_for_delayed_event_delay(request: SynapseRequest) -> Duration | None: """Parses from the request string the delay parameter for delayed event requests, and checks it for correctness. Args: request: the twisted HTTP request. - max_delay: the maximum allowed value of the delay parameter, - or None if no delay parameter is allowed. Returns: The value of the requested delay, or None if it was absent. Raises: - SynapseError: if the delay parameter is present and forbidden, - or if it exceeds the maximum allowed value. + SynapseError: if the delay parameter is present and invalid. """ - delay = parse_integer(request, "org.matrix.msc4140.delay") - if delay is None: - return None - if max_delay is None: - raise SynapseError( - HTTPStatus.BAD_REQUEST, - "Delayed events are not supported on this server", - Codes.UNKNOWN, - { - "org.matrix.msc4140.errcode": "M_MAX_DELAY_UNSUPPORTED", - }, - ) - if delay > max_delay: - raise SynapseError( - HTTPStatus.BAD_REQUEST, - "The requested delay exceeds the allowed maximum.", - Codes.UNKNOWN, - { - "org.matrix.msc4140.errcode": "M_MAX_DELAY_EXCEEDED", - "org.matrix.msc4140.max_delay": max_delay, - }, - ) - return delay + delay_ms = parse_integer(request, "org.matrix.msc4140.delay") + return Duration(milliseconds=delay_ms) if delay_ms is not None else None # TODO: Needs unit testing for room ID + alias joins @@ -1440,6 +1412,7 @@ class RoomRedactEventRestServlet(TransactionRestServlet): event_id=event_id, initial_redaction_event=event, relation_types=with_relations, + room_version=room_version, ) event_id = event.event_id @@ -1533,7 +1506,7 @@ class RoomAliasListServlet(RestServlet): PATTERNS = [ re.compile( r"^/_matrix/client/unstable/org\.matrix\.msc2432" - r"/rooms/(?P[^/]*)/aliases" + r"/rooms/(?P[^/]*)/aliases$" ), ] + list(client_patterns("/rooms/(?P[^/]*)/aliases$", unstable=False)) CATEGORY = "Client API requests" diff --git a/synapse/rest/client/room_membership.py b/synapse/rest/client/room_membership.py new file mode 100644 index 0000000000..026bee8214 --- /dev/null +++ b/synapse/rest/client/room_membership.py @@ -0,0 +1,103 @@ +# +# This file is licensed under the Affero General Public License (AGPL) version 3. +# +# Copyright (C) 2026 Element Creations Ltd +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# See the GNU Affero General Public License for more details: +# . +# + +import logging +from http import HTTPStatus +from typing import TYPE_CHECKING + +from synapse.api.constants import EventTypes, Membership +from synapse.api.errors import Codes, SynapseError +from synapse.appservice import Scopes +from synapse.http.server import HttpServer +from synapse.http.servlet import RestServlet, parse_string +from synapse.http.site import SynapseRequest +from synapse.rest.client._base import client_patterns +from synapse.types import JsonDict, RoomID, UserID +from synapse.util.stringutils import parse_and_validate_server_name + +if TYPE_CHECKING: + from synapse.server import HomeServer + +logger = logging.getLogger(__name__) + + +class AppserviceRoomMembershipRestServlet(RestServlet): + PATTERNS = client_patterns( + r"/io\.element\.msc4502/rooms/(?P[^/]*)/is_joined$", releases=() + ) + CATEGORY = "Client API requests" + + def __init__(self, hs: "HomeServer"): + super().__init__() + self.auth = hs.get_auth() + self.store = hs.get_datastores().main + self.storage_controllers = hs.get_storage_controllers() + self.is_mine_id = hs.is_mine_id + + async def on_GET( + self, request: SynapseRequest, room_id: str + ) -> tuple[int, JsonDict]: + requester = await self.auth.get_user_by_req(request, allow_guest=False) + self.auth.assert_requester_has_scope(requester, Scopes.QUERY_ROOM_MEMBERSHIP) + + if not RoomID.is_valid(room_id): + raise SynapseError( + HTTPStatus.BAD_REQUEST, "Invalid room ID", Codes.INVALID_PARAM + ) + + mxid = parse_string(request, "mxid") + server_name = parse_string(request, "server_name") + + if (mxid is None) == (server_name is None): + raise SynapseError( + HTTPStatus.BAD_REQUEST, + "Exactly one of 'mxid' or 'server_name' query parameters must be given", + Codes.MISSING_PARAM, + ) + + if mxid is not None: + if not UserID.is_valid(mxid): + raise SynapseError( + HTTPStatus.BAD_REQUEST, + f"Invalid MXID: {mxid}", + Codes.INVALID_PARAM, + ) + if self.is_mine_id(mxid): + joined = await self.store.check_local_user_in_room(mxid, room_id) + else: + event = await self.storage_controllers.state.get_current_state_event( + room_id, EventTypes.Member, mxid + ) + joined = ( + event is not None + and event.content.get("membership") == Membership.JOIN + ) + else: + assert server_name is not None + try: + parse_and_validate_server_name(server_name) + except ValueError: + raise SynapseError( + HTTPStatus.BAD_REQUEST, + f"Invalid server name: {server_name}", + Codes.INVALID_PARAM, + ) + joined = await self.store.is_host_joined(room_id, server_name) + + return HTTPStatus.OK, {"joined": joined} + + +def register_servlets(hs: "HomeServer", http_server: HttpServer) -> None: + if hs.config.experimental.msc4502_enabled: + AppserviceRoomMembershipRestServlet(hs).register(http_server) diff --git a/synapse/rest/client/sync.py b/synapse/rest/client/sync.py index 962317dedb..08002a6708 100644 --- a/synapse/rest/client/sync.py +++ b/synapse/rest/client/sync.py @@ -55,7 +55,13 @@ from synapse.http.servlet import ( from synapse.http.site import SynapseRequest from synapse.logging.opentracing import log_kv, set_tag, trace_with_opname from synapse.rest.admin.experimental_features import ExperimentalFeature -from synapse.types import JsonDict, Requester, SlidingSyncStreamToken, StreamToken +from synapse.types import ( + JsonDict, + JsonMapping, + Requester, + SlidingSyncStreamToken, + StreamToken, +) from synapse.types.rest.client import SlidingSyncBody from synapse.util.caches.lrucache import LruCache from synapse.util.cancellation import cancellable @@ -123,6 +129,9 @@ class SyncRestServlet(RestServlet): self._event_serializer = hs.get_event_client_serializer() self._msc2654_enabled = hs.config.experimental.msc2654_enabled self._msc3773_enabled = hs.config.experimental.msc3773_enabled + self._include_profile_updates_in_sync = ( + hs.config.server.include_profile_updates_in_sync + ) self._json_filter_cache: LruCache[str, bool] = LruCache( max_size=1000, @@ -351,6 +360,15 @@ class SyncRestServlet(RestServlet): if sync_result.to_device: response["to_device"] = {"events": sync_result.to_device} + if self._include_profile_updates_in_sync and sync_result.profile_updates: + # FIXME: See issue https://github.com/element-hq/synapse/issues/19981 + # for concerns around the current implementation of the profile + # updates stream. + response["org.matrix.msc4429.users"] = { + user_id: {"profile_updates": updates} + for user_id, updates in sync_result.profile_updates.items() + } + if sync_result.device_lists.changed: response["device_lists"]["changed"] = list(sync_result.device_lists.changed) if sync_result.device_lists.left: @@ -1132,8 +1150,33 @@ class SlidingSyncRestServlet(RestServlet): requester, extensions.sticky_events, ref_rooms_results ) + if extensions.profiles: + serialized_extensions[ + "org.matrix.msc4262.profiles" + ] = await self._serialise_profiles( + extensions.profiles, + ) + return serialized_extensions + async def _serialise_profiles( + self, + profiles: SlidingSyncResult.Extensions.ProfilesExtension, + ) -> JsonMapping: + """ + Serialise the profiles extension response. + + Args: + profiles: The generated profiles response object. + + Returns: + A dictionary containing the response `users` with the + generated profile updates. + """ + return { + "users": profiles.users, + } + async def _serialise_sticky_events( self, requester: Requester, diff --git a/synapse/rest/client/tags.py b/synapse/rest/client/tags.py index 5699ff35c7..f7272754e6 100644 --- a/synapse/rest/client/tags.py +++ b/synapse/rest/client/tags.py @@ -73,7 +73,7 @@ class TagServlet(RestServlet): """ PATTERNS = client_patterns( - "/user/(?P[^/]*)/rooms/(?P[^/]*)/tags/(?P[^/]*)" + "/user/(?P[^/]*)/rooms/(?P[^/]*)/tags/(?P[^/]*)$" ) CATEGORY = "Account data requests" diff --git a/synapse/rest/client/thirdparty.py b/synapse/rest/client/thirdparty.py index c17335eb48..b4be564eb0 100644 --- a/synapse/rest/client/thirdparty.py +++ b/synapse/rest/client/thirdparty.py @@ -37,7 +37,7 @@ logger = logging.getLogger(__name__) class ThirdPartyProtocolsServlet(RestServlet): - PATTERNS = client_patterns("/thirdparty/protocols") + PATTERNS = client_patterns("/thirdparty/protocols$") def __init__(self, hs: "HomeServer"): super().__init__() diff --git a/synapse/rest/client/tokenrefresh.py b/synapse/rest/client/tokenrefresh.py index 2b4f7f8953..873c6f3e4e 100644 --- a/synapse/rest/client/tokenrefresh.py +++ b/synapse/rest/client/tokenrefresh.py @@ -39,7 +39,7 @@ class TokenRefreshRestServlet(RestServlet): token. """ - PATTERNS = client_patterns("/tokenrefresh") + PATTERNS = client_patterns("/tokenrefresh$") def __init__(self, hs: "HomeServer"): super().__init__() diff --git a/synapse/rest/media/create_resource.py b/synapse/rest/media/create_resource.py index 1b6b001b45..c962d5f624 100644 --- a/synapse/rest/media/create_resource.py +++ b/synapse/rest/media/create_resource.py @@ -37,7 +37,7 @@ logger = logging.getLogger(__name__) class CreateResource(RestServlet): - PATTERNS = [re.compile("/_matrix/media/v1/create")] + PATTERNS = [re.compile("/_matrix/media/v1/create$")] def __init__(self, hs: "HomeServer", media_repo: "MediaRepository"): super().__init__() diff --git a/synapse/rest/media/thumbnail_resource.py b/synapse/rest/media/thumbnail_resource.py index 536fea4c32..35afb7ab54 100644 --- a/synapse/rest/media/thumbnail_resource.py +++ b/synapse/rest/media/thumbnail_resource.py @@ -25,7 +25,12 @@ import re from typing import TYPE_CHECKING from synapse.http.server import set_corp_headers, set_cors_headers -from synapse.http.servlet import RestServlet, parse_integer, parse_string +from synapse.http.servlet import ( + RestServlet, + parse_boolean, + parse_integer, + parse_string, +) from synapse.http.site import SynapseRequest from synapse.media._base import ( DEFAULT_MAX_TIMEOUT_MS, @@ -33,7 +38,7 @@ from synapse.media._base import ( respond_404, ) from synapse.media.media_storage import MediaStorage -from synapse.media.thumbnailer import ThumbnailProvider +from synapse.media.thumbnailer import ANIMATED_THUMBNAIL_TYPE, ThumbnailProvider from synapse.util.stringutils import parse_and_validate_server_name if TYPE_CHECKING: @@ -78,8 +83,9 @@ class ThumbnailResource(RestServlet): width = parse_integer(request, "width", required=True) height = parse_integer(request, "height", required=True) method = parse_string(request, "method", "scale") + animated = parse_boolean(request, "animated", default=False) # TODO Parse the Accept header to get an prioritised list of thumbnail types. - m_type = "image/png" + m_type = ANIMATED_THUMBNAIL_TYPE if animated else "image/png" max_timeout_ms = parse_integer( request, "timeout_ms", default=DEFAULT_MAX_TIMEOUT_MS ) @@ -97,6 +103,7 @@ class ThumbnailResource(RestServlet): max_timeout_ms, False, allow_authenticated=False, + animated=animated, ) else: await self.thumbnail_provider.respond_local_thumbnail( @@ -121,22 +128,33 @@ class ThumbnailResource(RestServlet): return ip_address = request.getClientAddress().host - remote_resp_function = ( - self.thumbnail_provider.select_or_generate_remote_thumbnail - if self.dynamic_thumbnails - else self.thumbnail_provider.respond_remote_thumbnail - ) - await remote_resp_function( - request, - server_name, - media_id, - width, - height, - method, - m_type, - max_timeout_ms, - ip_address, - use_federation=False, - allow_authenticated=False, - ) + if self.dynamic_thumbnails: + await self.thumbnail_provider.select_or_generate_remote_thumbnail( + request, + server_name, + media_id, + width, + height, + method, + m_type, + max_timeout_ms, + ip_address, + use_federation=False, + allow_authenticated=False, + animated=animated, + ) + else: + await self.thumbnail_provider.respond_remote_thumbnail( + request, + server_name, + media_id, + width, + height, + method, + m_type, + max_timeout_ms, + ip_address, + use_federation=False, + allow_authenticated=False, + ) self.media_repo.mark_recently_accessed(server_name, media_id) diff --git a/synapse/rest/synapse/client/__init__.py b/synapse/rest/synapse/client/__init__.py index e04b84ecac..3534d59120 100644 --- a/synapse/rest/synapse/client/__init__.py +++ b/synapse/rest/synapse/client/__init__.py @@ -3,6 +3,7 @@ # # Copyright 2021 The Matrix.org Foundation C.I.C. # Copyright (C) 2023 New Vector, Ltd +# Copyright (C) 2026 Element Creations Ltd # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License as @@ -24,6 +25,10 @@ from typing import TYPE_CHECKING, Mapping from twisted.web.resource import Resource from synapse.rest.synapse.client.federation_whitelist import FederationWhitelistResource +from synapse.rest.synapse.client.media_upload_limit_exceeded import ( + MEDIA_UPLOAD_LIMIT_EXCEEDED_PATH, + MediaUploadLimitExceededResource, +) from synapse.rest.synapse.client.new_user_consent import NewUserConsentResource from synapse.rest.synapse.client.pick_idp import PickIdpResource from synapse.rest.synapse.client.pick_username import pick_username_resource @@ -54,6 +59,12 @@ def build_synapse_client_resource_tree(hs: "HomeServer") -> Mapping[str, Resourc "/_synapse/client/sso_register": SsoRegisterResource(hs), # Unsubscribe to notification emails link "/_synapse/client/unsubscribe": UnsubscribeResource(hs), + # Fallback page served as the `info_uri` for media upload limits that + # don't have an explicit `info_uri`. Mounted unconditionally: in a + # worker deployment, the media repo (which generates the errors) + # typically runs on a different worker than the one serving + # `/_synapse/client`. + MEDIA_UPLOAD_LIMIT_EXCEEDED_PATH: MediaUploadLimitExceededResource(hs), } if hs.config.mas.enabled: diff --git a/synapse/rest/synapse/client/media_upload_limit_exceeded.py b/synapse/rest/synapse/client/media_upload_limit_exceeded.py new file mode 100644 index 0000000000..48acdd2fd6 --- /dev/null +++ b/synapse/rest/synapse/client/media_upload_limit_exceeded.py @@ -0,0 +1,42 @@ +# +# This file is licensed under the Affero General Public License (AGPL) version 3. +# +# Copyright (C) 2026 Element Creations Ltd +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# See the GNU Affero General Public License for more details: +# . +# + +from typing import TYPE_CHECKING + +from synapse.http.server import DirectServeHtmlResource, respond_with_html +from synapse.http.site import SynapseRequest + +if TYPE_CHECKING: + from synapse.server import HomeServer + +# The path at which the fallback media upload limit exceeded info page is served. +# This is used as the `info_uri` returned in the `M_USER_LIMIT_EXCEEDED` error +# when no `info_uri` has been specified for a media upload limit. +MEDIA_UPLOAD_LIMIT_EXCEEDED_PATH = "/_synapse/client/media_upload_limit_exceeded" + + +class MediaUploadLimitExceededResource(DirectServeHtmlResource): + """Serves a static fallback page explaining that a media upload limit has + been exceeded. + + This is used as the `info_uri` for the `M_USER_LIMIT_EXCEEDED` error when no + `info_uri` has been specified for a media upload limit. + """ + + def __init__(self, hs: "HomeServer"): + super().__init__(clock=hs.get_clock()) + self._template = hs.config.media.media_upload_limit_exceeded_template + + async def _async_render_GET(self, request: SynapseRequest) -> None: + respond_with_html(request, 200, self._template.render()) diff --git a/synapse/rest/synapse/mas/users.py b/synapse/rest/synapse/mas/users.py index 01db41bcfa..cc5717b103 100644 --- a/synapse/rest/synapse/mas/users.py +++ b/synapse/rest/synapse/mas/users.py @@ -19,6 +19,7 @@ from typing import TYPE_CHECKING, Any, TypedDict from pydantic import StrictBool, StrictStr, model_validator +from synapse.api.constants import ProfileFields from synapse.api.errors import NotFoundError, SynapseError from synapse.http.servlet import ( parse_and_validate_json_object_from_request, @@ -162,18 +163,18 @@ class MasProvisionUserResource(MasBaseResource): ) else: created = False + new_displayname = None if body.unset_displayname: - await self.profile_handler.set_displayname( - target_user=user_id, - requester=requester, - new_displayname="", - by_admin=True, - ) + new_displayname = "" elif body.set_displayname is not None: - await self.profile_handler.set_displayname( + new_displayname = body.set_displayname + + if new_displayname is not None: + await self.profile_handler.dispatch_set_profile_field( target_user=user_id, requester=requester, - new_displayname=body.set_displayname, + field_name=ProfileFields.DISPLAYNAME, + new_value=new_displayname, by_admin=True, ) @@ -221,18 +222,18 @@ class MasProvisionUserResource(MasBaseResource): if body.locked is not None: await self.store.set_user_locked_status(user_id.to_string(), body.locked) + new_avatar_url_value = None if body.unset_avatar_url: - await self.profile_handler.set_avatar_url( - target_user=user_id, - requester=requester, - new_avatar_url="", - by_admin=True, - ) + new_avatar_url_value = "" elif body.set_avatar_url is not None: - await self.profile_handler.set_avatar_url( + new_avatar_url_value = body.set_avatar_url + + if new_avatar_url_value is not None: + await self.profile_handler.dispatch_set_profile_field( target_user=user_id, requester=requester, - new_avatar_url=body.set_avatar_url, + field_name=ProfileFields.AVATAR_URL, + new_value=new_avatar_url_value, by_admin=True, ) @@ -380,10 +381,11 @@ class MasSetDisplayNameResource(MasBaseResource): requester = create_requester(user_id=user_id) - await self.profile_handler.set_displayname( + await self.profile_handler.dispatch_set_profile_field( target_user=requester.user, requester=requester, - new_displayname=body.displayname, + field_name=ProfileFields.DISPLAYNAME, + new_value=body.displayname, by_admin=True, ) @@ -424,10 +426,11 @@ class MasUnsetDisplayNameResource(MasBaseResource): requester = create_requester(user_id=user_id) - await self.profile_handler.set_displayname( + await self.profile_handler.dispatch_set_profile_field( target_user=requester.user, requester=requester, - new_displayname="", + field_name=ProfileFields.DISPLAYNAME, + new_value="", by_admin=True, ) diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index 8eeea20967..1df7f70b71 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -218,12 +218,18 @@ class SQLBaseStore(metaclass=ABCMeta): self.external_cached_functions[cache_name] = func -def db_to_json(db_content: memoryview | bytes | bytearray | str) -> Any: +def db_to_json( + db_content: memoryview | bytes | bytearray | str | dict[str, Any] | list[Any], +) -> Any: """ Take some data from a database row and return a JSON-decoded object. Args: db_content: The JSON-encoded contents from the database. + Supports TEXT columns, as well as JSON/JSONB columns containing lists or objects. + Note that psycopg will decode JSON/JSONB automatically but SQLite doesn't have + such a data type (and returns the text verbatim), so this function can help + paper over the difference. Returns: The object decoded from JSON. @@ -238,6 +244,13 @@ def db_to_json(db_content: memoryview | bytes | bytearray | str) -> Any: if isinstance(db_content, (bytes, bytearray)): db_content = db_content.decode("utf8") + if isinstance(db_content, (dict, list)): + # psycopg2 has already decoded this JSON or JSONB value + # Maybe we should be splitting this case out to a separate helper where + # we expect JSON/JSONB columns and switch behaviour based on + # the database driver + return db_content + try: return json_decoder.decode(db_content) except Exception: diff --git a/synapse/storage/background_updates.py b/synapse/storage/background_updates.py index 311534c5e7..8137b02036 100644 --- a/synapse/storage/background_updates.py +++ b/synapse/storage/background_updates.py @@ -262,6 +262,10 @@ class BackgroundUpdater: # enum? self._all_done = False + # A set of background updates that we have queried the database for and + # found to be completed. + self._completed_background_updates: set[str] = set() + # Whether we're currently running updates self._running = False @@ -394,9 +398,15 @@ class BackgroundUpdater: return perf def start_doing_background_updates(self) -> None: + """Start doing background updates in the background. + + This gets called both on startup and when the admin API is used to + reschedule background updates. + """ if self.enabled: # if we start a new background update, not all updates are done. self._all_done = False + self._completed_background_updates.clear() sleep = self.sleep_enabled self.hs.run_as_background_process( "background_updates", @@ -478,6 +488,9 @@ class BackgroundUpdater: if update_name == self._current_background_update: return False + if update_name in self._completed_background_updates: + return True + update_exists = await self.db_pool.simple_select_one_onecol( "background_updates", keyvalues={"update_name": update_name}, @@ -486,6 +499,9 @@ class BackgroundUpdater: allow_none=True, ) + if not update_exists: + self._completed_background_updates.add(update_name) + return not update_exists async def have_completed_background_updates( diff --git a/synapse/storage/database.py b/synapse/storage/database.py index 023014276b..469639fbf3 100644 --- a/synapse/storage/database.py +++ b/synapse/storage/database.py @@ -120,6 +120,7 @@ UNIQUE_INDEX_BACKGROUND_UPDATES = { "event_push_summary": "event_push_summary_unique_index2", "receipts_linearized": "receipts_linearized_unique_index", "receipts_graph": "receipts_graph_unique_index", + "e2e_cross_signing_signatures": "e2e_cross_signing_signatures_add_key_id_to_index", } @@ -2636,6 +2637,9 @@ def make_in_list_sql_clause( using the `ANY` form on postgres means that it views queries with different length iterables as the same, helping the query stats. + An empty `iterable` yields a constant clause: nothing can be a member of an empty + list, so the clause is `FALSE` (or `TRUE` when `negative`). + Args: database_engine column: Name of the column @@ -2646,6 +2650,11 @@ def make_in_list_sql_clause( A tuple of SQL query and the args """ + if not iterable: + # Spell the empty case out rather than relying on each engine's handling of an + # empty list: sqlite permits `IN ()` but postgres does not. + return ("TRUE" if negative else "FALSE"), [] + if database_engine.supports_using_any_list: # This should hopefully be faster, but also makes postgres query # stats easier to understand. diff --git a/synapse/storage/databases/main/cache.py b/synapse/storage/databases/main/cache.py index a4530796f2..45fe605d8b 100644 --- a/synapse/storage/databases/main/cache.py +++ b/synapse/storage/databases/main/cache.py @@ -70,6 +70,10 @@ GET_E2E_CROSS_SIGNING_SIGNATURES_FOR_DEVICE_CACHE_NAME = ( "_get_e2e_cross_signing_signatures_for_device" ) +# As above: this cache takes a single argument which is itself a tuple, which +# requires special handling. +GET_SERVER_KEYS_JSON_CACHE_NAME = "_get_server_keys_json" + # How long between cache invalidation table cleanups, once we have caught up # with the backlog. REGULAR_CLEANUP_INTERVAL = Duration(hours=1) @@ -305,6 +309,24 @@ class CacheInvalidationWorkerStore(SQLBaseStore): self._get_e2e_cross_signing_signatures_for_device.invalidate( # type: ignore[attr-defined] ((user_id, device_id),) ) + elif row.cache_func == GET_SERVER_KEYS_JSON_CACHE_NAME: + # As above: each entry in "keys" is a JSON-encoded + # (server_name, key_id) pair, since the cache takes a single + # argument which is itself a tuple and we cannot send nested + # information over replication. + for json_str in row.keys: + try: + server_name, key_id = json.loads(json_str) + except (json.JSONDecodeError, TypeError, ValueError): + logger.error( + "Failed to deserialise cache key as valid JSON: %s", + json_str, + ) + continue + + self._get_server_keys_json.invalidate( # type: ignore[attr-defined] + ((server_name, key_id),) + ) else: self._attempt_to_invalidate_cache(row.cache_func, row.keys) diff --git a/synapse/storage/databases/main/delayed_events.py b/synapse/storage/databases/main/delayed_events.py index 1727f589e2..35f78e3f97 100644 --- a/synapse/storage/databases/main/delayed_events.py +++ b/synapse/storage/databases/main/delayed_events.py @@ -17,7 +17,7 @@ from typing import TYPE_CHECKING, NewType import attr -from synapse.api.errors import NotFoundError +from synapse.api.errors import LimitExceededError, NotFoundError from synapse.storage._base import SQLBaseStore, db_to_json from synapse.storage.database import ( DatabasePool, @@ -28,6 +28,7 @@ from synapse.storage.database import ( from synapse.storage.engines import PostgresEngine from synapse.types import JsonDict, RoomID from synapse.util import stringutils +from synapse.util.duration import Duration from synapse.util.json import json_encoder if TYPE_CHECKING: @@ -63,6 +64,33 @@ class DelayedEventDetails(EventDetails): user_localpart: UserLocalpart +@attr.s(slots=True, frozen=True, auto_attribs=True) +class DelayedEventResponse: + """The representation of a delayed event in API format.""" + + delay_id: str + room_id: str + type: str + state_key: str | None + delay_ms: int + delayed_since_ts: int + content: JsonDict = attr.ib(converter=db_to_json) + + def asdict(self) -> JsonDict: + return attr.asdict(self, filter=lambda _attr, v: v is not None) + + +# TODO: Remove this class once the response format is stable +class DelayedEventResponseLegacyCompat(DelayedEventResponse): + """For backwards compatibility with field names from earlier revisions of MSC4140.""" + + def asdict(self) -> JsonDict: + return super().asdict() | { + "delay": self.delay_ms, + "running_since": self.delayed_since_ts, + } + + class DelayedEventsStore(SQLBaseStore): def __init__( self, @@ -122,20 +150,84 @@ class DelayedEventsStore(SQLBaseStore): state_key: str | None, origin_server_ts: int | None, content: JsonDict, - delay: int, + delay: Duration, sticky_duration_ms: int | None, + limit: int, ) -> tuple[DelayID, Timestamp]: """ Inserts a new delayed event in the DB. + Args: + user_localpart: The localpart of the requester of the delayed event, who will be its owner. + device_id: The device ID of the requester. + creation_ts: The timestamp of when the request to add the delayed event was made. + room_id: The ID of the room where the event should be sent to. + event_type: The type of event to be sent. + state_key: The state key of the event to be sent, or None if it is not a state event. + origin_server_ts: The custom timestamp to send the event with. + If None, the timestamp will be the actual time when the event is sent. + content: The content of the event to be sent. + delay: How long to wait before automatically sending the event. + sticky_duration_ms: If an MSC4354 sticky event: the sticky duration (in milliseconds). + The event will be attempted to be reliably delivered to clients and remote servers + during its sticky period. + limit: The maximum number of delayed events the DB may store for the given requester. + Must be greater than 0. Returns: The generated ID assigned to the added delayed event, and the send time of the next delayed event to be sent, which is either the event just added or one added earlier. + + Raises: + LimitExceededError: if the DB has reached the limit of + how many delayed events it may store for the given requester. + AssertionError: if the limit is not greater than 0. """ + assert limit > 0, "limit must be greater than 0" + delay_id = _generate_delay_id() - send_ts = Timestamp(creation_ts + delay) + delay_ms = delay.as_millis() + send_ts = creation_ts + delay_ms def add_delayed_event_txn(txn: LoggingTransaction) -> Timestamp: + num_existing: int = self.db_pool.simple_select_one_onecol_txn( + txn, + table="delayed_events", + keyvalues={"user_localpart": user_localpart}, + retcol="COUNT(*)", + ) + if num_existing >= limit: + # Find the send_ts threshold that will bring the queue back under the limit. + # When the amount of existing delayed events has reached the limit, + # this will be the send time of the next delayed event to be sent. + # When the amount has exceeded the limit (e.g., due to config changes), + # this will be the send time of the delayed event that will be sent + # once all earlier events that exceed the limit have been sent. + # + # FIXME: Remove "AS subquery" after dropping support for PostgreSQL <16 + txn.execute( + """ + SELECT MAX(send_ts) FROM ( + SELECT * FROM delayed_events + WHERE user_localpart = ? + ORDER BY send_ts ASC + LIMIT ? + ) AS subquery + """, + ( + user_localpart, + num_existing - limit + 1, + ), + ) + row = txn.fetchone() + assert row + retry_after_ms = row[0] - self.clock.time_msec() + err = LimitExceededError( + limiter_name="add_delayed_event", + retry_after_ms=retry_after_ms if retry_after_ms > 0 else None, + ) + err.msg = "The maximum number of delayed events has been reached." + raise err + self.db_pool.simple_insert_txn( txn, table="delayed_events", @@ -143,7 +235,7 @@ class DelayedEventsStore(SQLBaseStore): "delay_id": delay_id, "user_localpart": user_localpart, "device_id": device_id, - "delay": delay, + "delay": delay_ms, "send_ts": send_ts, "room_id": room_id, "event_type": event_type, @@ -225,11 +317,49 @@ class DelayedEventsStore(SQLBaseStore): _get_count_of_delayed_events, ) + async def get_delayed_event_for_user( + self, + delay_id: str, + user_localpart: str, + ) -> DelayedEventResponse: + """ + Returns the specified pending delayed event owned by the given user. + + Raises: + NotFoundError: if there is no matching delayed event. + """ + row = await self.db_pool.simple_select_one( + table="delayed_events", + keyvalues={ + "delay_id": delay_id, + "user_localpart": user_localpart, + "is_processed": False, + }, + retcols=( + "room_id", + "event_type", + "state_key", + "delay", + "send_ts - delay", + "content", + ), + allow_none=True, + desc="get_delayed_event_for_user", + ) + if row is None: + raise NotFoundError("Delayed event not found") + return DelayedEventResponse(delay_id, *row) + async def get_all_delayed_events_for_user( self, user_localpart: str, - ) -> list[JsonDict]: - """Returns all pending delayed events owned by the given user.""" + ) -> list[DelayedEventResponseLegacyCompat]: + """ + Return all pending delayed events owned by the given user. + Includes fields from earlier revisions of MSC4140 for + compatibility with clients that still expect them. + """ + # TODO: Remove legacy fields once stable # TODO: Support Pagination stream API ("next_batch" field) rows = await self.db_pool.execute( "get_all_delayed_events_for_user", @@ -240,7 +370,7 @@ class DelayedEventsStore(SQLBaseStore): event_type, state_key, delay, - send_ts, + send_ts - delay, content FROM delayed_events WHERE user_localpart = ? AND NOT is_processed @@ -248,18 +378,7 @@ class DelayedEventsStore(SQLBaseStore): """, user_localpart, ) - return [ - { - "delay_id": DelayID(row[0]), - "room_id": str(RoomID.from_string(row[1])), - "type": EventType(row[2]), - **({"state_key": StateKey(row[3])} if row[3] is not None else {}), - "delay": Delay(row[4]), - "running_since": Timestamp(row[5] - row[4]), - "content": db_to_json(row[6]), - } - for row in rows - ] + return [DelayedEventResponseLegacyCompat(*row) for row in rows] async def process_timeout_delayed_events( self, current_ts: Timestamp, reprocess_events: bool = False diff --git a/synapse/storage/databases/main/devices.py b/synapse/storage/databases/main/devices.py index b293b47fea..6cf9270ff2 100644 --- a/synapse/storage/databases/main/devices.py +++ b/synapse/storage/databases/main/devices.py @@ -83,6 +83,10 @@ BG_UPDATE_REMOVE_DUP_OUTBOUND_POKES = "remove_dup_outbound_pokes" # `device_lists_changes_in_room.inserted_ts`. BG_UPDATE_ADD_INSERTED_TS_INDEX = "device_lists_changes_in_room_inserted_ts_idx" +# Background update name for adding an index on unconverted rows in +# `device_lists_changes_in_room`. +BG_UPDATE_ADD_UNCONVERTED_IDX = "device_lists_changes_in_room_unconverted_idx" + # Prunes entries out of the `device_lists_changes_in_room` table that are more # than this old. @@ -918,33 +922,56 @@ class DeviceWorkerStore(RoomMemberWorkerStore, EndToEndKeyWorkerStore): def _mark_as_sent_devices_by_remote_txn( self, txn: LoggingTransaction, destination: str, stream_id: int ) -> None: - # We update the device_lists_outbound_last_success with the successfully - # poked users. + # Delete all sent outbound pokes, returning them so that we can update + # `device_lists_outbound_last_success` with the successfully poked users. + # + # This is a high frequency transaction (runs very often when processing a + # backlog of device list changes) and can bog down the database CPU with the + # sheer number of statements. + # + # We prefer to trade a little bit of processing time on the Python side + # (aggregating `max_stream_id_by_user_id`) as the alternative would be to have + # two separate queries; a `SELECT ... GROUP BY user_id` with the aggregation and + # then a `DELETE` which means we touch the same rows twice. We get to save the + # cost of one of those statements (less CPU on the database) and fewer + # statements per transaction (less round-trips) means connections turn over + # faster, and can move on to process the next thing. + # + # By the nature of `MAX_EDUS_PER_TRANSACTION`, we're only dealing with 100 rows + # at max which is pretty trivial for us to process on the Python side. sql = """ - SELECT user_id, coalesce(max(o.stream_id), 0) - FROM device_lists_outbound_pokes as o - WHERE destination = ? AND o.stream_id <= ? - GROUP BY user_id + DELETE FROM device_lists_outbound_pokes + WHERE destination = ? AND stream_id <= ? + RETURNING user_id, stream_id """ txn.execute(sql, (destination, stream_id)) - rows = txn.fetchall() + # Aggregate `max_stream_id_by_user_id` + max_stream_id_by_user_id: dict[str, int] = {} + for user_id, poke_stream_id in txn: + max_stream_id_by_user_id[user_id] = max( + max_stream_id_by_user_id.get(user_id, 0), poke_stream_id + ) + + # Update `device_lists_outbound_last_success` with the successfully poked + # users. + # + # We could potentially combine this in one big CTE with the query above but it + # isn't supported by SQLite (SQLite doesn't support `DELETE` in a CTE). self.db_pool.simple_upsert_many_txn( txn=txn, table="device_lists_outbound_last_success", key_names=("destination", "user_id"), - key_values=[(destination, user_id) for user_id, _ in rows], + key_values=[ + (destination, user_id) for user_id in max_stream_id_by_user_id.keys() + ], value_names=("stream_id",), - value_values=[(stream_id,) for _, stream_id in rows], + value_values=[ + (user_stream_id,) + for user_stream_id in max_stream_id_by_user_id.values() + ], ) - # Delete all sent outbound pokes - sql = """ - DELETE FROM device_lists_outbound_pokes - WHERE destination = ? AND stream_id <= ? - """ - txn.execute(sql, (destination, stream_id)) - async def add_user_signature_change_to_streams( self, from_user_id: str, user_ids: list[str] ) -> int: @@ -2204,7 +2231,22 @@ class DeviceWorkerStore(RoomMemberWorkerStore, EndToEndKeyWorkerStore): converted_upto_stream_id: int, ) -> None: """If we've calculated the outbound pokes for a given room/device list - update, mark any subsequent changes as already converted""" + update, mark any subsequent changes as already converted. + + This is an optimization only. Skipping it is always safe, and just + means the subsequent changes get converted individually. + """ + + # Without the index added by `BG_UPDATE_ADD_UNCONVERTED_IDX`, the + # UPDATE below scans the unconverted backlog on every call, getting + # slower the further behind we are. Skip it until the index exists. + unconverted_idx_ready = ( + await self.db_pool.updates.has_completed_background_update( + BG_UPDATE_ADD_UNCONVERTED_IDX + ) + ) + if not unconverted_idx_ready: + return sql = """ UPDATE device_lists_changes_in_room @@ -2446,17 +2488,69 @@ class DeviceWorkerStore(RoomMemberWorkerStore, EndToEndKeyWorkerStore): `FALSE` have not been converted. """ + return await self.db_pool.runInteraction( + desc="get_device_change_last_converted_pos", + func=self.get_device_change_last_converted_pos_txn, + db_autocommit=True, + ) + + def get_device_change_last_converted_pos_txn( + self, txn: LoggingTransaction + ) -> tuple[int, str]: + """Get the position of the last row in `device_list_changes_in_room` that has been + converted to `device_lists_outbound_pokes`. + + Rows with a strictly greater position where `converted_to_destinations` is + `FALSE` have not been converted.""" + # There should be only one row in this table, though we want to # future-proof ourselves for when we have multiple rows (one for each # instance). So to handle that case we take the minimum of all rows. - rows = await self.db_pool.simple_select_list( + rows = self.db_pool.simple_select_list_txn( + txn, table="device_lists_changes_converted_stream_position", keyvalues={}, retcols=["stream_id", "room_id"], - desc="get_device_change_last_converted_pos", ) return cast(tuple[int, str], min(rows)) + async def get_device_list_conversion_lag(self) -> tuple[int | None, int]: + """Get how far behind we are at converting rows in + `device_lists_changes_in_room` to `device_lists_outbound_pokes`. + + Returns: + A tuple of: + 1. the timestamp (ms) at which the oldest unconverted change + was inserted. None if there is nothing to convert, or if + the oldest row predates the `inserted_ts` column. + 2. the stream ID of the last converted position. + """ + + # Rows for one device list update share a `stream_id` (and insertion + # time), so ordering by `stream_id` alone is fine. + sql = """ + SELECT inserted_ts FROM device_lists_changes_in_room + WHERE + (stream_id, room_id) > (?, ?) AND + NOT converted_to_destinations + ORDER BY stream_id ASC + LIMIT 1 + """ + + def get_device_list_conversion_lag_txn( + txn: LoggingTransaction, + ) -> tuple[int | None, int]: + stream_id, room_id = self.get_device_change_last_converted_pos_txn(txn) + + txn.execute(sql, (stream_id, room_id)) + row = txn.fetchone() + return row[0] if row else None, stream_id + + return await self.db_pool.runInteraction( + "get_device_list_conversion_lag", + get_device_list_conversion_lag_txn, + ) + async def set_device_change_last_converted_pos( self, stream_id: int, @@ -2561,9 +2655,14 @@ class DeviceWorkerStore(RoomMemberWorkerStore, EndToEndKeyWorkerStore): # We default to 0 here as that is less than all possible stream IDs. min_stream_id = 0 - def prune_device_lists_changes_in_room_txn(txn: LoggingTransaction) -> int: - nonlocal min_stream_id - + def prune_device_lists_changes_in_room_txn( + txn: LoggingTransaction, min_stream_id: int + ) -> tuple[int, int]: + """ + Returns tuple of: + - number of rows deleted + - new `min_stream_id` for the next iteration + """ delete_sql = """ DELETE FROM device_lists_changes_in_room WHERE stream_id IN ( @@ -2596,13 +2695,14 @@ class DeviceWorkerStore(RoomMemberWorkerStore, EndToEndKeyWorkerStore): updatevalues={"stream_id": min_stream_id}, ) - return num_deleted + return num_deleted, min_stream_id progress_num_rows_deleted = 0 while True: - batch_deleted = await self.db_pool.runInteraction( + batch_deleted, min_stream_id = await self.db_pool.runInteraction( "prune_device_lists_changes_in_room", prune_device_lists_changes_in_room_txn, + min_stream_id, ) finished = batch_deleted < PRUNE_DEVICE_LISTS_BATCH_SIZE @@ -2693,6 +2793,15 @@ class DeviceBackgroundUpdateStore(SQLBaseStore): where_clause="inserted_ts IS NOT NULL", ) + # Add an index to speed up `mark_redundant_device_lists_pokes`. + self.db_pool.updates.register_background_index_update( + BG_UPDATE_ADD_UNCONVERTED_IDX, + index_name="device_lists_changes_in_room_unconverted_idx", + table="device_lists_changes_in_room", + columns=["user_id", "device_id", "room_id", "stream_id"], + where_clause="NOT converted_to_destinations", + ) + async def _drop_device_list_streams_non_unique_indexes( self, progress: JsonDict, batch_size: int ) -> int: diff --git a/synapse/storage/databases/main/end_to_end_keys.py b/synapse/storage/databases/main/end_to_end_keys.py index c93ebd3dda..3075d6a05e 100644 --- a/synapse/storage/databases/main/end_to_end_keys.py +++ b/synapse/storage/databases/main/end_to_end_keys.py @@ -78,6 +78,10 @@ class DeviceKeyLookupResult: class EndToEndKeyBackgroundStore(SQLBaseStore): + CROSS_SIGNING_KEYS_REMOVE_DUPLICATES_NAME = ( + "e2e_cross_signing_signatures_remove_duplicates" + ) + def __init__( self, database: DatabasePool, @@ -101,6 +105,80 @@ class EndToEndKeyBackgroundStore(SQLBaseStore): columns=("user_id", "device_id", "algorithm", "ts_added_ms"), ) + self.db_pool.updates.register_background_update_handler( + self.CROSS_SIGNING_KEYS_REMOVE_DUPLICATES_NAME, + self._background_cross_signing_signatures_remove_duplicates, + ) + + self.db_pool.updates.register_background_index_update( + update_name="e2e_cross_signing_signatures_add_key_id_to_index", + index_name="e2e_cross_signing_signatures_idx3", + table="e2e_cross_signing_signatures", + columns=("user_id", "target_user_id", "target_device_id", "key_id"), + unique=True, + replaces_index="e2e_cross_signing_signatures2_idx", + ) + + async def _background_cross_signing_signatures_remove_duplicates( + self, progress: dict, batch_size: int + ) -> int: + """Removes duplicate cross-signing signatures so that we can add a + unique index on `(user_id, target_user_id, target_device_id, key_id)` to + `e2e_cross_signing_signatures`. + """ + + def _remove_duplicate_signatures_txn(txn: LoggingTransaction) -> int: + sql = """ + SELECT user_id, key_id, target_user_id, target_device_id, MAX(signature) + FROM e2e_cross_signing_signatures + GROUP BY user_id, key_id, target_user_id, target_device_id + HAVING COUNT(*) > 1 + LIMIT ? + """ + txn.execute(sql, (batch_size,)) + duplicate_keys = cast(list[tuple[str, str, str, str, str]], list(txn)) + + for ( + user_id, + key_id, + target_user_id, + target_device_id, + signature, + ) in duplicate_keys: + sql = """ + DELETE FROM e2e_cross_signing_signatures + WHERE + user_id = ? AND + key_id = ? AND + target_user_id = ? AND + target_device_id = ? + """ + txn.execute(sql, (user_id, key_id, target_user_id, target_device_id)) + + sql = """ + INSERT INTO e2e_cross_signing_signatures + (user_id, key_id, target_user_id, target_device_id, signature) + VALUES + (?, ?, ?, ?, ?) + """ + txn.execute( + sql, (user_id, key_id, target_user_id, target_device_id, signature) + ) + + return len(duplicate_keys) + + number_deleted = await self.db_pool.runInteraction( + self.CROSS_SIGNING_KEYS_REMOVE_DUPLICATES_NAME, + _remove_duplicate_signatures_txn, + ) + + if number_deleted < batch_size: + await self.db_pool.updates._end_background_update( + self.CROSS_SIGNING_KEYS_REMOVE_DUPLICATES_NAME + ) + + return number_deleted + class EndToEndKeyWorkerStore(EndToEndKeyBackgroundStore, CacheInvalidationWorkerStore): def __init__( @@ -1808,7 +1886,7 @@ class EndToEndKeyWorkerStore(EndToEndKeyBackgroundStore, CacheInvalidationWorker ) async def store_e2e_cross_signing_signatures( - self, user_id: str, signatures: "Iterable[SignatureListItem]" + self, user_id: str, signatures: "list[SignatureListItem]" ) -> None: """Stores cross-signing signatures. @@ -1819,28 +1897,23 @@ class EndToEndKeyWorkerStore(EndToEndKeyBackgroundStore, CacheInvalidationWorker def _store_e2e_cross_signing_signatures( txn: LoggingTransaction, - signatures: "Iterable[SignatureListItem]", + signatures: "list[SignatureListItem]", ) -> None: - self.db_pool.simple_insert_many_txn( + self.db_pool.simple_upsert_many_txn( txn, "e2e_cross_signing_signatures", - keys=( - "user_id", - "key_id", - "target_user_id", - "target_device_id", - "signature", - ), - values=[ + key_names=("user_id", "key_id", "target_user_id", "target_device_id"), + key_values=[ ( user_id, item.signing_key_id, item.target_user_id, item.target_device_id, - item.signature, ) for item in signatures ], + value_names=("signature",), + value_values=[(item.signature,) for item in signatures], ) to_invalidate = [ diff --git a/synapse/storage/databases/main/event_federation.py b/synapse/storage/databases/main/event_federation.py index d84c58dcf8..1908b26f68 100644 --- a/synapse/storage/databases/main/event_federation.py +++ b/synapse/storage/databases/main/event_federation.py @@ -38,6 +38,7 @@ from synapse.api.constants import MAX_DEPTH from synapse.api.errors import StoreError from synapse.api.room_versions import EventFormatVersions, RoomVersion from synapse.events import EventBase, make_event_from_dict +from synapse.events.py_protocol import MSC4242Event, supports_msc4242_state_dag from synapse.logging.opentracing import tag_args, trace from synapse.metrics import SERVER_NAME_LABEL from synapse.metrics.background_process_metrics import wrap_as_background_process @@ -1194,6 +1195,200 @@ class EventFederationWorkerStore( # Return all events where not all sets can reach them. return {eid for eid, n in event_to_missing_sets.items() if n} + # FIXME(2026-04-22): Remove comment when used. Unused currently, but will be used in + # future MSC4242 PRs. + async def get_state_dag( + self, room_id: str, forward_extrems: set[str] + ) -> dict[str, MSC4242Event]: + """Get the current state DAG for the given room. + + This function is called when calculating a /send_join response. + This does not check that the room is an state DAG room, so check this + before calling this function! + + This functions guarantees that the returned state DAG is connected. + + Args: + room_id: The room to get the state dag for + forward_extrems: latest event IDs in the room. The state DAG is all events reachable from these events. + Returns: + A map of event_id => event + """ + + def _get_state_events_txn(txn: LoggingTransaction, room_id: str) -> list[str]: + sql = """ + SELECT event_id FROM msc4242_state_dag_edges WHERE room_id = ? + """ + txn.execute(sql, (room_id,)) + event_ids = [ev_id for (ev_id,) in txn] + return event_ids + + # Pull out all state events for this room using the events_by_room_and_type index. + event_ids = await self.db_pool.runInteraction( + "_get_state_events_txn", + _get_state_events_txn, + room_id, + ) + event_map = await self.get_events(event_ids) + # Filter the returned state events to only include ones on the paths back from the forward + # extremities. + result: dict[str, MSC4242Event] = {} + next_ids = forward_extrems + seen: set[str] = set() + while len(next_ids) > 0: + # Pull the event and add the prev_state_events. + # We must have the event. + event_id = next_ids.pop() + if event_id in seen: + continue + seen.add(event_id) + ev = event_map[event_id] + # `prev_state_events` only exists on MSC4242 event formats, and this is only + # called for state DAG rooms. + assert supports_msc4242_state_dag(ev) + result[event_id] = ev + for prev_state_event_id in ev.prev_state_events: + next_ids.add(prev_state_event_id) + + assert len(result) > 0 # we always return the forward extremities + # Assert that the create event was returned. Pick the first event (any will do) to verify + # that this room version supports room IDs as hashes. + first_event: MSC4242Event = next(iter(result.values())) + if first_event.room_version.msc4291_room_ids_as_hashes: + create_event_id = f"${room_id[1:]}" + assert create_event_id in result + + return result + + # FIXME(2026-04-22): Remove comment when used. Unused currently, but will be used in + # future MSC4242 PRs. + async def get_missing_events_state_dag( + self, + *, + room_id: str, + earliest_event_ids: list[str], + latest_event_ids: list[str], + limit: int, + ) -> list[EventBase]: + """Get parts of the state DAG in response to a /get_missing_events query. + + Args: + room_id: The state DAG to look at + earliest_event_ids: Which events the caller has seen. These events will not be returned. + latest_event_ids: Which events to start walking back from via prev_state_events. + limit: The max number of events to return. + Returns: + A list of events, deterministically ordered according to MSC4242. + """ + ids = await self.db_pool.runInteraction( + "get_missing_events_state_dag", + self._get_missing_events_state_dag_txn, + room_id, + earliest_event_ids, + latest_event_ids, + limit, + ) + return await self.get_events_as_list(ids) + + def _get_missing_events_state_dag_txn( + self, + txn: LoggingTransaction, + room_id: str, + earliest_event_ids: list[str], + latest_event_ids: list[str], + limit: int, + ) -> list[str]: + """Walk the state DAG backward from `latest_event_ids`, stopping at + `limit` results or the beginning of the DAG, whichever comes first. + + Earliest events are treated as already-visited: they are not emitted, + and their predecessors are not traversed via them. + + Results are deterministic and ordered by breadth-first search (BFS), with lexicographic + tie-breaking among siblings at the same hops away. + + Executes as a single recursive CTE in both SQLite and Postgres. + """ + earliest_set = set(earliest_event_ids) + # Sort the seeds lexicographically by event ID: seeds are all 0 hops away from + # themselves, so the event ID is the only tie-breaker available for them and we need + # the walk to start from a deterministic order. + # See https://github.com/matrix-org/matrix-spec-proposals/blob/kegan/placeholder-1/proposals/4242-state-dags.md#get_missing_events + seed_ids = sorted(set(latest_event_ids) - earliest_set) + if not seed_ids or limit <= 0: + return [] + + seed_clause, seed_args = make_in_list_sql_clause( + self.database_engine, "e.event_id", seed_ids + ) + + # With no earliest events this is a no-op clause that is TRUE for every row. + earliest_clause, earliest_args = make_in_list_sql_clause( + self.database_engine, + "e.prev_state_event_id", + earliest_event_ids, + negative=True, + ) + + query = f""" + WITH RECURSIVE walk(event_id, hops) AS ( + SELECT + e.prev_state_event_id, + 1 + FROM msc4242_state_dag_edges e + WHERE e.room_id = ? + AND {seed_clause} + -- The create event has no edges, so it is stored as a single row with a NULL + -- `prev_state_event_id`. Skipping NULLs therefore only skips that sentinel + -- row: the create event is still returned by the walk, because it appears as + -- the `prev_state_event_id` of the events that reference it. + AND e.prev_state_event_id IS NOT NULL + AND {earliest_clause} + + -- `UNION` rather than `UNION ALL`: the same (event_id, hops) pair can be + -- reached via multiple children, and de-duplicating here stops us expanding + -- the same row over and over. + UNION + + SELECT + e.prev_state_event_id, + w.hops + 1 + FROM walk w + JOIN msc4242_state_dag_edges e + ON e.room_id = ? + AND e.event_id = w.event_id + WHERE e.prev_state_event_id IS NOT NULL + AND {earliest_clause} + -- Bound the recursion by `limit`: every extra hop adds at least one event to + -- the result, so events more than `limit` hops away can never make it into a + -- `limit`-sized response. This is also what makes the query safe against a + -- cyclic edge set: a cycle cannot be created via the normal write path, but + -- if one did exist the walk would still stop after `limit` hops rather than + -- spinning forever. + AND w.hops < ? + ) + -- An event can be reached by several paths with different hop counts. MSC4242 + -- orders by distance from the seed events, which is the *shortest* such path, so + -- collapse each event to its minimum hop count before sorting. `hops` is only + -- selected because we need it as the primary sort key. + SELECT event_id, MIN(hops) AS hops + FROM walk + GROUP BY event_id + ORDER BY hops, event_id + LIMIT ? + """ + + params: list = [room_id] + params.extend(seed_args) + params.extend(earliest_args) + params.append(room_id) + params.extend(earliest_args) + params.append(limit) + params.append(limit) + + txn.execute(query, params) + return [row[0] for row in txn] + @trace @tag_args async def get_backfill_points_in_room( @@ -1983,6 +2178,13 @@ class EventFederationWorkerStore( latest_events: list[str], limit: int, ) -> list[EventBase]: + """ + Walk backwards in the DAG of events, + starting at `latest_events` and stopping at `earliest_events` (or when having reached `limit` events). + + This function will check that `latest_events` and `earliest_events` are in the correct + room (`room_id`), appropriately ignoring any that aren't. + """ ids = await self.db_pool.runInteraction( "get_missing_events", self._get_missing_events, @@ -2001,20 +2203,49 @@ class EventFederationWorkerStore( latest_events: list[str], limit: int, ) -> list[str]: + # It's OK that this has not been filtered by correct-room, + # because we will only compare based on event ID from the events + # we happen to run into. seen_events = set(earliest_events) - front = set(latest_events) - seen_events - event_results: list[str] = [] - query = ( - "SELECT prev_event_id FROM event_edges " - "WHERE event_id = ? AND NOT is_state " - "LIMIT ?" + # Pre-filter the `latest_events` to only include those + # that are in this room (and that we know about) + # This makes events in the wrong room get treated the same as unknown events. + events_clause, events_args = make_in_list_sql_clause( + self.database_engine, + "event_id", + # Don't waste time looking at events that the requester told us + # they already know about. + # (They probably shouldn't send this in the first place) + set(latest_events) - seen_events, ) + txn.execute( + f""" + SELECT event_id + FROM events + WHERE {events_clause} AND room_id = ? + """, + (*events_args, room_id), + ) + # Start walking back from the legitimate and known `latest_events` + front = {latest_event_id for (latest_event_id,) in txn} + + event_results: list[str] = [] while front and len(event_results) < limit: new_front = set() for event_id in front: - txn.execute(query, (event_id, limit - len(event_results))) + txn.execute( + """ + SELECT ee.prev_event_id FROM event_edges AS ee + JOIN events ON events.event_id = ee.prev_event_id + WHERE ee.event_id = ? + AND events.room_id = ? + AND NOT ee.is_state + LIMIT ? + """, + (event_id, room_id, limit - len(event_results)), + ) new_results = {t[0] for t in txn} - seen_events new_front |= new_results diff --git a/synapse/storage/databases/main/events.py b/synapse/storage/databases/main/events.py index d92bbeeae3..35f387576f 100644 --- a/synapse/storage/databases/main/events.py +++ b/synapse/storage/databases/main/events.py @@ -42,6 +42,7 @@ from synapse.api.constants import ( EventContentFields, EventTypes, Membership, + ProfileUpdateAction, RelationTypes, ) from synapse.api.errors import PartialStateConflictError @@ -76,6 +77,7 @@ from synapse.types import ( MutableStateMap, StateMap, StrCollection, + UserID, ) from synapse.types.handlers import SLIDING_SYNC_DEFAULT_BUMP_EVENT_TYPES from synapse.types.state import StateFilter @@ -267,6 +269,9 @@ class PersistEventsStore: self._clock = hs.get_clock() self._instance_name = hs.get_instance_name() self._msc4354_enabled = hs.config.experimental.msc4354_enabled + self._include_profile_updates_in_sync = ( + hs.config.server.include_profile_updates_in_sync + ) self._ephemeral_messages_enabled = hs.config.server.enable_ephemeral_messages self.is_mine_id = hs.is_mine_id @@ -2118,6 +2123,129 @@ class PersistEventsStore: txn, {m for m in members_to_cache_bust if not self.hs.is_mine_id(m)} ) + if self._include_profile_updates_in_sync: + # Handle changes to the profile updates stream. + # We've already done a bunch of work calculating the changes needed + # for the sliding sync tables, so we may as well re-use that information + # here to avoid parsing the state delta again, and handling various + # edge cases. + # FIXME: See issue https://github.com/element-hq/synapse/issues/19981 + # for concerns around the current implementation of the profile + # updates stream. + profile_update_additions = { + c.user_id + for c in sliding_sync_table_changes.to_insert_membership_snapshots + if self.hs.is_mine_id(c.user_id) + # FIXME: Ideally we would filter out JOIN -> JOIN. See note below. + and c.membership == Membership.JOIN + } + profile_update_leaves = { + c.user_id + for c in sliding_sync_table_changes.to_insert_membership_snapshots + if self.hs.is_mine_id(c.user_id) + # Any transition from JOIN to something else counts as a leave here. + # Even the 'invalid' transitions might effectively happen due to + # state resolution. + and c.membership != Membership.JOIN + } | ( + # We also need to consider users that get fully state reset out of the room. + # These should be treated as 'leave' + set(sliding_sync_table_changes.to_delete_membership_snapshots) + ) + + if profile_update_additions: + # Write the profile updates for additions to the room, from either + # a join, knock, invite, etc. + # FIXME this will add rows also when a display name changes due to + # the facts that `sliding_sync_table_changes` contains a JOIN + # membership event in that case. We should aim to filter these + # unnecessary rows out, as we're also generating an UPDATE profile + # update action row for the actual display name change itself. + # See https://github.com/element-hq/synapse/issues/19981 + self.store.record_profile_updates_for_user_joined_room_txn( + txn=txn, + room_id=room_id, + joined_users=profile_update_additions, + ) + if profile_update_leaves: + # Write the profile updates for LEAVE events + for user_id in profile_update_leaves: + self._record_profile_updates_for_user_left_room_txn( + txn=txn, + user_id=UserID.from_string(user_id), + room_id=room_id, + ) + + def _record_profile_updates_for_user_left_room_txn( + self, + txn: LoggingTransaction, + user_id: UserID, + room_id: str, + ) -> None: + """ + Record updates into the profile updates stream for when a user leaves a room. + + If this was the last shared room with a set of users, clear all old rows from + the `profile_updates_per_user` table relating to those users, to avoid exposing + any profile field changes past the point of not being in any common rooms with + the user. + + Currently, updates are only recorded for local users. + + Note, this method lives here in the events store file due to the profile + store not having access to the membership store (which the events store does), + which we need to re-use the `do_users_share_a_room_txn` method there. + + Args: + user_id: The user who left the room. + room_id: The room that was left. + """ + # Get the local members of the room + room_members = self.db_pool.simple_select_onecol_txn( + txn=txn, + table="local_current_membership", + retcol="user_id", + keyvalues={ + "membership": Membership.JOIN, + "room_id": room_id, + }, + ) + # For each user check if we still share rooms + users_sharing_rooms = self.store.do_users_share_a_room_txn( + txn=txn, + user_id=user_id.to_string(), + other_user_ids=set(room_members), + ) + users_no_longer_sharing_rooms = set(room_members) - set( + users_sharing_rooms.keys() + ) + + # First clear the previous rows from the table + user_clause, user_args = make_in_list_sql_clause( + txn.database_engine, + "user_id", + users_no_longer_sharing_rooms, + ) + txn.execute( + f""" + DELETE FROM profile_updates_per_user + WHERE {user_clause} + AND stream_id IN ( + SELECT stream_id FROM profile_updates WHERE user_id = ? + ) + """, + (*user_args, user_id.to_string()), + ) + + # Now record the "left room" action in the stream + self.store.record_profile_updates_txn( + txn=txn, + user_id=user_id, + action=ProfileUpdateAction.LEFT_ROOM, + field_names=[], + target_users=users_no_longer_sharing_rooms, + ) + @classmethod def _get_relevant_sliding_sync_current_state_event_ids_txn( cls, txn: LoggingTransaction, room_id: str diff --git a/synapse/storage/databases/main/keys.py b/synapse/storage/databases/main/keys.py index f81257b5a1..3e733643e7 100644 --- a/synapse/storage/databases/main/keys.py +++ b/synapse/storage/databases/main/keys.py @@ -113,10 +113,20 @@ class KeyStore(CacheInvalidationWorkerStore): # invalidate takes a tuple corresponding to the params of # _get_server_keys_json. _get_server_keys_json only takes one # param, which is itself the 2-tuple (server_name, key_id). - self._invalidate_cache_and_stream_bulk( + # + # Invalidate the local cache directly, but we can only send + # primitive types per argument over replication, so JSON-encode the + # nested key and unpack it on the receiving side (see + # `CacheInvalidationWorkerStore.process_replication_rows`). + for key_id in verify_keys: + txn.call_after( + self._get_server_keys_json.invalidate, + ((server_name, key_id),), + ) + self._send_invalidation_to_replication_bulk( txn, - self._get_server_keys_json, - [((server_name, key_id),) for key_id in verify_keys], + self._get_server_keys_json.__name__, + [(json.dumps([server_name, key_id]),) for key_id in verify_keys], ) self._invalidate_cache_and_stream_bulk( txn, diff --git a/synapse/storage/databases/main/profile.py b/synapse/storage/databases/main/profile.py index 68548434a9..1a2dea7460 100644 --- a/synapse/storage/databases/main/profile.py +++ b/synapse/storage/databases/main/profile.py @@ -19,13 +19,21 @@ # # import json -from typing import TYPE_CHECKING, cast +from collections.abc import Set +from typing import TYPE_CHECKING, Collection, cast +import attr from canonicaljson import encode_canonical_json -from synapse.api.constants import ProfileFields +from synapse.api.constants import ( + EventTypes, + Membership, + ProfileFields, + ProfileUpdateAction, +) from synapse.api.errors import Codes, StoreError -from synapse.storage._base import SQLBaseStore +from synapse.replication.tcp.streams._base import ProfileUpdatesStream +from synapse.storage._base import SQLBaseStore, db_to_json, make_in_list_sql_clause from synapse.storage.database import ( DatabasePool, LoggingDatabaseConnection, @@ -33,7 +41,9 @@ from synapse.storage.database import ( ) from synapse.storage.databases.main.roommember import ProfileInfo from synapse.storage.engines import PostgresEngine, Sqlite3Engine +from synapse.storage.util.id_generators import MultiWriterIdGenerator from synapse.types import JsonDict, JsonValue, UserID +from synapse.util.json import json_encoder if TYPE_CHECKING: from synapse.server import HomeServer @@ -43,6 +53,16 @@ if TYPE_CHECKING: MAX_PROFILE_SIZE = 65536 +@attr.s(slots=True, frozen=True, auto_attribs=True) +class ProfileUpdate: + """An update to a user's profile.""" + + stream_id: int + user_id: str + action: str + affected_fields: frozenset[str] | None + + class ProfileWorkerStore(SQLBaseStore): def __init__( self, @@ -52,6 +72,7 @@ class ProfileWorkerStore(SQLBaseStore): ): super().__init__(database, db_conn, hs) self.server_name: str = hs.hostname + self._instance_name: str = hs.get_instance_name() self.database_engine = database.engine self.db_pool.updates.register_background_index_update( "profiles_full_user_id_key_idx", @@ -65,6 +86,24 @@ class ProfileWorkerStore(SQLBaseStore): "populate_full_user_id_profiles", self.populate_full_user_id_profiles ) + self._include_profile_updates_in_sync = ( + hs.config.server.include_profile_updates_in_sync + ) + self._is_events_writer = self._instance_name in hs.config.worker.writers.events + self._profile_updates_id_gen: MultiWriterIdGenerator = MultiWriterIdGenerator( + db_conn=db_conn, + db=database, + notifier=hs.get_replication_notifier(), + stream_name="profile_updates", + server_name=self.server_name, + instance_name=self._instance_name, + tables=[ + ("profile_updates", "instance_name", "stream_id"), + ], + sequence_name="profile_updates_sequence", + writers=hs.config.worker.writers.events, + ) + async def populate_full_user_id_profiles( self, progress: JsonDict, batch_size: int ) -> int: @@ -76,13 +115,15 @@ class ProfileWorkerStore(SQLBaseStore): lower_bound_id = progress.get("lower_bound_id", "") def _get_last_id(txn: LoggingTransaction) -> str | None: - sql = """ - SELECT user_id FROM profiles - WHERE user_id > ? - ORDER BY user_id - LIMIT 1 OFFSET 1000 - """ - txn.execute(sql, (lower_bound_id,)) + txn.execute( + """ + SELECT user_id FROM profiles + WHERE user_id > ? + ORDER BY user_id + LIMIT 1 OFFSET 1000 + """, + (lower_bound_id,), + ) res = txn.fetchone() if res: upper_bound_id = res[0] @@ -93,21 +134,22 @@ class ProfileWorkerStore(SQLBaseStore): def _process_batch( txn: LoggingTransaction, lower_bound_id: str, upper_bound_id: str ) -> None: - sql = """ - UPDATE profiles - SET full_user_id = '@' || user_id || ? - WHERE ? < user_id AND user_id <= ? AND full_user_id IS NULL - """ - txn.execute(sql, (f":{self.server_name}", lower_bound_id, upper_bound_id)) + txn.execute( + """ + UPDATE profiles + SET full_user_id = '@' || user_id || ? + WHERE ? < user_id AND user_id <= ? AND full_user_id IS NULL + """, + (f":{self.server_name}", lower_bound_id, upper_bound_id), + ) def _final_batch(txn: LoggingTransaction, lower_bound_id: str) -> None: - sql = """ - UPDATE profiles - SET full_user_id = '@' || user_id || ? - WHERE ? < user_id AND full_user_id IS NULL - """ txn.execute( - sql, + """ + UPDATE profiles + SET full_user_id = '@' || user_id || ? + WHERE ? < user_id AND full_user_id IS NULL + """, ( f":{self.server_name}", lower_bound_id, @@ -115,10 +157,11 @@ class ProfileWorkerStore(SQLBaseStore): ) if isinstance(self.database_engine, PostgresEngine): - sql = """ - ALTER TABLE profiles VALIDATE CONSTRAINT full_user_id_not_null - """ - txn.execute(sql) + txn.execute( + """ + ALTER TABLE profiles VALIDATE CONSTRAINT full_user_id_not_null + """, + ) upper_bound_id = await self.db_pool.runInteraction( "populate_full_user_id_profiles", _get_last_id @@ -152,6 +195,13 @@ class ProfileWorkerStore(SQLBaseStore): return 50 + def process_replication_position( + self, stream_name: str, instance_name: str, token: int + ) -> None: + if stream_name == ProfileUpdatesStream.NAME: + self._profile_updates_id_gen.advance(instance_name, token) + super().process_replication_position(stream_name, instance_name, token) + async def get_profileinfo(self, user_id: UserID) -> ProfileInfo: """ Fetch the display name and avatar URL of a user. @@ -210,7 +260,9 @@ class ProfileWorkerStore(SQLBaseStore): desc="get_profile_avatar_url", ) - async def get_profile_field(self, user_id: UserID, field_name: str) -> JsonValue: + async def get_profile_field( + self, user_id: UserID, field_name: str + ) -> JsonValue | dict[str, JsonValue]: """ Get a custom profile field for a user. @@ -222,42 +274,54 @@ class ProfileWorkerStore(SQLBaseStore): The string value if the field exists, otherwise raises 404. """ - def get_profile_field(txn: LoggingTransaction) -> JsonValue: + def get_profile_field( + txn: LoggingTransaction, + ) -> JsonValue | dict[str, JsonValue]: # This will error if field_name has double quotes in it, but that's not # possible due to the grammar. field_path = f'$."{field_name}"' if isinstance(self.database_engine, PostgresEngine): - sql = """ - SELECT JSONB_PATH_EXISTS(fields, ?), JSONB_EXTRACT_PATH(fields, ?) - FROM profiles - WHERE user_id = ? - """ txn.execute( - sql, + """ + SELECT JSONB_PATH_EXISTS(fields, ?), JSONB_EXTRACT_PATH(fields, ?) + FROM profiles + WHERE user_id = ? + """, (field_path, field_name, user_id.localpart), ) + row = txn.fetchone() + if row is None: + # The user has no profile at all. + raise StoreError(404, "No row found") + # Test exists first since value being None is used for both # missing and a null JSON value. - exists, value = cast(tuple[bool, JsonValue], txn.fetchone()) + exists, value = cast(tuple[bool, JsonValue | dict[str, JsonValue]], row) if not exists: raise StoreError(404, "No row found") return value else: - sql = """ - SELECT JSON_TYPE(fields, ?), JSON_EXTRACT(fields, ?) - FROM profiles - WHERE user_id = ? - """ txn.execute( - sql, + """ + SELECT JSON_TYPE(fields, ?), JSON_EXTRACT(fields, ?) + FROM profiles + WHERE user_id = ? + """, (field_path, field_path, user_id.localpart), ) + row = txn.fetchone() + if row is None: + # The user has no profile at all. + raise StoreError(404, "No row found") + # If value_type is None, then the value did not exist. - value_type, value = cast(tuple[str | None, JsonValue], txn.fetchone()) + value_type, value = cast( + tuple[str | None, JsonValue | dict[str, JsonValue]], row + ) if not value_type: raise StoreError(404, "No row found") # If value_type is object or array, then need to deserialize the JSON. @@ -285,12 +349,330 @@ class ProfileWorkerStore(SQLBaseStore): retcol="fields", desc="get_profile_fields", ) - # The SQLite driver doesn't automatically convert JSON to - # Python objects + # The SQLite driver doesn't have a JSON datatype. if isinstance(self.database_engine, Sqlite3Engine) and result: result = json.loads(result) return result or {} + def get_max_profile_updates_stream_id(self) -> int: + """Get the current maximum stream_id for profile updates.""" + return self._profile_updates_id_gen.get_current_token() + + def get_profile_updates_stream_id_generator(self) -> MultiWriterIdGenerator: + return self._profile_updates_id_gen + + async def get_updated_profile_updates( + self, *, from_id: int, to_id: int, limit: int + ) -> list[tuple[int, str, str, frozenset[str] | None]]: + """Get updates to profile updates between two stream IDs. + + Bounds: from_id < ... <= to_id + + Args: + from_id: The starting stream ID (exclusive) + to_id: The ending stream ID (inclusive) + limit: The maximum number of rows to return + + Returns: + list of tuples representing stream_id, user_id, action and field_name + """ + if from_id >= to_id: + return [] + + def _get_updated_profile_updates_txn( + txn: LoggingTransaction, + ) -> list[tuple[int, str, str, frozenset[str] | None]]: + txn.execute( + """ + SELECT + stream_id, user_id, action, affected_fields + FROM profile_updates + WHERE + ? < stream_id AND stream_id <= ? + ORDER BY stream_id ASC LIMIT ? + """, + (from_id, to_id, limit), + ) + + return [ + ( + stream_id, + user_id, + action, + ( + # affected_fields is a JSON array, turn it to a frozenset[str] + frozenset(db_to_json(affected_fields)) + if affected_fields is not None + else None + ), + ) + for stream_id, user_id, action, affected_fields in txn + ] + + return await self.db_pool.runInteraction( + "get_updated_profile_updates", _get_updated_profile_updates_txn + ) + + # FIXME this function should be deleted, it's not used. + async def get_profile_updates_for_fields( + self, + *, + from_id: int, + to_id: int, + field_names: Set[str], + ) -> list[ProfileUpdate]: + """Get profile update markers for the given fields in a stream range. + + Bounds: from_id < ... <= to_id + + Args: + from_id: The starting stream ID (exclusive) + to_id: The ending stream ID (inclusive) + field_names: List of field names to filter against. + + Returns: + list of ProfileUpdates update rows + The `affected_fields` entry in the ProfileUpdates will be filtered. + """ + if from_id >= to_id: + return [] + + if not field_names: + return [] + + def _get_profile_updates_for_fields_txn( + txn: LoggingTransaction, + ) -> list[ProfileUpdate]: + wanted_field_in_elems_clause, wanted_field_in_elems_args = ( + make_in_list_sql_clause( + txn.database_engine, "field_names.value", field_names + ) + ) + + if isinstance(txn.database_engine, PostgresEngine): + # Note that if we had a GIN index on `affected_fields`, this would defeat it. + # If we decide we want one, we should consider using the `?|` operator or its + # clearer-named `jsonb_exists_any` equivalent. + all_field_names_table_expression = ( + "jsonb_array_elements_text(affected_fields) AS field_names(value)" + ) + else: + # json_each is a table-valued function that gives `value` as one of its column names + all_field_names_table_expression = ( + "json_each(affected_fields) AS field_names" + ) + + txn.execute( + f""" + SELECT stream_id, user_id, action, affected_fields + FROM profile_updates + WHERE ? < stream_id AND stream_id <= ? + AND ( + (EXISTS (SELECT 1 FROM {all_field_names_table_expression} WHERE {wanted_field_in_elems_clause})) + OR action != ? + ) + ORDER BY stream_id ASC + """, + ( + from_id, + to_id, + *wanted_field_in_elems_args, + ProfileUpdateAction.UPDATE.value, + ), + ) + rows = cast(list[tuple[int, str, str, str | None]], txn.fetchall()) + + updates: list[ProfileUpdate] = [] + for stream_id, user_id, action, affected_fields_dbjson in rows: + updates.append( + ProfileUpdate( + stream_id=stream_id, + user_id=user_id, + action=action, + affected_fields=( + # Get the field names that were affected by this update + # and intersect with the field names we care about + frozenset(db_to_json(affected_fields_dbjson)) & field_names + ) + if affected_fields_dbjson is not None + else None, + ) + ) + + return updates + + return await self.db_pool.runInteraction( + "get_profile_updates_for_fields", _get_profile_updates_for_fields_txn + ) + + async def get_profile_updates_for_user_and_fields( + self, + *, + from_id: int, + to_id: int, + user_id: str, + field_names: Set[str] | None, + include_users: set[str] | None = None, + ) -> list[ProfileUpdate]: + """Get profile update markers for a user in a stream range. + + The returned profile update rows are restricted to those with a + corresponding `profile_updates_per_user` row for the syncing user. + + Bounds: from_id < ... <= to_id + + Args: + from_id: The starting stream ID (exclusive). + to_id: The ending stream ID (inclusive). + user_id: The full user ID to filter on. + field_names: Set of field names to filter update actions against. + `None` means "include all fields". + include_users: If given, only include updates for these user IDs. + + Returns: + A list of ProfileUpdate update rows, in stream order + """ + if from_id >= to_id: + return [] + + if field_names is not None and len(field_names) == 0: + return [] + + if include_users is not None and len(include_users) == 0: + # All updates have been filtered out by lazy-loading. + return [] + + def _get_profile_updates_for_user_and_fields_txn( + txn: LoggingTransaction, + ) -> list[ProfileUpdate]: + # Build a `field_clause` that matches updates containing the fields we are interested in + if field_names is None: + # We are interested in all fields, so match any update with fields + field_clause = "pu.affected_fields IS NOT NULL" + field_args: list[str] = [] + else: + wanted_field_in_elems_clause, field_args = make_in_list_sql_clause( + txn.database_engine, "field_names.value", field_names + ) + + if isinstance(txn.database_engine, PostgresEngine): + # Note that if we had a GIN index on `affected_fields`, this would defeat it. + # If we decide we want one, we should consider using the `?|` operator or its + # clearer-named `jsonb_exists_any` equivalent. + all_field_names_table_expression = "jsonb_array_elements_text(pu.affected_fields) AS field_names(value)" + else: + # json_each is a table-valued function that gives `value` as one of its column names + all_field_names_table_expression = ( + "json_each(pu.affected_fields) AS field_names" + ) + field_clause = f"(EXISTS (SELECT 1 FROM {all_field_names_table_expression} WHERE {wanted_field_in_elems_clause}))" + + user_clause = "" + user_args: list[str] = [] + if include_users is not None: + # Filter out rows that aren't in `include_users`, if defined. + # This is only relevant when lazy-loading. + user_clause, user_args = make_in_list_sql_clause( + txn.database_engine, "pu.user_id", include_users + ) + user_clause = f"AND {user_clause}" + + # Retrieve profile updates where there's a corresponding row in + # `profile_updates_per_user` within the given `stream_id` bounds + # and the `user_id` and `field_names` match. + txn.execute( + f""" + SELECT pu.stream_id, pu.user_id, pu.action, pu.affected_fields + FROM profile_updates AS pu + INNER JOIN profile_updates_per_user AS puf + ON pu.stream_id = puf.stream_id + WHERE ? < pu.stream_id AND pu.stream_id <= ? + AND puf.user_id = ? + {user_clause} + AND ( + {field_clause} + OR pu.action != ? + ) + ORDER BY pu.stream_id ASC + """, + ( + from_id, + to_id, + user_id, + *user_args, + *field_args, + ProfileUpdateAction.UPDATE.value, + ), + ) + rows = cast(list[tuple[int, str, str, str | None]], txn.fetchall()) + + updates: list[ProfileUpdate] = [] + for stream_id, updated_user_id, action, affected_fields_dbjson in rows: + if affected_fields_dbjson is not None: + # Get the field names that were affected by this update + affected_fields = frozenset(db_to_json(affected_fields_dbjson)) + if field_names is not None: + # Only include the field names that we care about + affected_fields &= field_names + else: + affected_fields = None + updates.append( + ProfileUpdate( + stream_id=stream_id, + user_id=updated_user_id, + action=action, + affected_fields=affected_fields, + ) + ) + + return updates + + return await self.db_pool.runInteraction( + "get_profile_updates_for_user_and_fields", + _get_profile_updates_for_user_and_fields_txn, + ) + + async def get_profile_data_for_users( + self, user_ids: Collection[str] + ) -> dict[str, dict[str, JsonValue | dict[str, JsonValue]]]: + """Fetch displayname/avatar_url/custom fields for a list of users. + + Currently, this returns only local users as the `profiles` table only + tracks local users. + + Args: + user_ids: List of user IDs to filter against. + + Returns: + Dictionary of displayname/avatar_url/custom fields for a list of users. + """ + if not user_ids: + return {} + + rows = await self.db_pool.simple_select_many_batch( + table="profiles", + column="full_user_id", + iterable=user_ids, + retcols=("full_user_id", "displayname", "avatar_url", "fields"), + desc="get_profile_data_for_users", + ) + + results: dict[str, dict[str, JsonValue | dict[str, JsonValue]]] = {} + for full_user_id, displayname, avatar_url, fields in rows: + user_fields = fields or {} + # The SQLite driver doesn't have a JSON datatype. + if isinstance(self.database_engine, Sqlite3Engine) and fields: + user_fields = json.loads(fields) + base_fields = { + ProfileFields.DISPLAYNAME: displayname, + ProfileFields.AVATAR_URL: avatar_url, + } + user_fields.update(base_fields) + + results[full_user_id] = user_fields + + return results + async def create_profile(self, user_id: UserID) -> None: """ Create a blank profile for a user, if one does not already exist. @@ -312,7 +694,7 @@ class ProfileWorkerStore(SQLBaseStore): txn: LoggingTransaction, user_id: UserID, new_field_name: str, - new_value: JsonValue, + new_value: JsonValue | dict[str, JsonValue], ) -> None: # For each entry there are 4 quotes (2 each for key and value), 1 colon, # and 1 comma. @@ -346,7 +728,10 @@ class ProfileWorkerStore(SQLBaseStore): # possible due to the grammar. (f'$."{new_field_name}"', user_id.localpart), ) - row = cast(tuple[int | None, int | None, int | None], txn.fetchone()) + row = cast("tuple[int | None, int | None, int | None] | None", txn.fetchone()) + # The user may have no profile row at all; treat as an empty profile. + if row is None: + row = (None, None, None) # The values return null if the column is null. total_bytes = ( @@ -372,103 +757,57 @@ class ProfileWorkerStore(SQLBaseStore): if total_bytes > MAX_PROFILE_SIZE: raise StoreError(400, "Profile too large", Codes.PROFILE_TOO_LARGE) - async def set_profile_displayname( - self, user_id: UserID, new_displayname: str | None - ) -> None: + def _set_profile_field_txn( + self, + txn: LoggingTransaction, + user_id: UserID, + field_name: str, + new_value: JsonValue | dict[str, JsonValue], + ) -> int | None: """ - Set the display name of a user. + Wrapper function to set a profile field value and write to the profile + update stream tables in one transaction. Args: - user_id: The user's ID. - new_displayname: The new display name. If this is None, the user's display - name is removed. + txn: The transaction to use + user_id: The user to set the profile field for + field_name: The field to set the value for + new_value: New value for the profile field + + Returns: + The profile updates stream ID that was created in this transaction """ - user_localpart = user_id.localpart + if self._include_profile_updates_in_sync: + assert self._is_events_writer - def set_profile_displayname(txn: LoggingTransaction) -> None: - if new_displayname is not None: - self._check_profile_size( - txn, user_id, ProfileFields.DISPLAYNAME, new_displayname - ) + self._check_profile_size(txn, user_id, field_name, new_value) + if field_name in (ProfileFields.DISPLAYNAME, ProfileFields.AVATAR_URL): self.db_pool.simple_upsert_txn( txn, table="profiles", - keyvalues={"user_id": user_localpart}, + keyvalues={"user_id": user_id.localpart}, values={ - "displayname": new_displayname, + field_name: new_value, "full_user_id": user_id.to_string(), }, ) - - await self.db_pool.runInteraction( - "set_profile_displayname", set_profile_displayname - ) - - async def set_profile_avatar_url( - self, user_id: UserID, new_avatar_url: str | None - ) -> None: - """ - Set the avatar of a user. - - Args: - user_id: The user's ID. - new_avatar_url: The new avatar URL. If this is None, the user's avatar is - removed. - """ - user_localpart = user_id.localpart - - def set_profile_avatar_url(txn: LoggingTransaction) -> None: - if new_avatar_url is not None: - self._check_profile_size( - txn, user_id, ProfileFields.AVATAR_URL, new_avatar_url - ) - - self.db_pool.simple_upsert_txn( - txn, - table="profiles", - keyvalues={"user_id": user_localpart}, - values={ - "avatar_url": new_avatar_url, - "full_user_id": user_id.to_string(), - }, - ) - - await self.db_pool.runInteraction( - "set_profile_avatar_url", set_profile_avatar_url - ) - - async def set_profile_field( - self, user_id: UserID, field_name: str, new_value: JsonValue - ) -> None: - """ - Set a custom profile field for a user. - - Args: - user_id: The user's ID. - field_name: The name of the custom profile field. - new_value: The value of the custom profile field. - """ - - # Encode to canonical JSON. - canonical_value = encode_canonical_json(new_value) - - def set_profile_field(txn: LoggingTransaction) -> None: - self._check_profile_size(txn, user_id, field_name, new_value) + else: + # Encode to canonical JSON. + canonical_value = encode_canonical_json(new_value) if isinstance(self.database_engine, PostgresEngine): from psycopg2.extras import Json # Note that the || jsonb operator is not recursive, any duplicate # keys will be taken from the second value. - sql = """ - INSERT INTO profiles (user_id, full_user_id, fields) VALUES (?, ?, JSON_BUILD_OBJECT(?, ?::jsonb)) - ON CONFLICT (user_id) - DO UPDATE SET full_user_id = EXCLUDED.full_user_id, fields = COALESCE(profiles.fields, '{}'::jsonb) || EXCLUDED.fields - """ - txn.execute( - sql, + """ + INSERT INTO profiles + (user_id, full_user_id, fields) VALUES (?, ?, JSON_BUILD_OBJECT(?, ?::jsonb)) + ON CONFLICT (user_id) + DO UPDATE SET full_user_id = EXCLUDED.full_user_id, fields = COALESCE(profiles.fields, '{}'::jsonb) || EXCLUDED.fields + """, ( user_id.localpart, user_id.to_string(), @@ -479,32 +818,212 @@ class ProfileWorkerStore(SQLBaseStore): ), ) else: - # You may be tempted to use json_patch instead of providing the parameters - # twice, but that recursively merges objects instead of replacing. - sql = """ - INSERT INTO profiles (user_id, full_user_id, fields) VALUES (?, ?, JSON_OBJECT(?, JSON(?))) - ON CONFLICT (user_id) - DO UPDATE SET full_user_id = EXCLUDED.full_user_id, fields = JSON_SET(COALESCE(profiles.fields, '{}'), ?, JSON(?)) - """ # This will error if field_name has double quotes in it, but that's not # possible due to the grammar. json_field_name = f'$."{field_name}"' txn.execute( - sql, + # You may be tempted to use json_patch instead of providing the parameters + # twice, but that recursively merges objects instead of replacing. + """ + INSERT INTO profiles (user_id, full_user_id, fields) VALUES (?, ?, JSON_OBJECT(?, JSON(?))) + ON CONFLICT (user_id) + DO UPDATE SET full_user_id = EXCLUDED.full_user_id, fields = JSON_SET(COALESCE(profiles.fields, '{}'), ?, JSON(?)) + """, ( user_id.localpart, user_id.to_string(), - json_field_name, + field_name, canonical_value, json_field_name, canonical_value, ), ) - await self.db_pool.runInteraction("set_profile_field", set_profile_field) + if not self._include_profile_updates_in_sync: + return None - async def delete_profile_field(self, user_id: UserID, field_name: str) -> None: + # Record updates in the profile updates stream + stream_id = self.record_profile_updates_txn( + txn=txn, + user_id=user_id, + action=ProfileUpdateAction.UPDATE, + field_names=[field_name], + ) + + return stream_id + + def record_profile_updates_for_user_joined_room_txn( + self, *, txn: LoggingTransaction, room_id: str, joined_users: set[str] + ) -> None: + """ + Record profile updates for membership additions to a room. + + Currently, updates are only recorded for local users. + + Args: + txn: The transaction to use. + room_id: The room ID concerned. + joined_users: A list of users who have "joined" the room, which here also + means "invited" or "knocked", as in either case we consider that the + users profile should be pushed to the client, should they need it + already even if the user hasn't actually joined the room. + """ + if not self._include_profile_updates_in_sync: + return + + assert self._is_events_writer + + # Ensure we're working with local users only + users = {user_id for user_id in joined_users if self.hs.is_mine_id(user_id)} + + # Record the profile updates for each user + for user_id in users: + self.record_profile_updates_txn( + txn=txn, + user_id=UserID.from_string(user_id), + action=ProfileUpdateAction.JOINED_ROOM, + field_names=None, + user_rooms={room_id}, + ) + + def record_profile_updates_txn( + self, + *, + txn: LoggingTransaction, + user_id: UserID, + action: ProfileUpdateAction, + field_names: Collection[str] | None, + user_rooms: set[str] | None = None, + target_users: set[str] | None = None, + ) -> int | None: + """ + Record updates into the profile updates stream tables. + + Currently, updates are only recorded for local users. + + Args: + txn: Transaction to use + user_id: User ID that made the profile update + action: The profile update action, either `update`, `left_room` or + `joined_room`. + field_names: A list of fields that were set, if ProfileUpdateAction.UPDATE + user_rooms: Optionally, a set of rooms that the update concerns. If not + given, a database lookup will be done to fetch all the users rooms. + target_users: Optionally, set of users to create profile update stream rows + for. If not given, a database lookup will be done based on `user_rooms`, + or if that is not set, the result of the rooms lookup. + + Returns: + The latest stream ID created in this transaction + """ + if not self._include_profile_updates_in_sync: + return None + + if action == ProfileUpdateAction.UPDATE: + assert field_names + else: + assert not field_names + + if not target_users: + if not user_rooms: + rows = self.db_pool.simple_select_onecol_txn( + txn=txn, + table="current_state_events", + keyvalues={ + "type": EventTypes.Member, + "membership": Membership.JOIN, + "state_key": user_id.to_string(), + }, + retcol="room_id", + ) + user_rooms = set(rows) + + rows = self.db_pool.simple_select_many_txn( + txn=txn, + table="local_current_membership", + column="room_id", + iterable=user_rooms, + retcols=("user_id",), + keyvalues={ + "membership": Membership.JOIN, + }, + ) + target_users = {row[0] for row in rows} + + # Ensure we only write updates for local users + users = {user for user in target_users if self.hs.is_mine_id(user)} + + if action in (ProfileUpdateAction.JOINED_ROOM, ProfileUpdateAction.LEFT_ROOM): + users.discard(user_id.to_string()) + if not users: + # No point writing an update for ourselves, if a membership change and no + # other users interested + return None + elif action == ProfileUpdateAction.UPDATE: + # Always include ourselves when updating field values + users.add(user_id.to_string()) + + # Record the profile update + inserted_ts = self.clock.time_msec() + stream_id = self._profile_updates_id_gen.get_next_txn(txn) + + self.db_pool.simple_insert_txn( + txn, + table="profile_updates", + values={ + "stream_id": stream_id, + "instance_name": self._instance_name, + "user_id": user_id.to_string(), + "action": action.value, + "affected_fields": json_encoder.encode(sorted(field_names)) + if field_names + else None, + "inserted_ts": inserted_ts, + }, + ) + + # Add per user tracking rows for each generated stream ID + per_user_values = [(stream_id, user_id, inserted_ts) for user_id in users] + self.db_pool.simple_insert_many_txn( + txn, + table="profile_updates_per_user", + keys=[ + "stream_id", + "user_id", + "inserted_ts", + ], + values=per_user_values, + ) + return stream_id + + async def set_profile_field( + self, + user_id: UserID, + field_name: str, + new_value: JsonValue | dict[str, JsonValue], + ) -> int | None: + """ + Set a custom profile field for a user. + + Args: + user_id: The user's ID. + field_name: The name of the custom profile field. + new_value: The value of the custom profile field. + """ + return await self.db_pool.runInteraction( + "set_profile_field", + self._set_profile_field_txn, + user_id, + field_name, + new_value, + ) + + async def delete_profile_field( + self, + user_id: UserID, + field_name: str, + ) -> int | None: """ Remove a custom profile field for a user. @@ -513,39 +1032,68 @@ class ProfileWorkerStore(SQLBaseStore): field_name: The name of the custom profile field. """ - def delete_profile_field(txn: LoggingTransaction) -> None: + if self._include_profile_updates_in_sync: + assert self._is_events_writer + + def delete_profile_field(txn: LoggingTransaction) -> int | None: if isinstance(self.database_engine, PostgresEngine): - sql = """ - UPDATE profiles SET fields = fields - ? - WHERE user_id = ? - """ txn.execute( - sql, + """ + UPDATE profiles SET fields = fields - ? + WHERE user_id = ? + """, (field_name, user_id.localpart), ) else: - sql = """ - UPDATE profiles SET fields = json_remove(fields, ?) - WHERE user_id = ? - """ txn.execute( - sql, + """ + UPDATE profiles SET fields = json_remove(fields, ?) + WHERE user_id = ? + """, # This will error if field_name has double quotes in it. (f'$."{field_name}"', user_id.localpart), ) - await self.db_pool.runInteraction("delete_profile_field", delete_profile_field) + if not self._include_profile_updates_in_sync: + return None - async def delete_profile(self, user_id: UserID) -> None: + stream_id = self.record_profile_updates_txn( + txn=txn, + user_id=user_id, + action=ProfileUpdateAction.UPDATE, + field_names=[field_name], + ) + return stream_id + + return await self.db_pool.runInteraction( + "delete_profile_field", delete_profile_field + ) + + async def delete_profile( + self, + user_id: UserID, + ) -> None: """ - Deletes an entire user profile, including displayname, avatar_url and all custom fields. - Used at user deactivation when erasure is requested. + Deletes an entire user profile, including displayname, avatar_url and all + custom fields. Used at user deactivation when erasure is requested. + + Args: + user_id: User ID whose profile is going to be deleted. """ - await self.db_pool.simple_delete( - desc="delete_profile", - table="profiles", - keyvalues={"full_user_id": user_id.to_string()}, + def _delete_profile_txn(txn: LoggingTransaction) -> None: + # Delete the profile + txn.execute( + """ + DELETE FROM profiles + WHERE full_user_id = ? + """, + (user_id.to_string(),), + ) + + await self.db_pool.runInteraction( + "delete_profile", + _delete_profile_txn, ) diff --git a/synapse/storage/databases/main/push_rule.py b/synapse/storage/databases/main/push_rule.py index d361166cec..5086f4639b 100644 --- a/synapse/storage/databases/main/push_rule.py +++ b/synapse/storage/databases/main/push_rule.py @@ -19,6 +19,7 @@ # # import logging +from http import HTTPStatus from typing import ( TYPE_CHECKING, Any, @@ -31,7 +32,7 @@ from typing import ( from twisted.internet import defer -from synapse.api.errors import StoreError +from synapse.api.errors import Codes, StoreError, SynapseError from synapse.config.homeserver import ExperimentalConfig from synapse.logging.context import make_deferred_yieldable, run_in_background from synapse.replication.tcp.streams import PushRulesStream @@ -112,6 +113,22 @@ def _load_rules( return filtered_rules +def _push_rule_size_for_limits(*, conditions_json: str, actions_json: str) -> int: + """ + Returns the size of a push rule, as used for applying the size limit. + + We aren't tied to any particular definition, but currently this is + simply the size in bytes of the conditions and actions JSON added together, + so not rocket science, but this function provides a 'label' for it. + """ + # This is not a very predictable way of calculating the size from the + # point of view of the client, but since it's an out-of-spec limit + # entirely at our discretion, we don't really have to worry about + # the exact calculation. + # FIXME: Spec a predictable push rule size limit + return len(conditions_json.encode("utf-8")) + len(actions_json.encode("utf-8")) + + class PushRulesWorkerStore( ApplicationServiceWorkerStore, PusherWorkerStore, @@ -170,6 +187,8 @@ class PushRulesWorkerStore( self._push_rule_id_gen = IdGenerator(db_conn, "push_rules", "id") self._push_rules_enable_id_gen = IdGenerator(db_conn, "push_rules_enable", "id") + self._config = hs.config.push_rules + def get_max_push_rules_stream_id(self) -> int: """Get the position of the push rules stream. @@ -409,6 +428,29 @@ class PushRulesWorkerStore( conditions_json = json_encoder.encode(conditions) actions_json = json_encoder.encode(actions) + + rule_id_len = len(rule_id.encode("utf-8")) + if rule_id_len > self._config.limits.rule_id_length: + raise SynapseError( + HTTPStatus.REQUEST_ENTITY_TOO_LARGE, + f"Push rule ID length exceeds server limit ({rule_id_len} bytes > {self._config.limits.rule_id_length} bytes).", + # FIXME: Provide a better error code. + # None of the existing options seem entirely correct, though. + Codes.UNKNOWN, + ) + + rule_body_size = _push_rule_size_for_limits( + conditions_json=conditions_json, actions_json=actions_json + ) + if rule_body_size > self._config.limits.rule_size: + raise SynapseError( + HTTPStatus.REQUEST_ENTITY_TOO_LARGE, + f"Push rule size exceeds server limit ({rule_body_size} bytes > {self._config.limits.rule_size} bytes).", + # FIXME: Provide a better error code. + # None of the existing options seem entirely correct, though. + Codes.UNKNOWN, + ) + async with self._push_rules_stream_id_gen.get_next() as stream_id: event_stream_ordering = self._stream_id_gen.get_current_token() @@ -578,13 +620,19 @@ class PushRulesWorkerStore( actions_json: str, update_stream: bool = True, ) -> None: + """Specialised version of simple_upsert_txn that picks a push_rule_id + using the _push_rule_id_gen if it needs to insert the rule. + + Preconditions: + - this worker is a push writer + - the "push_rules" table is locked + - the push rule has already been validated, + including for rule ID length and rule body size. + """ + if not self._is_push_writer: raise Exception("Not a push writer") - """Specialised version of simple_upsert_txn that picks a push_rule_id - using the _push_rule_id_gen if it needs to insert the rule. It assumes - that the "push_rules" table is locked""" - sql = ( "UPDATE push_rules" " SET priority_class = ?, priority = ?, conditions = ?, actions = ?" @@ -597,6 +645,27 @@ class PushRulesWorkerStore( ) if txn.rowcount == 0: + # About to add a new rule, so check our limits first. + txn.execute( + """ + SELECT COUNT(*) FROM push_rules + WHERE user_name = ? + """, + (user_id,), + ) + (num_push_rules,) = cast(tuple[int], txn.fetchone()) + if num_push_rules >= self._config.limits.rule_count: + raise SynapseError( + HTTPStatus.BAD_REQUEST, + f"Creating a push rule would exceed the limit on the number of push rules associated with your account ({num_push_rules + 1} rules > {self._config.limits.rule_count} rules)", + # FIXME: Provide a better error code, especially for this case. + # None of the existing options seem entirely correct, though. + # `M_USER_LIMIT_EXCEEDED` comes closest but needs an `info_uri`. + # Should follow up and add a built-in error page template, similarly to + # https://github.com/element-hq/synapse/pull/18876 ? + Codes.UNKNOWN, + ) + # We didn't update a row with the given rule_id so insert one push_rule_id = self._push_rule_id_gen.get_next() @@ -839,6 +908,28 @@ class PushRulesWorkerStore( ) else: try: + # Before updating the push rule, we need to check that we won't exceed + # the size limit on push rules. + # For that, we need to fetch the `conditions` JSON. + conditions_json = self.db_pool.simple_select_one_onecol_txn( + txn, + "push_rules", + {"user_name": user_id, "rule_id": rule_id}, + "conditions", + ) + + rule_body_size = _push_rule_size_for_limits( + conditions_json=conditions_json, actions_json=actions_json + ) + if rule_body_size > self._config.limits.rule_size: + raise SynapseError( + HTTPStatus.REQUEST_ENTITY_TOO_LARGE, + f"Push rule size exceeds server limit ({rule_body_size} bytes > {self._config.limits.rule_size} bytes).", + # FIXME: Provide a better error code. + # None of the existing options seem entirely correct, though. + Codes.UNKNOWN, + ) + self.db_pool.simple_update_one_txn( txn, "push_rules", diff --git a/synapse/storage/databases/main/roommember.py b/synapse/storage/databases/main/roommember.py index 736f3e4c78..72ac8c7ed2 100644 --- a/synapse/storage/databases/main/roommember.py +++ b/synapse/storage/databases/main/roommember.py @@ -25,6 +25,7 @@ from typing import ( AbstractSet, Collection, Iterable, + Literal, Mapping, Sequence, cast, @@ -844,63 +845,111 @@ class RoomMemberWorkerStore(EventsWorkerStore, CacheInvalidationWorkerStore): @cached(max_entries=10000) async def does_pair_of_users_share_a_room( - self, user_id: str, other_user_id: str + self, + user_id: str, + other_user_id: str, + exclude_room_ids: list[str] | None = None, ) -> bool: raise NotImplementedError() - @cachedList( - cached_method_name="does_pair_of_users_share_a_room", list_name="other_user_ids" - ) - async def _do_users_share_a_room( - self, user_id: str, other_user_ids: Collection[str] - ) -> Mapping[str, bool | None]: + def do_users_share_a_room_txn( + self, + txn: LoggingTransaction, + user_id: str, + other_user_ids: Collection[str], + exclude_room_id: str | None = None, + ) -> dict[str, Literal[True]]: """Return mapping from user ID to whether they share a room with the given user. - Note: `None` and `False` are equivalent and mean they don't share a - room. + Optionally, exclude a room when querying the database. + + Users sharing rooms get `True` returned, users who don't are omitted from the return. + (This is for friendliness with `cachedList` on `_do_users_share_a_room`) """ + state_key_clause, state_key_args = make_in_list_sql_clause( + self.database_engine, "state_key", other_user_ids + ) + # Build SQL args based on whether we are excluding a room ID or not + exclude_room_id_clause = "" + exclude_room_id_args: tuple[str, ...] = () - def do_users_share_a_room_txn( - txn: LoggingTransaction, user_ids: Collection[str] - ) -> dict[str, bool]: - clause, args = make_in_list_sql_clause( - self.database_engine, "state_key", user_ids - ) + if exclude_room_id: + exclude_room_id_clause = "AND room_id != ?" + exclude_room_id_args = (exclude_room_id,) - # This query works by fetching both the list of rooms for the target - # user and the set of other users, and then checking if there is any - # overlap. - sql = f""" + # This query works by fetching both the list of rooms for the target + # user and the set of other users, and then checking if there is any + # overlap. + txn.execute( + f""" SELECT DISTINCT b.state_key FROM ( SELECT room_id FROM current_state_events - WHERE type = 'm.room.member' AND membership = 'join' AND state_key = ? + WHERE type = 'm.room.member' + AND membership = 'join' + AND state_key = ? + {exclude_room_id_clause} ) AS a INNER JOIN ( SELECT room_id, state_key FROM current_state_events - WHERE type = 'm.room.member' AND membership = 'join' AND {clause} - ) AS b using (room_id) - """ + WHERE type = 'm.room.member' + AND membership = 'join' + AND {state_key_clause} + {exclude_room_id_clause} + ) AS b USING (room_id) + """, + ( + user_id, + *exclude_room_id_args, + *state_key_args, + *exclude_room_id_args, + ), + ) + return {u: True for (u,) in txn} - txn.execute(sql, (user_id, *args)) - return {u: True for (u,) in txn} + @cachedList( + cached_method_name="does_pair_of_users_share_a_room", + list_name="other_user_ids", + ) + async def _do_users_share_a_room( + self, + user_id: str, + other_user_ids: Collection[str], + exclude_room_id: str | None = None, + ) -> Mapping[str, Literal[True] | None]: + """Return mapping from user ID to whether they share a room with the + given user. + + Optionally, exclude a room when querying the database. + + This returns `True` for users that share a room and `None` for users that don't. + (This is because of the `cachedList` annotation.) + """ to_return = {} for batch_user_ids in batch_iter(other_user_ids, 1000): res = await self.db_pool.runInteraction( - "do_users_share_a_room", do_users_share_a_room_txn, batch_user_ids + "do_users_share_a_room", + self.do_users_share_a_room_txn, + user_id, + batch_user_ids, + exclude_room_id, ) to_return.update(res) return to_return async def do_users_share_a_room( - self, user_id: str, other_user_ids: Collection[str] + self, + user_id: str, + other_user_ids: Collection[str], + exclude_room_id: str | None = None, ) -> set[str]: """Return the set of users who share a room with the first users""" - - user_dict = await self._do_users_share_a_room(user_id, other_user_ids) + user_dict = await self._do_users_share_a_room( + user_id, other_user_ids, exclude_room_id + ) return {u for u, share_room in user_dict.items() if share_room} @@ -971,17 +1020,55 @@ class RoomMemberWorkerStore(EventsWorkerStore, CacheInvalidationWorkerStore): return {u for u, share_room in user_dict.items() if share_room} - async def get_users_who_share_room_with_user(self, user_id: str) -> set[str]: - """Returns the set of users who share a room with `user_id`""" + async def get_users_who_share_room_with_user( + self, user_id: str, excluded_rooms: AbstractSet[str] = frozenset() + ) -> set[str]: + """Returns the set of users who share a room with `user_id`. + + Args: + user_id: The user to find the co-occupants of. + excluded_rooms: Rooms which should not, on their own, count as a + shared room. + """ room_ids = await self.get_rooms_for_user(user_id) user_who_share_room: set[str] = set() for room_id in room_ids: + if room_id in excluded_rooms: + continue user_ids = await self.get_users_in_room(room_id) user_who_share_room.update(user_ids) return user_who_share_room + async def get_local_users_who_share_room_with_user( + self, + user_id: str, + limit_to_rooms: set[str] | None = None, + ) -> set[str]: + """ + Returns the set of local users who share a room with `user_id`. + + This also includes the `user_id` themselves. + + Args: + user_id: The user ID to find the local users who share rooms. + limit_to_rooms: Optional set of rooms to limit to. + + Returns: + Set of local user ID's who share a room with the given user. + """ + room_ids = await self.get_rooms_for_user(user_id) + if limit_to_rooms is not None: + room_ids = room_ids.intersection(limit_to_rooms) + + user_who_share_room: set[str] = set() + for room_id in room_ids: + user_ids = await self.get_local_users_in_room(room_id) + user_who_share_room.update(user_ids) + + return user_who_share_room + @cached(cache_context=True, iterable=True) async def get_mutual_rooms_between_users( self, user_ids: frozenset[str], cache_context: _CacheContext diff --git a/synapse/storage/databases/main/search.py b/synapse/storage/databases/main/search.py index d6eace5efa..b350978d34 100644 --- a/synapse/storage/databases/main/search.py +++ b/synapse/storage/databases/main/search.py @@ -208,9 +208,7 @@ class SearchBackgroundUpdateStore(SearchWorkerStore): value = content["body"] elif etype == "m.room.topic": key = "content.topic" - value = ( - get_plain_text_topic_from_event_content(content) or "", - ) + value = get_plain_text_topic_from_event_content(content) or "" elif etype == "m.room.name": key = "content.name" value = content["name"] diff --git a/synapse/storage/databases/state/deletion.py b/synapse/storage/databases/state/deletion.py index 23150e8626..9f08f570d8 100644 --- a/synapse/storage/databases/state/deletion.py +++ b/synapse/storage/databases/state/deletion.py @@ -14,6 +14,7 @@ import contextlib +import logging from typing import ( TYPE_CHECKING, AbstractSet, @@ -30,11 +31,14 @@ from synapse.storage.database import ( make_in_list_sql_clause, ) from synapse.storage.engines import PostgresEngine +from synapse.util.duration import Duration from synapse.util.stringutils import shortstr if TYPE_CHECKING: from synapse.server import HomeServer +logger = logging.getLogger(__name__) + class StateDeletionDataStore: """Manages deletion of state groups in a safe manner. @@ -81,6 +85,12 @@ class StateDeletionDataStore: # event will fail to persist (as well as any event in the same batch). DELAY_BEFORE_DELETION_MS = 10 * 60 * 1000 + # How old a row in `state_groups_persisting` has to be before we assume the + # persist that wrote it has gone away. This should be much longer than any + # persist can take. If we clear the row of a live persist, then its state + # groups can be deleted while it is still using them. + STALE_PERSISTING_DURATION = Duration(days=7) + def __init__( self, database: DatabasePool, @@ -91,17 +101,35 @@ class StateDeletionDataStore: self.db_pool = database self._instance_name = hs.get_instance_name() - with db_conn.cursor(txn_name="_clear_existing_persising") as txn: - self._clear_existing_persising(txn) + with db_conn.cursor(txn_name="_clear_existing_persisting") as txn: + self._clear_existing_persisting(txn) - def _clear_existing_persising(self, txn: LoggingTransaction) -> None: + def _clear_existing_persisting(self, txn: LoggingTransaction) -> None: """On startup we clear any entries in `state_groups_persisting` that - match our instance name, in case of a previous unclean shutdown""" + match our instance name (or are very old), in case of a previous unclean + shutdown.""" - self.db_pool.simple_delete_txn( - txn, - table="state_groups_persisting", - keyvalues={"instance_name": self._instance_name}, + # Delete any rows that are very old, or that match our instance name. We + # clear all stale rows, even if they don't match our instance name, as + # we don't know if the instance that created them is still running. + cutoff = self._clock.time_msec() - self.STALE_PERSISTING_DURATION.as_millis() + sql = """ + DELETE FROM state_groups_persisting + WHERE inserted_ts < ? OR (instance_name = ?) + RETURNING state_group + """ + txn.execute(sql, (cutoff, self._instance_name)) + + # Two instances can each have a stale row for the same state group. + state_groups = {state_group for (state_group,) in txn} + + if not state_groups: + return + + logger.info( + "Cleared %d stale state groups from state_groups_persisting: %s", + len(state_groups), + shortstr(state_groups), ) async def check_state_groups_and_bump_deletion( @@ -277,11 +305,20 @@ class StateDeletionDataStore: f"state groups have been deleted: {shortstr(missing_state_groups)}" ) - self.db_pool.simple_insert_many_txn( + # There is a unique key on (state_group, instance_name) so we need to + # handle the case where we have already marked a state group as being + # persisted. This can happen if we fail to persist an event and then + # retry. + now = self._clock.time_msec() + self.db_pool.simple_upsert_many_txn( txn, table="state_groups_persisting", - keys=("state_group", "instance_name"), - values=[(state_group, self._instance_name) for state_group in state_groups], + key_names=("state_group", "instance_name"), + key_values=[ + (state_group, self._instance_name) for state_group in state_groups + ], + value_names=("inserted_ts",), + value_values=[(now,) for _ in state_groups], ) def _finish_persisting_txn( diff --git a/synapse/storage/engines/postgres.py b/synapse/storage/engines/postgres.py index 7cd50fb8f1..225cd45e21 100644 --- a/synapse/storage/engines/postgres.py +++ b/synapse/storage/engines/postgres.py @@ -65,6 +65,24 @@ class PostgresEngine( self.statement_timeout: int | None = database_config.get( "statement_timeout", Duration(minutes=10).as_millis() ) + + # Abort transactions that sit idle for too long. + # + # Idle transactions can block maintenance tasks server-side like + # vacuums, which can lead to bloat and performance issues. + # + # We should never hit this timeout in normal operation, as Synapse + # should always be actively using the connection when in a transaction + # and so it should only ever be briefly idle. If we do hit this timeout, + # it's likely that no progress is being made and so aborting the session + # is safe. + # + # In certain cases we have seen connections leak, particularly when + # using a connection pooler like pgcat, and this timeout will help with + # that. + self.idle_in_transaction_session_timeout: int | None = database_config.get( + "idle_in_transaction_session_timeout", Duration(minutes=30).as_millis() + ) self._version: int | None = None # unknown as yet self.isolation_level_map: Mapping[int, int] = { @@ -187,6 +205,14 @@ class PostgresEngine( if self.statement_timeout is not None: cursor.execute("SET statement_timeout TO ?", (self.statement_timeout,)) + # Abort transactions that sit idle for too long, as they hold locks + # and block vacuum. + if self.idle_in_transaction_session_timeout is not None: + cursor.execute( + "SET idle_in_transaction_session_timeout TO ?", + (self.idle_in_transaction_session_timeout,), + ) + cursor.close() db_conn.commit() diff --git a/synapse/storage/prepare_database.py b/synapse/storage/prepare_database.py index 2def1e130c..979d57ca06 100644 --- a/synapse/storage/prepare_database.py +++ b/synapse/storage/prepare_database.py @@ -100,17 +100,19 @@ def prepare_database( """Prepares a physical database for usage. Will either create all necessary tables or upgrade from an older schema version. - If `config` is None then prepare_database will assert that no upgrade is - necessary, *or* will create a fresh database if the database is empty. - Args: db_conn: database_engine: - config : - application config, or None if we are connecting to an existing - database which we expect to be configured already + config: + application config, or `None` if we are initialising a new empty/blank + database. Note that anything which requires the config, like module schemas, + is skipped when `None` is passed. databases: The name of the databases that will be used with this physical database. Defaults to all databases. + Raises: + ValueError: Passing `config=None` when a database already has a schema raises + `ValueError`, as we can't upgrade an existing database without the config. + UpgradeDatabaseException: If you try to initialize a new database from a worker. """ try: diff --git a/synapse/storage/schema/__init__.py b/synapse/storage/schema/__init__.py index 1afc6d0b2a..926121d8e1 100644 --- a/synapse/storage/schema/__init__.py +++ b/synapse/storage/schema/__init__.py @@ -175,6 +175,8 @@ Changes in SCHEMA_VERSION = 93 Changes in SCHEMA_VERSION = 94 - Add `recheck` column (boolean, default true) to the `redactions` table. - MSC4242: Add state DAG tables. + - MSC4429/MSC4262: Track updates to user profile fields via a new stream. + - Add an `inserted_ts` column to the `state_groups_persisting` table. """ diff --git a/synapse/storage/schema/main/delta/88/01_add_delayed_events.sql b/synapse/storage/schema/main/delta/88/01_add_delayed_events.sql index 78ba5129af..4abe0ccaf4 100644 --- a/synapse/storage/schema/main/delta/88/01_add_delayed_events.sql +++ b/synapse/storage/schema/main/delta/88/01_add_delayed_events.sql @@ -22,6 +22,8 @@ CREATE TABLE delayed_events ( state_key TEXT, origin_server_ts BIGINT, content bytea NOT NULL, + -- is_processed = TRUE means that the work of sending the delayed event has begun. + -- Once the send is complete, the delayed event is removed from this table. is_processed BOOLEAN NOT NULL DEFAULT FALSE, PRIMARY KEY (user_localpart, delay_id) ); diff --git a/synapse/storage/schema/main/delta/94/07_profile_updates.sql b/synapse/storage/schema/main/delta/94/07_profile_updates.sql new file mode 100644 index 0000000000..e053421be3 --- /dev/null +++ b/synapse/storage/schema/main/delta/94/07_profile_updates.sql @@ -0,0 +1,60 @@ +-- +-- This file is licensed under the Affero General Public License (AGPL) version 3. +-- +-- Copyright (C) 2026 Element Creations Ltd. +-- +-- This program is free software: you can redistribute it and/or modify +-- it under the terms of the GNU Affero General Public License as +-- published by the Free Software Foundation, either version 3 of the +-- License, or (at your option) any later version. +-- +-- See the GNU Affero General Public License for more details: +-- . + +-- Track updates to profile fields. +-- For MSC4429 and MSC4262 down sync and others. +-- See https://github.com/element-hq/synapse/issues/19981 for potential future directions of this table. +CREATE TABLE IF NOT EXISTS profile_updates ( + stream_id BIGINT NOT NULL PRIMARY KEY, + instance_name TEXT NOT NULL, + + -- The full user ID + user_id TEXT NOT NULL, + + -- Profile action that has happened, see ProfileUpdateAction enum. + action TEXT NOT NULL, + + -- JSON array of the profile field names that have been + -- added, updated or removed in this update. + -- See https://spec.matrix.org/unstable/client-server-api/#profiles + -- This is only present if `action` is `update`. + -- + -- We support multiple field updates at once because it is easy to foresee features + -- involving multiple fields (where getting the illusion of a torn write might be harmful), + -- as well as synchronisation over federation being likely to lead to multiple field changes + -- at once. + affected_fields JSONB NULL, + + -- Unix timestamp (milliseconds) for debugging purposes + inserted_ts BIGINT NOT NULL +); + +CREATE INDEX IF NOT EXISTS profile_updates_by_user ON profile_updates (user_id, stream_id); + +-- We aren't creating a GIN index on `affected_fields` at this time because we don't expect +-- field names to be very selective and therefore an index might not be that useful. + +-- Track which local users should receive each profile update. +CREATE TABLE IF NOT EXISTS profile_updates_per_user ( + -- Stream ID reference to `profile_updates` + stream_id BIGINT NOT NULL REFERENCES profile_updates (stream_id), + + -- The full user ID of the local user that should receive the profile update. + user_id TEXT NOT NULL, + + -- Unix timestamp (milliseconds). Used to determine when to prune rows (to prevent the table + -- from growing indefinitely). + inserted_ts BIGINT NOT NULL, + + PRIMARY KEY (user_id, stream_id) +); diff --git a/synapse/storage/schema/main/delta/94/07_profile_updates_seq.sql.postgres b/synapse/storage/schema/main/delta/94/07_profile_updates_seq.sql.postgres new file mode 100644 index 0000000000..9abf79b68d --- /dev/null +++ b/synapse/storage/schema/main/delta/94/07_profile_updates_seq.sql.postgres @@ -0,0 +1,18 @@ +-- +-- This file is licensed under the Affero General Public License (AGPL) version 3. +-- +-- Copyright (C) 2026 Element Creations Ltd. +-- +-- This program is free software: you can redistribute it and/or modify +-- it under the terms of the GNU Affero General Public License as +-- published by the Free Software Foundation, either version 3 of the +-- License, or (at your option) any later version. +-- +-- See the GNU Affero General Public License for more details: +-- . + +CREATE SEQUENCE profile_updates_sequence; +-- Synapse streams start at 2, because the default position is 1 +-- so any item inserted at position 1 is ignored. +-- We have to use nextval not START WITH 2, see https://github.com/element-hq/synapse/issues/18712 +SELECT nextval('profile_updates_sequence'); diff --git a/synapse/storage/schema/main/delta/94/08_device_lists_changes_in_room_unconverted_idx.sql b/synapse/storage/schema/main/delta/94/08_device_lists_changes_in_room_unconverted_idx.sql new file mode 100644 index 0000000000..6bebcab11d --- /dev/null +++ b/synapse/storage/schema/main/delta/94/08_device_lists_changes_in_room_unconverted_idx.sql @@ -0,0 +1,22 @@ +-- +-- This file is licensed under the Affero General Public License (AGPL) version 3. +-- +-- Copyright (C) 2026 Element Creations Ltd +-- +-- This program is free software: you can redistribute it and/or modify +-- it under the terms of the GNU Affero General Public License as +-- published by the Free Software Foundation, either version 3 of the +-- License, or (at your option) any later version. +-- +-- See the GNU Affero General Public License for more details: +-- . + + +-- Add an index on `device_lists_changes_in_room(user_id, device_id, room_id, +-- stream_id)` for unconverted rows, so that marking redundant rows as +-- converted (in `mark_redundant_device_lists_pokes`) does not require a scan +-- of the unconverted backlog. +-- +-- This is a partial index as we only ever query for unconverted rows. +INSERT INTO background_updates (ordering, update_name, progress_json) VALUES + (9408, 'device_lists_changes_in_room_unconverted_idx', '{}'); diff --git a/synapse/storage/schema/main/delta/94/09_e2e_cross_signing_signatures_index.sql b/synapse/storage/schema/main/delta/94/09_e2e_cross_signing_signatures_index.sql new file mode 100644 index 0000000000..cde6ca1500 --- /dev/null +++ b/synapse/storage/schema/main/delta/94/09_e2e_cross_signing_signatures_index.sql @@ -0,0 +1,21 @@ +-- +-- This file is licensed under the Affero General Public License (AGPL) version 3. +-- +-- Copyright (C) 2026 New Vector, Ltd +-- +-- This program is free software: you can redistribute it and/or modify +-- it under the terms of the GNU Affero General Public License as +-- published by the Free Software Foundation, either version 3 of the +-- License, or (at your option) any later version. +-- +-- See the GNU Affero General Public License for more details: +-- . + +-- Remove any rows `e2e_cross_signing_signatures` that have duplicate cross-signing signatures. +-- Ensures that rows are unique across `(user_id, target_user_id, target_device_id, key_id)`, so that +-- we can create an index on those columns. See +-- `./10_e2e_cross_signing_signatures_add_key_id_to_index.sql`, which adds said +-- index. + +INSERT INTO background_updates (ordering, update_name, progress_json) VALUES + (9409, 'e2e_cross_signing_signatures_remove_duplicates', '{}'); diff --git a/synapse/storage/schema/main/delta/94/10_e2e_cross_signing_signatures_add_key_id_to_index.sql b/synapse/storage/schema/main/delta/94/10_e2e_cross_signing_signatures_add_key_id_to_index.sql new file mode 100644 index 0000000000..65b5df6c86 --- /dev/null +++ b/synapse/storage/schema/main/delta/94/10_e2e_cross_signing_signatures_add_key_id_to_index.sql @@ -0,0 +1,18 @@ +-- +-- This file is licensed under the Affero General Public License (AGPL) version 3. +-- +-- Copyright (C) 2026 New Vector, Ltd +-- +-- This program is free software: you can redistribute it and/or modify +-- it under the terms of the GNU Affero General Public License as +-- published by the Free Software Foundation, either version 3 of the +-- License, or (at your option) any later version. +-- +-- See the GNU Affero General Public License for more details: +-- . + +-- Adds the `key_id` to the e2e_cross_signing_signatures index, since the +-- ("user_id", "key_id", "target_user_id", "target_device_id") should be +-- unique. +INSERT INTO background_updates (ordering, update_name, progress_json) VALUES + (9410, 'e2e_cross_signing_signatures_add_key_id_to_index', '{}'); \ No newline at end of file diff --git a/synapse/storage/schema/state/delta/94/01_state_groups_persisting_inserted_ts.sql.postgres b/synapse/storage/schema/state/delta/94/01_state_groups_persisting_inserted_ts.sql.postgres new file mode 100644 index 0000000000..4e5c0eea96 --- /dev/null +++ b/synapse/storage/schema/state/delta/94/01_state_groups_persisting_inserted_ts.sql.postgres @@ -0,0 +1,22 @@ +-- +-- This file is licensed under the Affero General Public License (AGPL) version 3. +-- +-- Copyright (C) 2026 Element Creations Ltd +-- +-- This program is free software: you can redistribute it and/or modify +-- it under the terms of the GNU Affero General Public License as +-- published by the Free Software Foundation, either version 3 of the +-- License, or (at your option) any later version. +-- +-- See the GNU Affero General Public License for more details: +-- . + +-- Record when we marked a state group as being persisted, so that we can clear +-- out rows left behind by a persist that never finished. +-- +-- The rows already in the table are stamped with the time this runs, so they get +-- the same grace period as a new row before we treat them as stale. The default +-- covers instances that haven't been upgraded yet, which insert without the +-- column, and is what lets the column be NOT NULL during a rolling upgrade. +ALTER TABLE state_groups_persisting + ADD COLUMN inserted_ts BIGINT NOT NULL DEFAULT extract(epoch from now()) * 1000; diff --git a/synapse/storage/schema/state/delta/94/01_state_groups_persisting_inserted_ts.sql.sqlite b/synapse/storage/schema/state/delta/94/01_state_groups_persisting_inserted_ts.sql.sqlite new file mode 100644 index 0000000000..7720a814a5 --- /dev/null +++ b/synapse/storage/schema/state/delta/94/01_state_groups_persisting_inserted_ts.sql.sqlite @@ -0,0 +1,31 @@ +-- +-- This file is licensed under the Affero General Public License (AGPL) version 3. +-- +-- Copyright (C) 2026 Element Creations Ltd +-- +-- This program is free software: you can redistribute it and/or modify +-- it under the terms of the GNU Affero General Public License as +-- published by the Free Software Foundation, either version 3 of the +-- License, or (at your option) any later version. +-- +-- See the GNU Affero General Public License for more details: +-- . + +-- Record when we marked a state group as being persisted, so that we can clear +-- out rows left behind by a persist that never finished. +-- +-- On SQLite we must be in monolith mode, so every row already in the table was +-- written by this process before it restarted, and is dead. Zero marks them as +-- stale straight away, which is what we want. Contrast Postgres, where such a +-- row may belong to a worker that is still running and so gets a grace period. +-- +-- Note that this also correctly handles the case of Synapse version rolling +-- back, as new rows will be written with a 0 inserted_ts, *but* since its +-- running with the old code nothing deletes those row. On upgrade the restart +-- will mean any rows are automatically stale as above. +-- +-- The default only applies to an insert that omits the column, which our own +-- code never does. SQLite only accepts a constant default in ADD COLUMN, so it +-- could not be the current time in any case. +ALTER TABLE state_groups_persisting + ADD COLUMN inserted_ts BIGINT NOT NULL DEFAULT 0; diff --git a/synapse/storage/util/id_generators.py b/synapse/storage/util/id_generators.py index c9c339b235..d294119bb2 100644 --- a/synapse/storage/util/id_generators.py +++ b/synapse/storage/util/id_generators.py @@ -906,14 +906,44 @@ class _MultiWriterCtxManager: stream_ids: list[int] = attr.Factory(list) async def __aenter__(self) -> int | list[int]: + def _load(txn: LoggingTransaction) -> list[int]: + ids = self.id_gen._load_next_mult_id_txn(txn, self.multiple_ids or 1) + # Record the allocated IDs on the context manager as a side effect + # (rather than only via the return value), so that if this coroutine + # is cancelled after the transaction has committed we still know + # which IDs to release below. + self.stream_ids = ids + return ids + # It's safe to run this in autocommit mode as fetching values from a # sequence ignores transaction semantics anyway. - self.stream_ids = await self.id_gen._db.runInteraction( - "_load_next_mult_id", - self.id_gen._load_next_mult_id_txn, - self.multiple_ids or 1, - db_autocommit=True, - ) + try: + await self.id_gen._db.runInteraction( + "_load_next_mult_id", + _load, + db_autocommit=True, + ) + except BaseException: + # We catch `BaseException` rather than `Exception`, + # because request cancellation surfaces here as exceptions that are + # not `Exception` subclasses: `asyncio.CancelledError` + # and `GeneratorExit` (raised when a paused coroutine is garbage + # collected). + # + # If we're interrupted (e.g. the enclosing request was cancelled) + # after the transaction allocated the IDs but before we returned, + # then `__aexit__` will never run, because Python only invokes it + # once `__aenter__` has returned. The allocated IDs would then be + # leaked into `_unfinished_ids` forever, permanently pinning the + # persisted stream position and, e.g., wedging presence. + # + # So mark them as finished here to unblock the position. This mirrors + # what `__aexit__` does on the failure path (marking the IDs finished + # and notifying replication, but not persisting a new position). + if self.stream_ids: + self.id_gen._mark_ids_as_finished(self.stream_ids) + self.notifier.notify_replication() + raise if self.multiple_ids is None: return self.stream_ids[0] * self.id_gen._return_factor diff --git a/synapse/storage/util/sequence.py b/synapse/storage/util/sequence.py index 5bee3cf34f..0f3cfa7a7e 100644 --- a/synapse/storage/util/sequence.py +++ b/synapse/storage/util/sequence.py @@ -41,7 +41,7 @@ Postgres sequence '%(seq)s' is inconsistent with associated stream position of '%(stream_name)s' in the 'stream_positions' table. This is likely a programming error and should be reported at -https://github.com/matrix-org/synapse. +https://github.com/element-hq/synapse. A temporary workaround to fix this error is to shut down Synapse (including any and all workers) and run the following SQL: diff --git a/synapse/streams/events.py b/synapse/streams/events.py index 36490fcb35..24120eb736 100644 --- a/synapse/streams/events.py +++ b/synapse/streams/events.py @@ -86,6 +86,7 @@ class EventSources: thread_subscriptions_key = self.store.get_max_thread_subscriptions_stream_id() sticky_events_key = self.store.get_max_sticky_events_stream_id() quarantined_media_key = self.store.get_quarantined_media_stream_token() + profile_updates_key = self.store.get_max_profile_updates_stream_id() token = StreamToken( room_key=self.sources.room.get_current_key(), @@ -102,6 +103,7 @@ class EventSources: thread_subscriptions_key=thread_subscriptions_key, sticky_events_key=sticky_events_key, quarantined_media_key=quarantined_media_key, + profile_updates_key=profile_updates_key, ) return token @@ -131,6 +133,7 @@ class EventSources: StreamKeyType.THREAD_SUBSCRIPTIONS: self.store.get_thread_subscriptions_stream_id_generator(), StreamKeyType.STICKY_EVENTS: self.store.get_sticky_events_stream_id_generator(), StreamKeyType.QUARANTINED_MEDIA: self.store.get_quarantined_media_stream_id_generator(), + StreamKeyType.PROFILE_UPDATES: self.store.get_profile_updates_stream_id_generator(), } for _, key in StreamKeyType.__members__.items(): diff --git a/synapse/types/__init__.py b/synapse/types/__init__.py index a6fc806701..7516847303 100644 --- a/synapse/types/__init__.py +++ b/synapse/types/__init__.py @@ -37,6 +37,7 @@ from typing import ( MutableMapping, NoReturn, Optional, + Sequence, TypedDict, TypeVar, Union, @@ -102,6 +103,47 @@ JsonMapping = Mapping[str, Any] # A JSON-serialisable object. JsonSerializable = object +StrictJsonValue = Union[ + None, + bool, + int, + float, + str, + "StrictJsonList", + "StrictJsonDict", + "StrictJsonSequence", + "StrictJsonMapping", +] +""" +Type that represents any valid JSON value, recursively. +Does not fall back to `Any` at deeper levels, which makes it more safe than `JsonValue`. + +Can also represent immutable mapping and tuple types. +(Not sure if we would be better splitting them out.) +""" + +StrictJsonList = list["StrictJsonValue"] +""" +Type that represents a list of any valid JSON value. +""" + +StrictJsonDict = dict[str, "StrictJsonValue"] +""" +Type that represents a dict with string keys (as per JSON) and values of any +valid JSON type. +Does not fall back to `Any` at deeper levels, which makes it more safe than `JsonDict`. +""" + +StrictJsonSequence = Sequence["StrictJsonValue"] +""" +Like `StrictJsonList` but using a `Sequence` as the collection type. +""" + +StrictJsonMapping = Mapping[str, "StrictJsonValue"] +""" +Like `StrictJsonDict` but using `Mapping` as the collection type. +""" + # Collection[str] that does not include str itself; str being a Sequence[str] # is very misleading and results in bugs. # @@ -1095,6 +1137,7 @@ class StreamKeyType(Enum): THREAD_SUBSCRIPTIONS = "thread_subscriptions_key" STICKY_EVENTS = "sticky_events_key" QUARANTINED_MEDIA = "quarantined_media_key" + PROFILE_UPDATES = "profile_updates_key" @attr.s(slots=True, frozen=True, auto_attribs=True) @@ -1102,7 +1145,7 @@ class StreamToken: """A collection of keys joined together by underscores in the following order and which represent the position in their respective streams. - ex. `s2633508_17_338_6732159_1082514_541479_274711_265584_1_379_4242_4141_4343` + ex. `s2633508_17_338_6732159_1082514_541479_274711_265584_1_379_4242_4141_4343_4444` 1. `room_key`: `s2633508` which is a `RoomStreamToken` - `RoomStreamToken`'s can also look like `t426-2633508` or `m56~2.58~3.59` - See the docstring for `RoomStreamToken` for more details. @@ -1118,6 +1161,7 @@ class StreamToken: 11. `thread_subscriptions_key`: 4242 12. `sticky_events_key`: 4141 13. `quarantined_media_key`: 4343 + 14. `profile_updates_key`: 4444 You can see how many of these keys correspond to the various fields in a "/sync" response: @@ -1181,6 +1225,7 @@ class StreamToken: quarantined_media_key: MultiWriterStreamToken = attr.ib( validator=attr.validators.instance_of(MultiWriterStreamToken) ) + profile_updates_key: int _SEPARATOR = "_" START: ClassVar["StreamToken"] @@ -1211,6 +1256,7 @@ class StreamToken: thread_subscriptions_key, sticky_events_key, quarantined_media_key, + profile_updates_key, ) = keys return cls( @@ -1231,6 +1277,7 @@ class StreamToken: quarantined_media_key=await MultiWriterStreamToken.parse( store, quarantined_media_key ), + profile_updates_key=int(profile_updates_key), ) except CancelledError: raise @@ -1256,6 +1303,7 @@ class StreamToken: str(self.thread_subscriptions_key), str(self.sticky_events_key), await self.quarantined_media_key.to_string(store), + str(self.profile_updates_key), ] ) @@ -1329,6 +1377,7 @@ class StreamToken: StreamKeyType.UN_PARTIAL_STATED_ROOMS, StreamKeyType.THREAD_SUBSCRIPTIONS, StreamKeyType.STICKY_EVENTS, + StreamKeyType.PROFILE_UPDATES, ], ) -> int: ... @@ -1384,9 +1433,10 @@ class StreamToken: f"typing: {self.typing_key}, receipt: {self.receipt_key}, " f"account_data: {self.account_data_key}, push_rules: {self.push_rules_key}, " f"to_device: {self.to_device_key}, device_list: {self.device_list_key}, " - f"groups: {self.groups_key}, un_partial_stated_rooms: {self.un_partial_stated_rooms_key}," - f"thread_subscriptions: {self.thread_subscriptions_key}, sticky_events: {self.sticky_events_key}" - f"quarantined_media: {self.quarantined_media_key})" + f"groups: {self.groups_key}, un_partial_stated_rooms: {self.un_partial_stated_rooms_key}, " + f"thread_subscriptions: {self.thread_subscriptions_key}, sticky_events: {self.sticky_events_key}, " + f"quarantined_media: {self.quarantined_media_key}, " + f"profile_updates: {self.profile_updates_key})" ) @@ -1404,6 +1454,7 @@ StreamToken.START = StreamToken( thread_subscriptions_key=0, sticky_events_key=0, quarantined_media_key=MultiWriterStreamToken(stream=0), + profile_updates_key=0, ) diff --git a/synapse/types/handlers/sliding_sync.py b/synapse/types/handlers/sliding_sync.py index dd913250ba..f6d30460b5 100644 --- a/synapse/types/handlers/sliding_sync.py +++ b/synapse/types/handlers/sliding_sync.py @@ -442,6 +442,19 @@ class SlidingSyncResult: def __bool__(self) -> bool: return bool(self.room_id_to_sticky_events) + @attr.s(slots=True, frozen=True, auto_attribs=True) + class ProfilesExtension: + """The Profile Updates extension (MSC4262) + + Attributes: + users: map (user_id -> [profile_updates]) + """ + + users: Mapping[str, JsonMapping | None] + + def __bool__(self) -> bool: + return bool(self.users) + to_device: ToDeviceExtension | None = None e2ee: E2eeExtension | None = None account_data: AccountDataExtension | None = None @@ -449,6 +462,7 @@ class SlidingSyncResult: typing: TypingExtension | None = None thread_subscriptions: ThreadSubscriptionsExtension | None = None sticky_events: StickyEventsExtension | None = None + profiles: ProfilesExtension | None = None def __bool__(self) -> bool: """Are there any updates that should be returned immediately to @@ -461,6 +475,7 @@ class SlidingSyncResult: or self.typing or self.thread_subscriptions or self.sticky_events + or self.profiles ) next_pos: SlidingSyncStreamToken @@ -919,6 +934,7 @@ class PerConnectionState: receipts: The status of each room for the receipts stream. room_configs: Map from room_id to the `RoomSyncConfig` of all rooms that we have previously sent down. + account_data: The status of each room for the account_data stream. """ last_used_ts: int | None = None @@ -951,7 +967,12 @@ class PerConnectionState: ) def __len__(self) -> int: - return len(self.rooms) + len(self.receipts) + len(self.room_configs) + return ( + len(self.account_data) + + len(self.rooms) + + len(self.receipts) + + len(self.room_configs) + ) @attr.s(auto_attribs=True) diff --git a/synapse/types/rest/client/__init__.py b/synapse/types/rest/client/__init__.py index a7cb4d8b08..dbd4455b82 100644 --- a/synapse/types/rest/client/__init__.py +++ b/synapse/types/rest/client/__init__.py @@ -26,6 +26,7 @@ from pydantic import ( ConfigDict, Field, GetCoreSchemaHandler, + GetPydanticSchema, StrictBool, StrictInt, StrictStr, @@ -33,7 +34,7 @@ from pydantic import ( field_validator, model_validator, ) -from pydantic_core import CoreSchema, PydanticCustomError +from pydantic_core import CoreSchema from typing_extensions import Annotated, Self from synapse.types import Absent, AbsentType, NonNegativeStrictInt @@ -61,7 +62,7 @@ class AuthenticationData(RequestBodyModel): ClientSecretStr = Annotated[ str, StringConstraints( - pattern="[0-9a-zA-Z.=_-]", + pattern="^[0-9a-zA-Z.=_-]+$", min_length=1, max_length=255, strict=True, @@ -91,25 +92,33 @@ class EmailRequestTokenBody(ThreepidRequestTokenBody): # know the exact spelling (eg. upper and lower case) of address in the database. # Without this, an email stored in the database as "foo@bar.com" would cause # user requests for "FOO@bar.com" to raise a Not Found error. + # + # A ValueError produces a Pydantic error of type "value_error", which + # validate_json_object translates to M_INVALID_PARAM, the errcode the spec + # lists for an invalid address on /account/3pid/email/requestToken: + # https://spec.matrix.org/v1.19/client-server-api/#post_matrixclientv3account3pidemailrequesttoken @field_validator("email") @classmethod def _email_validator(cls, email: StrictStr) -> StrictStr: - try: - return validate_email(email) - except ValueError as e: - # To ensure backward compatibility of HTTP error codes, we return a - # Pydantic error with the custom, unrecognized error type - # "email_custom_err_type" instead of the default error type - # "value_error". This results in the more generic BAD_JSON HTTP - # error instead of the more specific INVALID_PARAM one. - raise PydanticCustomError("email_custom_err_type", str(e), None) from e + return validate_email(email) -ISO3116_1_Alpha_2 = Annotated[str, StringConstraints(pattern="[A-Z]{2}", strict=True)] +ISO3166_1_Alpha_2 = Annotated[ + str, + GetPydanticSchema( + lambda source, handler: pydantic_core.core_schema.custom_error_schema( + pydantic_core.core_schema.str_schema(pattern="[A-Z]{2}", strict=True), + custom_error_type="value_error", + custom_error_context={ + "error": "Not a valid ISO 3166-1 alpha-2 country code" + }, + ) + ), +] class MsisdnRequestTokenBody(ThreepidRequestTokenBody): - country: ISO3116_1_Alpha_2 + country: ISO3166_1_Alpha_2 phone_number: StrictStr @@ -478,6 +487,18 @@ class SlidingSyncBody(RequestBodyModel): limit: NonNegativeStrictInt = 100 since: SlidingSyncStickyEventsToken | AbsentType = Absent + class ProfilesExtension(RequestBodyModel): + """The Profile Updates extension (MSC4262) + + Attributes: + enabled + fields: List of fields to filter upon (optional) + """ + + enabled: StrictBool = False + # Optionally filter on specific fields + fields: list[StrictStr] | AbsentType = Absent + to_device: ToDeviceExtension | None = None e2ee: E2eeExtension | None = None account_data: AccountDataExtension | None = None @@ -489,6 +510,9 @@ class SlidingSyncBody(RequestBodyModel): sticky_events: StickyEventsExtension | AbsentType = Field( Absent, alias="org.matrix.msc4354.sticky_events" ) + profiles: ProfilesExtension | AbsentType = Field( + Absent, alias="org.matrix.msc4262.profiles" + ) conn_id: StrictStr | None = None lists: ( diff --git a/synapse/util/async_helpers.py b/synapse/util/async_helpers.py index 5211bcc8e1..4ef1620a84 100644 --- a/synapse/util/async_helpers.py +++ b/synapse/util/async_helpers.py @@ -391,6 +391,7 @@ T4 = TypeVar("T4") T5 = TypeVar("T5") T6 = TypeVar("T6") T7 = TypeVar("T7") +T8 = TypeVar("T8") @overload @@ -544,6 +545,32 @@ async def gather_optional_coroutines( ]: ... +@overload +async def gather_optional_coroutines( + *coroutines: Unpack[ + tuple[ + Coroutine[Any, Any, T1] | None, + Coroutine[Any, Any, T2] | None, + Coroutine[Any, Any, T3] | None, + Coroutine[Any, Any, T4] | None, + Coroutine[Any, Any, T5] | None, + Coroutine[Any, Any, T6] | None, + Coroutine[Any, Any, T7] | None, + Coroutine[Any, Any, T8] | None, + ] + ], +) -> tuple[ + T1 | None, + T2 | None, + T3 | None, + T4 | None, + T5 | None, + T6 | None, + T7 | None, + T8 | None, +]: ... + + async def gather_optional_coroutines( *coroutines: Unpack[tuple[Coroutine[Any, Any, T1] | None, ...]], ) -> tuple[T1 | None, ...]: diff --git a/synapse/util/task_scheduler.py b/synapse/util/task_scheduler.py index c1790fd3ae..c01ddf7339 100644 --- a/synapse/util/task_scheduler.py +++ b/synapse/util/task_scheduler.py @@ -400,10 +400,10 @@ class TaskScheduler: """Clean old complete or failed jobs to avoid clutter the DB.""" now = self._clock.time_msec() for task in await self._store.get_scheduled_tasks( - statuses=[TaskStatus.FAILED, TaskStatus.COMPLETE], + statuses=[TaskStatus.FAILED, TaskStatus.CANCELLED, TaskStatus.COMPLETE], max_timestamp=now - TaskScheduler.KEEP_TASKS_FOR_MS, ): - # FAILED and COMPLETE tasks should never be running + # FAILED, CANCELLED and COMPLETE tasks should never be running assert task.id not in self._running_tasks await self._store.delete_scheduled_task(task.id) diff --git a/tests/app/test_homeserver_shutdown.py b/tests/app/test_homeserver_shutdown.py index 20d314cb68..1ff07bdcbf 100644 --- a/tests/app/test_homeserver_shutdown.py +++ b/tests/app/test_homeserver_shutdown.py @@ -58,6 +58,11 @@ class HomeserverCleanShutdownTestCase(HomeserverTestCase): homeserver_to_use=SynapseHomeServer, clock=self.clock, ) + # Since we're driving the entire homeserver creation in these tests (something + # normally handled by `HomeserverTestCase` if we defined some `servlets`), make + # sure we get the Rust side instantiated to verify nothing on the Rust side is + # holding onto a Python reference that would prevent shutdown. + self.hs.get_rust_handlers() self.wait_for_background_updates() hs_ref = weakref.ref(self.hs) @@ -109,6 +114,11 @@ class HomeserverCleanShutdownTestCase(HomeserverTestCase): homeserver_to_use=SynapseHomeServer, clock=self.clock, ) + # Since we're driving the entire homeserver creation in these tests (something + # normally handled by `HomeserverTestCase` if we defined some `servlets`), make + # sure we get the Rust side instantiated to verify nothing on the Rust side is + # holding onto a Python reference that would prevent shutdown. + self.hs.get_rust_handlers() # Pump the background updates by a single iteration, just to ensure any extra # resources it uses have been started. diff --git a/tests/appservice/test_appservice.py b/tests/appservice/test_appservice.py index 620c2b907b..94bfdd3fba 100644 --- a/tests/appservice/test_appservice.py +++ b/tests/appservice/test_appservice.py @@ -24,7 +24,11 @@ from unittest.mock import AsyncMock, Mock from twisted.internet import defer -from synapse.appservice import ApplicationService, Namespace +from synapse.appservice import ( + ApplicationService, + Namespace, + Scopes, +) from synapse.types import UserID from tests import unittest @@ -257,3 +261,99 @@ class ApplicationServiceTestCase(unittest.TestCase): ) ) ) + + +class ApplicationServiceScopesTestCase(unittest.TestCase): + def test_has_no_scopes_by_default(self) -> None: + service = ApplicationService( + id="unique_identifier", + sender=UserID.from_string("@as:test"), + token="some_token", + ) + self.assertEqual(len(service.scopes), 0) + self.assertFalse(service.has_scope(Scopes.QUERY_ROOM_MEMBERSHIP)) + + def test_has_valid_scope_if_specified(self) -> None: + service = ApplicationService( + id="unique_identifier", + sender=UserID.from_string("@as:test"), + token="some_token", + scopes=[Scopes.QUERY_ROOM_MEMBERSHIP], + ) + self.assertEqual(len(service.scopes), 1) + self.assertTrue(service.has_scope(Scopes.QUERY_ROOM_MEMBERSHIP)) + + def test_unknown_scope_raises(self) -> None: + with self.assertRaises(ValueError): + ApplicationService( + id="unique_identifier", + sender=UserID.from_string("@as:test"), + token="some_token", + scopes=["does:not:exist"], + ) + + +class ApplicationServiceProxyPrefixTestCase(unittest.TestCase): + """Tests the proxying configuration for application services from MSC4512.""" + + def _make_service(self, **kwargs: Any) -> ApplicationService: + kwargs.setdefault("id", "unique_identifier") + kwargs.setdefault("sender", UserID.from_string("@as:test")) + kwargs.setdefault("token", "some_token") + return ApplicationService(**kwargs) + + def test_proxy_prefix_without_proxy_url_raises(self) -> None: + with self.assertRaises(KeyError): + self._make_service(proxy_url=None, proxy_prefix="rtc/livekit") + + def test_proxy_prefix_with_empty_proxy_url_raises(self) -> None: + with self.assertRaises(ValueError): + self._make_service(proxy_url="", proxy_prefix="rtc/livekit") + + def test_proxy_url_without_proxy_prefix_raises(self) -> None: + with self.assertRaises(KeyError): + self._make_service(proxy_url="http://proxy.example.com") + + def test_proxy_url_with_empty_proxy_prefix_raises(self) -> None: + with self.assertRaises(ValueError): + self._make_service(proxy_url="http://proxy.example.com", proxy_prefix="") + + def test_proxy_prefix_with_proxy_url_is_stored(self) -> None: + service = self._make_service( + proxy_url="http://proxy.example.com", + proxy_prefix="rtc/livekit", + ) + self.assertEqual(service.proxy_prefix, "rtc/livekit") + self.assertEqual(service.proxy_url, "http://proxy.example.com") + + def test_proxy_url_trailing_slash_is_stripped(self) -> None: + service = self._make_service( + proxy_url="http://proxy.example.com/", + proxy_prefix="rtc/livekit", + ) + self.assertEqual(service.proxy_url, "http://proxy.example.com") + + def test_nested_proxy_prefix_is_allowed(self) -> None: + service = self._make_service( + proxy_url="http://proxy.example.com", + proxy_prefix="rtc/livekit/foo", + ) + self.assertEqual(service.proxy_prefix, "rtc/livekit/foo") + + def test_disallowed_proxy_prefix_raises(self) -> None: + with self.assertRaises(ValueError): + self._make_service( + proxy_url="http://proxy.example.com", proxy_prefix="not/allowed" + ) + + def test_no_proxy_prefix_defaults_to_none(self) -> None: + service = self._make_service() + self.assertIsNone(service.proxy_prefix) + self.assertIsNone(service.proxy_url) + + def test_trailing_slash_on_proxy_prefix_is_stripped(self) -> None: + service = self._make_service( + proxy_url="http://proxy.example.com", + proxy_prefix="rtc/livekit/foo/", + ) + self.assertEqual(service.proxy_prefix, "rtc/livekit/foo") diff --git a/tests/config/test_experimental.py b/tests/config/test_experimental.py new file mode 100644 index 0000000000..0fede98ee9 --- /dev/null +++ b/tests/config/test_experimental.py @@ -0,0 +1,71 @@ +# +# This file is licensed under the Affero General Public License (AGPL) version 3. +# +# Copyright (C) 2026 Element Creations Ltd. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# See the GNU Affero General Public License for more details: +# . + + +import yaml +from parameterized import parameterized + +from synapse.config._base import RootConfig +from synapse.config.experimental import ExperimentalConfig +from synapse.config.homeserver import HomeServerConfig +from synapse.types import JsonDict + +from tests import unittest + + +class ExperimentalConfigTestCase(unittest.TestCase): + @parameterized.expand( + [ + [ + "single", + { + "experimental_features": { + "msc3575_enabled": True, + } + }, + ], + [ + "multi", + { + "experimental_features": { + "msc3575_enabled": True, + "msc3030_enabled": True, + } + }, + ], + # This has historically worked and this is being added as a regression test + ["none", {"experimental_features": None}], + ] + ) + def test_experimental_features_parsing( + self, test_description: str, config_values: JsonDict + ) -> None: + """ + Test the that `experimental_features` parses with these values + """ + + _read_config(config_values) + + +def _read_config(config_values: JsonDict) -> None: + ExperimentalConfig(RootConfig()).read_config( + yaml.safe_load( + HomeServerConfig().generate_config( + config_dir_path="CONFDIR", + data_dir_path="/data_dir_path", + server_name="che.org", + ) + ) + | config_values, + allow_secrets_in_config=False, + ) diff --git a/tests/config/test_server.py b/tests/config/test_server.py index d3c59ae14c..f718e20a2a 100644 --- a/tests/config/test_server.py +++ b/tests/config/test_server.py @@ -18,11 +18,16 @@ # # + +from typing import Any + import yaml +from parameterized import parameterized from synapse.config._base import ConfigError, RootConfig from synapse.config.homeserver import HomeServerConfig from synapse.config.server import ServerConfig, generate_ip_set, is_threepid_reserved +from synapse.types import JsonDict from tests import unittest @@ -189,6 +194,77 @@ class ServerConfigTestCase(unittest.TestCase): self.assertEqual(conf["listeners"], expected_listeners) + def test_max_delayed_events_enforces_positive(self) -> None: + """ + Test that the configured maximum allowed delay must be a positive value if set, + as per documentation + """ + + def generate_config(value: int) -> JsonDict: + return {"max_event_delay_duration": value} + + _read_config(generate_config(1)) + + with self.assertRaises(ConfigError): + _read_config(generate_config(0)) + + with self.assertRaises(ConfigError): + _read_config(generate_config(-1)) + + def test_max_delayed_events_per_user_enforces_non_negative_int(self) -> None: + """ + Test that the configured maximum number of delayed events must be a non-negative value if set, + as a negative limit can never be satisfied + """ + + def generate_config(value: Any) -> JsonDict: + return { + "experimental_features": {"msc4140_max_delayed_events_per_user": value} + } + + for allowed_value in (0, 1): + _read_config(generate_config(allowed_value)) + + for disallowed_value in (-1, 0.5): + with self.assertRaises(ConfigError): + _read_config(generate_config(disallowed_value)) + + @parameterized.expand( + [ + [ + "single", + { + "experimental_features": { + "msc4140_max_delayed_events_per_user": 3, + } + }, + ], + # This has historically worked and this is being added as a regression test + ["none", {"experimental_features": None}], + ] + ) + def test_experimental_features_parsing( + self, test_description: str, config_values: JsonDict + ) -> None: + """ + Test the that `experimental_features` parses with these values + """ + + _read_config(config_values) + + +def _read_config(config_values: JsonDict) -> None: + ServerConfig(RootConfig()).read_config( + yaml.safe_load( + HomeServerConfig().generate_config( + config_dir_path="CONFDIR", + data_dir_path="/data_dir_path", + server_name="che.org", + ) + ) + | config_values + ) + class GenerateIpSetTestCase(unittest.TestCase): def test_empty(self) -> None: diff --git a/tests/events/test_utils.py b/tests/events/test_utils.py index 8435d6308a..9fd85fc542 100644 --- a/tests/events/test_utils.py +++ b/tests/events/test_utils.py @@ -24,7 +24,7 @@ from typing import TYPE_CHECKING, Any, Mapping from synapse.api.constants import EventContentFields from synapse.api.room_versions import RoomVersions -from synapse.events import EventBase +from synapse.events import EventBase, StrippedStateEvent from synapse.events.utils import ( FilteredEvent, PowerLevelsContent, @@ -35,6 +35,7 @@ from synapse.events.utils import ( format_event_for_client_v2_without_room_id, format_event_raw, maybe_upsert_event_field, + parse_stripped_state_event, prune_event, ) from synapse.types import JsonDict @@ -1003,6 +1004,120 @@ class CopyPowerLevelsContentTestCase(stdlib_unittest.TestCase): copy_and_fixup_power_levels_contents({"a": {"b": {"c": 1}}}) # type: ignore[dict-item] +class TestParseStrippedStateEvent(stdlib_unittest.TestCase): + def test_valid_dict(self) -> None: + """A valid dict should be parsed into a StrippedStateEvent.""" + raw = { + "type": "m.room.member", + "state_key": "@alice:example.com", + "sender": "@alice:example.com", + "content": {"membership": "join"}, + } + result = parse_stripped_state_event(raw) + self.assertEqual( + result, + StrippedStateEvent( + type="m.room.member", + state_key="@alice:example.com", + sender="@alice:example.com", + content={"membership": "join"}, + ), + ) + + def test_invalid_type(self) -> None: + """Non-dict inputs should return None.""" + self.assertIsNone(parse_stripped_state_event("string")) + self.assertIsNone(parse_stripped_state_event(123)) + self.assertIsNone(parse_stripped_state_event([])) + self.assertIsNone(parse_stripped_state_event(None)) + + def test_missing_fields(self) -> None: + """Dicts with missing required fields should return None.""" + self.assertIsNone( + parse_stripped_state_event( + { + "state_key": "@alice:example.com", + "sender": "@alice:example.com", + "content": {"membership": "join"}, + } + ) + ) + self.assertIsNone( + parse_stripped_state_event( + { + "type": "m.room.member", + "sender": "@alice:example.com", + "content": {"membership": "join"}, + } + ) + ) + self.assertIsNone( + parse_stripped_state_event( + { + "type": "m.room.member", + "state_key": "@alice:example.com", + "content": {"membership": "join"}, + } + ) + ) + self.assertIsNone( + parse_stripped_state_event( + { + "type": "m.room.member", + "state_key": "@alice:example.com", + "sender": "@alice:example.com", + } + ) + ) + + def test_invalid_field_types(self) -> None: + """Dicts with invalid field types should return None.""" + # Type must be string + self.assertIsNone( + parse_stripped_state_event( + { + "type": 123, + "state_key": "@alice:example.com", + "sender": "@alice:example.com", + "content": {"membership": "join"}, + } + ) + ) + # State key must be string + self.assertIsNone( + parse_stripped_state_event( + { + "type": "m.room.member", + "state_key": 123, + "sender": "@alice:example.com", + "content": {"membership": "join"}, + } + ) + ) + # Sender must be string + self.assertIsNone( + parse_stripped_state_event( + { + "type": "m.room.member", + "state_key": "@alice:example.com", + "sender": 123, + "content": {"membership": "join"}, + } + ) + ) + # Content must be dict + self.assertIsNone( + parse_stripped_state_event( + { + "type": "m.room.member", + "state_key": "@alice:example.com", + "sender": "@alice:example.com", + "content": "membership_join", + } + ) + ) + + class FormatEventForClientTestCase(stdlib_unittest.TestCase): """Tests for the standalone `format_event_*` transforms. diff --git a/tests/federation/_remote_join.py b/tests/federation/_remote_join.py new file mode 100644 index 0000000000..8a4e438466 --- /dev/null +++ b/tests/federation/_remote_join.py @@ -0,0 +1,369 @@ +# +# This file is licensed under the Affero General Public License (AGPL) version 3. +# +# Copyright (C) 2026 Element Creations Ltd +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# See the GNU Affero General Public License for more details: +# . +# + +import logging +import urllib.parse +from http import HTTPStatus +from typing import Callable, TypeVar +from unittest.mock import Mock + +import attr + +from synapse.api.constants import EventContentFields, EventTypes, Membership +from synapse.api.room_versions import KNOWN_ROOM_VERSIONS, RoomVersion +from synapse.config.server import DEFAULT_ROOM_VERSION +from synapse.events import EventBase +from synapse.events.utils import strip_event +from synapse.federation.transport.client import SendJoinResponse +from synapse.http.matrixfederationclient import ByteParser +from synapse.http.types import QueryParams +from synapse.types import JsonDict + +from tests.test_utils.event_builders import make_test_event, make_test_pdu_event +from tests.unittest import FederatingHomeserverTestCase + +logger = logging.getLogger(__name__) + + +@attr.s(slots=True, auto_attribs=True) +class RemoteStateEvent: + """ + A state event to insert into the remote room `/send_join` response. + """ + + type: str + state_key: str + content: JsonDict + # Defaults to the remote room creator. + sender: str | None = None + + +class RemoteJoinHelper: + """ + Helps a `FederatingHomeserverTestCase` join a remote (non-resident) room + over federation. + + Spiritually inspired by `test_federation_out_of_band_membership.py` + but in a more reusable form. + + Constructor Args: + test_case: the current `FederatingHomeserverTestCase` + federation_http_client: Mocked form of the main test homeserver's federation HTTP client; + should have method slots for `get_json` and `put_json`. + remote_creator_user_id: User ID of the remote user creating the room + room_version: Desired room version + create_content: Desired `content` of the `m.room.create` event + state_events: Extra state events to create in the mock room + (They will be made available in the `/send_join` response) + + Usage: + + helper = RemoteJoinHelper( + self, + create_content={"predecessor": {"room_id": old_room_id, ...}}, + ) + # helper.room_id is available now; you can e.g. point a tombstone at it. + helper.join(local_user_id, local_user_tok) + """ + + room_id: str + """ + The room ID of the mock remote room that will be joined. + """ + + def __init__( + self, + test_case: FederatingHomeserverTestCase, + federation_http_client: Mock, + *, + remote_creator_user_id: str | None = None, + room_version: RoomVersion = KNOWN_ROOM_VERSIONS[DEFAULT_ROOM_VERSION], + create_content: JsonDict | None = None, + state_events: list[RemoteStateEvent] | None = None, + ) -> None: + if remote_creator_user_id is None: + remote_creator_user_id = f"@remote-user:{test_case.OTHER_SERVER_NAME}" + + self._test_case = test_case + self._federation_http_client = federation_http_client + self._remote_creator_user_id = remote_creator_user_id + self._room_version = room_version + + # 1. Create the room creation event + create_content_full: JsonDict = { + EventContentFields.ROOM_CREATOR: remote_creator_user_id, + EventContentFields.ROOM_VERSION: room_version.identifier, + } + if create_content is not None: + create_content_full.update(create_content) + + create_event_dict: JsonDict = { + "sender": remote_creator_user_id, + "depth": 1, + "origin_server_ts": 1, + "type": EventTypes.Create, + "state_key": "", + "content": create_content_full, + "auth_events": [], + "prev_events": [], + } + if not room_version.msc4291_room_ids_as_hashes: + # For room versions that _don't_ derive the room ID from the content, + # we need to set our own. + # We could consider exposing a parameter to allow varying the localpart + # (and perturb the create event for hashes-as-room-ID rooms) + create_event_dict["room_id"] = f"!remote-room:{test_case.OTHER_SERVER_NAME}" + + room_create_event = make_test_event( + test_case.add_hashes_and_signatures_from_other_server( + create_event_dict, + room_version=room_version, + ), + room_version=room_version, + ) + + self.room_id = room_create_event.room_id + + # 2. Create the room creator's membership event + creator_membership_event = make_test_event( + test_case.add_hashes_and_signatures_from_other_server( + { + "room_id": self.room_id, + "sender": remote_creator_user_id, + "depth": 2, + "origin_server_ts": 2, + "type": EventTypes.Member, + "state_key": remote_creator_user_id, + "content": {"membership": Membership.JOIN}, + "auth_events": [room_create_event.event_id] + if not room_version.msc4291_room_ids_as_hashes + else [], + "prev_events": [room_create_event.event_id], + }, + room_version=room_version, + ), + room_version=room_version, + ) + + # 3. Create requested extra state events (in a linear chain from the membership) + extra_state_events: list[EventBase] = [] + prev_event = creator_membership_event + depth = 3 + for spec in state_events or []: + sender = spec.sender or remote_creator_user_id + event = make_test_event( + test_case.add_hashes_and_signatures_from_other_server( + { + "room_id": self.room_id, + "sender": sender, + "depth": depth, + "origin_server_ts": depth, + "type": spec.type, + "state_key": spec.state_key, + "content": spec.content, + "auth_events": [ + room_create_event.event_id, + creator_membership_event.event_id, + ] + if not room_version.msc4291_room_ids_as_hashes + else [creator_membership_event.event_id], + "prev_events": [prev_event.event_id], + }, + room_version=room_version, + ), + room_version=room_version, + ) + extra_state_events.append(event) + prev_event = event + depth += 1 + + self._room_create_event = room_create_event + self._creator_membership_event = creator_membership_event + self._extra_state_events = extra_state_events + + def join(self, local_user_id: str, local_user_tok: str) -> None: + """ + Invite `local_user_id` and perform the federation join dance. + """ + remote_room_id = self.room_id + room_version = self._room_version + + room_create_event = self._room_create_event + creator_membership_event = self._creator_membership_event + extra_events = self._extra_state_events + + # 1. Create an invite event and make it appear on the 'real' homeserver + depth = 3 + len(extra_events) + + invite_membership_event = make_test_event( + self._test_case.add_hashes_and_signatures_from_other_server( + { + "room_id": remote_room_id, + "sender": self._remote_creator_user_id, + "depth": depth, + "origin_server_ts": depth, + "type": EventTypes.Member, + "state_key": local_user_id, + "content": {"membership": Membership.INVITE}, + "auth_events": [ + room_create_event.event_id, + creator_membership_event.event_id, + ] + if not room_version.msc4291_room_ids_as_hashes + else [creator_membership_event.event_id], + "prev_events": [ + extra_events[-1].event_id + if extra_events + else creator_membership_event.event_id + ], + }, + room_version=room_version, + ), + room_version=room_version, + ) + + channel = self._test_case.make_signed_federation_request( + "PUT", + f"/_matrix/federation/v2/invite/{remote_room_id}/{invite_membership_event.event_id}", + content={ + "event": invite_membership_event.get_dict(), + "invite_room_state": [ + strip_event(room_create_event), + ], + "room_version": room_version.identifier, + }, + ) + assert channel.code == HTTPStatus.OK, channel.json_body + + # 2. Mock `/make_join` and `/send_join`. + # Start by creating a join membership event. + join_membership_event_template = make_test_event( + { + "room_id": remote_room_id, + "sender": local_user_id, + "depth": depth + 1, + "origin_server_ts": depth + 1, + "type": EventTypes.Member, + "state_key": local_user_id, + "content": {"membership": Membership.JOIN}, + "auth_events": [ + room_create_event.event_id, + invite_membership_event.event_id, + ] + if not room_version.msc4291_room_ids_as_hashes + else [invite_membership_event.event_id], + "prev_events": [invite_membership_event.event_id], + }, + room_version=room_version, + ) + + T = TypeVar("T") + + async def _get_json( + destination: str, + path: str, + args: QueryParams | None = None, + retry_on_dns_fail: bool = True, + timeout: int | None = None, + ignore_backoff: bool = False, + try_trailing_slash_on_400: bool = False, + parser: ByteParser[T] | None = None, + ) -> JsonDict | T: + make_join_path = ( + f"/_matrix/federation/v1/make_join/" + f"{urllib.parse.quote_plus(remote_room_id)}/{urllib.parse.quote_plus(local_user_id)}" + ) + if path == make_join_path: + return { + "event": join_membership_event_template.get_pdu_json(), + "room_version": room_version.identifier, + } + raise NotImplementedError( + "We have not mocked a response for `get_json(...)` for the following endpoint yet: " + + f"{destination}{path}" + ) + + self._federation_http_client.get_json.side_effect = _get_json + + send_join_state = [ + room_create_event, + creator_membership_event, + *extra_events, + invite_membership_event, + ] + + async def _put_json( + destination: str, + path: str, + args: QueryParams | None = None, + data: JsonDict | None = None, + json_data_callback: Callable[[], JsonDict] | None = None, + long_retries: bool = False, + timeout: int | None = None, + ignore_backoff: bool = False, + backoff_on_404: bool = False, + try_trailing_slash_on_400: bool = False, + parser: ByteParser[T] | None = None, + backoff_on_all_error_codes: bool = False, + ) -> JsonDict | T | SendJoinResponse: + if ( + path.startswith( + f"/_matrix/federation/v2/send_join/{urllib.parse.quote_plus(remote_room_id)}/" + ) + and data is not None + and data.get("type") == EventTypes.Member + and data.get("state_key") == local_user_id + and parser is not None + ): + # As the remote server, sign the join event before returning it. + join_membership_event_signed = make_test_event( + self._test_case.add_hashes_and_signatures_from_other_server( + data, room_version=room_version + ), + room_version=room_version, + ) + return SendJoinResponse( + auth_events=[ + room_create_event, + invite_membership_event, + ], + state=send_join_state, + event_dict=join_membership_event_signed.get_pdu_json(), + event=join_membership_event_signed, + members_omitted=False, + servers_in_room=[ + self._test_case.OTHER_SERVER_NAME, + ], + ) + + if path.startswith("/_matrix/federation/v1/send/") and data is not None: + # Just acknowledge everything. + return { + make_test_pdu_event(pdu, room_version).event_id: {} + for pdu in data.get("pdus", []) + } + + raise NotImplementedError( + "We have not mocked a response for `put_json(...)` for the following endpoint yet: " + + f"{destination}{path} with the following body data: {data}" + ) + + self._federation_http_client.put_json.side_effect = _put_json + + # 3. Issue the client-server API request to join the room + self._test_case.helper.join(remote_room_id, local_user_id, tok=local_user_tok) + + # 4. Reset mocks + self._federation_http_client.get_json.side_effect = None + self._federation_http_client.put_json.side_effect = None diff --git a/tests/federation/test_federation_join_upgraded_room.py b/tests/federation/test_federation_join_upgraded_room.py new file mode 100644 index 0000000000..4557cf1b55 --- /dev/null +++ b/tests/federation/test_federation_join_upgraded_room.py @@ -0,0 +1,308 @@ +# +# This file is licensed under the Affero General Public License (AGPL) version 3. +# +# Copyright (C) 2026 Element Creations Ltd +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# See the GNU Affero General Public License for more details: +# . +# + +import logging +from unittest.mock import Mock + +from twisted.internet.testing import MemoryReactor + +from synapse.api.constants import EventTypes +from synapse.rest import admin +from synapse.rest.client import login, room +from synapse.server import HomeServer +from synapse.types import JsonDict, RoomAlias +from synapse.util.clock import Clock + +from tests import unittest +from tests.federation._remote_join import RemoteJoinHelper + +logger = logging.getLogger(__name__) + + +def _predecessor(room_id: str) -> JsonDict: + """`create_content` for a remote room that claims `room_id` as its predecessor.""" + return { + "predecessor": { + "room_id": room_id, + # inert dummy + "event_id": "$some_tombstone_event:test", + } + } + + +class FederationJoinUpgradedRoomTestCase(unittest.FederatingHomeserverTestCase): + servlets = [ + admin.register_servlets, + login.register_servlets, + room.register_servlets, + ] + + def make_homeserver(self, reactor: MemoryReactor, clock: Clock) -> HomeServer: + self._federation_http_client = Mock( + # The problem with using `spec=MatrixFederationHttpClient` here is that it + # requires everything to be mocked which is a lot of work that I don't want + # to do when the code only uses a few methods (`get_json` and `put_json`). + ) + return self.setup_test_homeserver( + federation_http_client=self._federation_http_client + ) + + def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None: + super().prepare(reactor, clock, hs) + + self.store = self.hs.get_datastores().main + self.storage_controllers = hs.get_storage_controllers() + + def _room_is_public(self, room_id: str) -> bool: + """`is_public` flag from the `rooms` table (asserts the row exists).""" + room = self.get_success(self.store.get_room(room_id)) + assert room is not None, f"no rooms row for {room_id}" + is_public, _ = room + return is_public + + def test_alias_transferred_on_federation_join_to_upgraded_room(self) -> None: + """ + Tests that joining an upgraded room over federation, + where the predecessor has a valid corresponding tombstone, + should transfer all room aliases from the old room to the new room. + + The old room is set up with a couple of local aliases and a tombstone + event pointing at the new (remote) room. + After the federation join, all room aliases should have been + transferred from the old room to the new room. + """ + local_user_id = self.register_user("user1", "pass") + local_user_tok = self.login(local_user_id, "pass") + + # Set up an old room and point 2 room aliases at it + old_room_id = self.helper.create_room_as( + room_creator=local_user_id, + tok=local_user_tok, + ) + + alias1 = RoomAlias.from_string("#old_room:test") + alias2 = RoomAlias.from_string("#old_room_alt:test") + for alias in (alias1, alias2): + self.get_success( + self.store.create_room_alias_association( + alias, old_room_id, [self.hs.hostname] + ) + ) + + # Now set up a replacement room, which we will remote-room-join in a moment + join_helper = RemoteJoinHelper( + self, + self._federation_http_client, + create_content=_predecessor(old_room_id), + ) + + # Place a tombstone in the old room, essentially authorising + # the replacement room to replace it + self.helper.send_state( + old_room_id, + EventTypes.Tombstone, + {"replacement_room": join_helper.room_id}, + tok=local_user_tok, + ) + + # Trigger the remote room join + join_helper.join(local_user_id, local_user_tok) + + # The new room should have acquired the aliases... + new_aliases = self.get_success( + self.store.get_aliases_for_room(join_helper.room_id) + ) + self.assertCountEqual(new_aliases, [alias1.to_string(), alias2.to_string()]) + + # ...and the old room therefore must have given them up + old_aliases = self.get_success(self.store.get_aliases_for_room(old_room_id)) + self.assertEqual(old_aliases, []) + + def test_room_directory_visibility_transferred(self) -> None: + """ + On a valid federation join of an upgraded room, the room directory public + flag should move from the old room to the new room. + + A public old room becomes private (so people don't accidentally join it) + and the newly-joined room is marked public. + """ + local_user_id = self.register_user("user1", "pass") + local_user_tok = self.login(local_user_id, "pass") + + # Set up an old room and mark it as public (for room directory purposes) + old_room_id = self.helper.create_room_as( + room_creator=local_user_id, + tok=local_user_tok, + ) + self.get_success(self.store.set_room_is_public(old_room_id, True)) + self.assertTrue(self._room_is_public(old_room_id)) + + # Now set up a replacement room, which we will remote-room-join in a moment + join_helper = RemoteJoinHelper( + self, + self._federation_http_client, + create_content=_predecessor(old_room_id), + ) + + # Place a tombstone in the old room, essentially authorising + # the replacement room to replace it + self.helper.send_state( + old_room_id, + EventTypes.Tombstone, + {"replacement_room": join_helper.room_id}, + tok=local_user_tok, + ) + + # Trigger the remote room join + join_helper.join(local_user_id, local_user_tok) + + # The room directory publicity should have shifted to the new room + # (So users don't accidentally join the old room from the directory) + self.assertFalse(self._room_is_public(old_room_id)) + self.assertTrue(self._room_is_public(join_helper.room_id)) + + def test_room_directory_visibility_not_transferred_for_private_room(self) -> None: + """ + On a valid federation join of an upgraded room, a private old room should + leave the room directory visibility untouched for both the old and new + rooms (both remain private). + """ + local_user_id = self.register_user("user1", "pass") + local_user_tok = self.login(local_user_id, "pass") + + # Set up an old room and mark it as public (for room directory purposes) + old_room_id = self.helper.create_room_as( + room_creator=local_user_id, + tok=local_user_tok, + ) + self.assertFalse(self._room_is_public(old_room_id)) + + # Now set up a replacement room, which we will remote-room-join in a moment + join_helper = RemoteJoinHelper( + self, + self._federation_http_client, + create_content=_predecessor(old_room_id), + ) + + # Place a tombstone in the old room, essentially authorising + # the replacement room to replace it + self.helper.send_state( + old_room_id, + EventTypes.Tombstone, + {"replacement_room": join_helper.room_id}, + tok=local_user_tok, + ) + + # Trigger the remote room join + join_helper.join(local_user_id, local_user_tok) + + self.assertFalse(self._room_is_public(old_room_id)) + self.assertFalse(self._room_is_public(join_helper.room_id)) + + def test_no_transfer_when_predecessor_room_has_no_tombstone(self) -> None: + """ + Tests that when joining a remote room over federation, + if the room has an illegitimate predecessor (a predecessor pointing + to a room that does not have a corresponding tombstone to vouch for it + as the successor), room aliases are not transferred. + """ + local_user_id = self.register_user("user1", "pass") + local_user_tok = self.login(local_user_id, "pass") + + # Set up a room with an alias + old_room_id = self.helper.create_room_as( + room_creator=local_user_id, + tok=local_user_tok, + ) + + alias = RoomAlias.from_string("#old_room:test") + self.get_success( + self.store.create_room_alias_association( + alias, old_room_id, [self.hs.hostname] + ) + ) + + join_helper = RemoteJoinHelper( + self, + self._federation_http_client, + # The new room (illegitimately) claims to be the successor + # of the old room. + create_content=_predecessor(old_room_id), + ) + + # Notably, we do NOT set up a tombstone in the 'old' room. + + # Do the remote room join dance + join_helper.join(local_user_id, local_user_tok) + + # Check that the room alias did _not_ get transferred... + new_aliases = self.get_success( + self.store.get_aliases_for_room(join_helper.room_id) + ) + self.assertCountEqual(new_aliases, []) + + # ...and that the old room still has it + old_aliases = self.get_success(self.store.get_aliases_for_room(old_room_id)) + self.assertCountEqual(old_aliases, [alias.to_string()]) + + def test_no_transfer_when_tombstone_does_not_match(self) -> None: + """ + A predecessor room whose tombstone points to a different room than the + one being joined must not trigger an alias transfer. + + The tombstone's `replacement_room` must match the joined room for the + upgrade link to be considered valid. + """ + local_user_id = self.register_user("user1", "pass") + local_user_tok = self.login(local_user_id, "pass") + + # Set up a room with an alias + old_room_id = self.helper.create_room_as( + room_creator=local_user_id, + tok=local_user_tok, + ) + + alias = RoomAlias.from_string("#old_room:test") + self.get_success( + self.store.create_room_alias_association( + alias, old_room_id, [self.hs.hostname] + ) + ) + + # Tombstone points at a _different_, room. + self.helper.send_state( + old_room_id, + EventTypes.Tombstone, + {"replacement_room": "!the_real_replacement_room:example.com"}, + tok=local_user_tok, + ) + + join_helper = RemoteJoinHelper( + self, + self._federation_http_client, + # The new room (illegitimately) claims to be the successor + # of the old room. + create_content=_predecessor(old_room_id), + ) + join_helper.join(local_user_id, local_user_tok) + + # Check that the room alias did _not_ get transferred... + new_aliases = self.get_success( + self.store.get_aliases_for_room(join_helper.room_id) + ) + self.assertCountEqual(new_aliases, []) + + # ...and that the old room still has it + old_aliases = self.get_success(self.store.get_aliases_for_room(old_room_id)) + self.assertCountEqual(old_aliases, [alias.to_string()]) diff --git a/tests/federation/test_federation_out_of_band_membership.py b/tests/federation/test_federation_out_of_band_membership.py index 1707081863..85e221a64a 100644 --- a/tests/federation/test_federation_out_of_band_membership.py +++ b/tests/federation/test_federation_out_of_band_membership.py @@ -189,7 +189,7 @@ class OutOfBandMembershipTests(unittest.FederatingHomeserverTestCase): # Create a remote room room_creator_user_id = f"@remote-user:{self.OTHER_SERVER_NAME}" remote_room_id = f"!remote-room:{self.OTHER_SERVER_NAME}" - room_version = RoomVersions.V10 + room_version = RoomVersions.V11 room_create_event = make_test_event( self.add_hashes_and_signatures_from_other_server( @@ -201,9 +201,6 @@ class OutOfBandMembershipTests(unittest.FederatingHomeserverTestCase): "type": EventTypes.Create, "state_key": "", "content": { - # The `ROOM_CREATOR` field could be removed if we used a room - # version > 10 (in favor of relying on `sender`) - EventContentFields.ROOM_CREATOR: room_creator_user_id, EventContentFields.ROOM_VERSION: room_version.identifier, }, "auth_events": [], diff --git a/tests/federation/test_federation_server.py b/tests/federation/test_federation_server.py index 6e62fdd267..665411f957 100644 --- a/tests/federation/test_federation_server.py +++ b/tests/federation/test_federation_server.py @@ -28,7 +28,7 @@ from parameterized import parameterized from twisted.internet.testing import MemoryReactor from synapse.api.constants import EventTypes, Membership -from synapse.api.errors import FederationError, SynapseError +from synapse.api.errors import Codes, FederationError, SynapseError from synapse.api.room_versions import KNOWN_ROOM_VERSIONS, RoomVersions from synapse.config.server import DEFAULT_ROOM_VERSION from synapse.crypto.event_signing import add_hashes_and_signatures @@ -39,7 +39,7 @@ from synapse.rest import admin from synapse.rest.client import login, room from synapse.server import HomeServer from synapse.storage.controllers.state import server_acl_evaluator_from_event -from synapse.types import JsonDict +from synapse.types import JsonDict, UserID from synapse.util.clock import Clock from tests import unittest @@ -95,6 +95,293 @@ class FederationServerTests(unittest.FederatingHomeserverTestCase): self.assertEqual(500, channel.code, channel.result) +class GetMissingEventsRoomCheckTests(unittest.FederatingHomeserverTestCase): + """ + Regression tests for room confusion in /get_missing_events + https://github.com/element-hq/synapse/security/advisories/GHSA-27p5-4f45-gx76 + """ + + servlets = [ + admin.register_servlets, + login.register_servlets, + room.register_servlets, + ] + + def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None: + super().prepare(reactor, clock, hs) + + # Local user + self.local_user_id = self.register_user("alice", "pass") + self.local_user_token = self.login("alice", "pass") + self.local_user = UserID.from_string(self.local_user_id) + + # Create 2 rooms (one with the remote server, one without). + # - The remote server will be in this room + self.room_allowed = self.helper.create_room_as( + self.local_user_id, tok=self.local_user_token + ) + self.inject_room_member( + self.room_allowed, f"@remote:{self.OTHER_SERVER_NAME}", "join" + ) + # - The remote server will _not_ be in this room + self.room_blocked = self.helper.create_room_as( + self.local_user_id, tok=self.local_user_token + ) + + # Insert a linear chain of events in both rooms + self.room_allowed_event_ids = self.helper.send_messages( + self.room_allowed, num_events=5, tok=self.local_user_token + ) + self.room_blocked_event_ids = self.helper.send_messages( + self.room_blocked, num_events=5, tok=self.local_user_token + ) + + def _extract_returned_event_ids(self, json_body: JsonDict) -> set[str]: + """ + Given the response body of `/get_missing_events`, return the event IDs + of the events that were returned in the response. + This only includes event IDs from `self.room_allowed_event_ids` and + `self.room_blocked_event_ids`; other events are ignored. + + As the federation PDU format doesn't include event IDs + (at least not for every room version), we match on the + `(room_id, content.body, prev_events)` triple against the events + we sent in the setup. + """ + store = self.hs.get_datastores().main + events = self.get_success( + store.get_events_as_list( + list(self.room_allowed_event_ids) + list(self.room_blocked_event_ids) + ) + ) + # (room_id, content.body, prev_events) -> event ID + event_lookup: dict[tuple[str, str, tuple[str, ...]], str] = {} + for event in events: + key = ( + event.room_id, + event.content["body"], + tuple(event.prev_event_ids()), + ) + event_lookup[key] = event.event_id + + returned_event_ids: set[str] = set() + for pdu in json_body["events"]: + key = ( + pdu.get("room_id"), + pdu.get("content", {}).get("body"), + tuple(pdu.get("prev_events", [])), + ) + event_id = event_lookup.get(key) + if event_id is None: + # Not one of the events we created; ignore it. + continue + returned_event_ids.add(event_id) + return returned_event_ids + + def test_get_missing_events_returns_events_from_correct_room(self) -> None: + """ + Tests the happy path when `latest_events` and `earliest_events` + are both in the correct room. + + returned + | + v + e1 <- e2 <- e3 <- e4 <- e5 + ^ ^ + | | + earliest latest + + Not a regression test; I'm just filling a gap in our (in-repo) testing + as far as I can tell. + """ + channel = self.make_signed_federation_request( + "POST", + f"/_matrix/federation/v1/get_missing_events/{self.room_allowed}", + content={ + "earliest_events": [self.room_allowed_event_ids[1]], + "latest_events": [self.room_allowed_event_ids[3]], + "limit": 10, + }, + ) + self.assertEqual(HTTPStatus.OK, channel.code, channel.result) + self.assertEqual( + self._extract_returned_event_ids(channel.json_body), + {self.room_allowed_event_ids[2]}, + ) + + def test_get_missing_events_with_empty_earliest_events(self) -> None: + """ + Tests that `/get_missing_events`, when given no `earliest_events`, + walks back to the start of the room, capped at `limit`. + + (Not a regression test; documents pre-existing behaviour) + """ + channel = self.make_signed_federation_request( + "POST", + f"/_matrix/federation/v1/get_missing_events/{self.room_allowed}", + content={ + "earliest_events": [], + "latest_events": [self.room_allowed_event_ids[-1]], + "limit": 10, + }, + ) + self.assertEqual(HTTPStatus.OK, channel.code, channel.result) + self.assertEqual( + self._extract_returned_event_ids(channel.json_body), + set(self.room_allowed_event_ids[:-1]), + ) + + def test_get_missing_events_with_unknown_earliest_event(self) -> None: + """ + Tests that `/get_missing_events` ignores unknown event IDs given in + `earliest_events`. + + This makes sense as the `earliest_events` are intuitively + 'events to stop at' when walking backwards. + Since we don't know about those events, we don't use them as stopping conditions. + (In other words, this falls back to the same behaviour as + `test_get_missing_events_with_empty_earliest_events`.) + + (Not a regression test; documents pre-existing behaviour) + """ + channel = self.make_signed_federation_request( + "POST", + f"/_matrix/federation/v1/get_missing_events/{self.room_allowed}", + content={ + "earliest_events": ["$someUnknownEventId"], + "latest_events": [self.room_allowed_event_ids[-1]], + "limit": 10, + }, + ) + self.assertEqual(HTTPStatus.OK, channel.code, channel.result) + self.assertEqual( + self._extract_returned_event_ids(channel.json_body), + set(self.room_allowed_event_ids[:-1]), + ) + + def test_get_missing_events_with_no_latest_event(self) -> None: + """ + Tests that when the `/get_missing_events` request references + no events in `latest_events`, the response is 200 OK + with an empty `events` list. + + (Not a regression test; documents pre-existing behaviour) + """ + channel = self.make_signed_federation_request( + "POST", + f"/_matrix/federation/v1/get_missing_events/{self.room_allowed}", + content={ + "earliest_events": ["$someOtherUnknownEventId"], + "latest_events": [], + "limit": 10, + }, + ) + self.assertEqual(channel.code, HTTPStatus.OK, channel.result) + self.assertEqual(channel.json_body, {"events": []}) + + def test_get_missing_events_with_unknown_latest_event(self) -> None: + """ + Tests that when the `/get_missing_events` request references + unknown events in `latest_events`, the response is 200 OK + with an empty `events` list. + + I imagine this makes sense as you might request several events + in `latest_events` to start walking back from and we need to be + tolerant of the fact that servers don't always know about every event. + + (Not a regression test; documents pre-existing behaviour) + """ + channel = self.make_signed_federation_request( + "POST", + f"/_matrix/federation/v1/get_missing_events/{self.room_allowed}", + content={ + "earliest_events": ["$someOtherUnknownEventId"], + "latest_events": ["$someUnknownEventId"], + "limit": 10, + }, + ) + self.assertEqual(channel.code, HTTPStatus.OK, channel.result) + self.assertEqual(channel.json_body, {"events": []}) + + def test_get_missing_events_ignores_events_from_other_room(self) -> None: + """ + Tests that providing `earliest_events` and `latest_events` from the wrong room + treats them the same as being unknown. + + From `test_get_missing_events_with_unknown_latest_event` we established that + unknown events in `latest_events` get skipped (to the point of returning an empty + `events: []` response) + + From `test_get_missing_events_with_unknown_earliest_event` we established that + unknown events in `earliest_events` get ignored as stopping conditions. + + This regression test previously failed. + """ + channel = self.make_signed_federation_request( + "POST", + f"/_matrix/federation/v1/get_missing_events/{self.room_allowed}", + content={ + "earliest_events": [self.room_blocked_event_ids[0]], + "latest_events": [self.room_blocked_event_ids[-1]], + "limit": 10, + }, + ) + self.assertEqual(channel.code, HTTPStatus.OK, channel.result) + self.assertEqual(channel.json_body, {"events": []}) + + def test_get_missing_events_skips_latest_events_from_other_room(self) -> None: + """ + Tests that providing `latest_events` from the wrong room + treats it as being unknown, even if `earliest_events` are from the correct + room. + + From `test_get_missing_events_with_unknown_latest_event` we established that + unknown events in `latest_events` get skipped (to the point of returning an empty + `events: []` response) + + This regression test previously failed. + """ + channel = self.make_signed_federation_request( + "POST", + f"/_matrix/federation/v1/get_missing_events/{self.room_allowed}", + content={ + "earliest_events": [self.room_allowed_event_ids[0]], + "latest_events": [self.room_blocked_event_ids[-1]], + "limit": 10, + }, + ) + self.assertEqual(channel.code, HTTPStatus.OK, channel.result) + self.assertEqual(channel.json_body, {"events": []}) + + def test_get_missing_events_ignores_earliest_events_from_other_room(self) -> None: + """ + Tests that providing `earliest_events` from the wrong room causes those + events to be ignored as stopping conditions, + even though `latest_events` are from the correct room. + + From `test_get_missing_events_with_unknown_earliest_event` we established that + unknown events in `earliest_events` get ignored as stopping conditions. + + This test was previously fine, but is an obvious extra case. + """ + channel = self.make_signed_federation_request( + "POST", + f"/_matrix/federation/v1/get_missing_events/{self.room_allowed}", + content={ + # Use [-3] here as we want to see if the walk-back algorithm + # confuses depth (topological ordering) across the two rooms. + "earliest_events": [self.room_blocked_event_ids[-3]], + "latest_events": [self.room_allowed_event_ids[-1]], + "limit": 10, + }, + ) + self.assertEqual(HTTPStatus.OK, channel.code, channel.result) + self.assertEqual( + self._extract_returned_event_ids(channel.json_body), + set(self.room_allowed_event_ids[:-1]), + ) + + def _create_acl_event(content: JsonDict) -> EventBase: return make_test_event( { @@ -325,6 +612,95 @@ class StateQueryTests(unittest.FederatingHomeserverTestCase): self.assertEqual(channel.json_body["errcode"], "M_FORBIDDEN") +class TimestampToEventTests(unittest.FederatingHomeserverTestCase): + """Tests for `GET /_matrix/federation/v1/timestamp_to_event/`.""" + + servlets = [ + admin.register_servlets, + room.register_servlets, + login.register_servlets, + ] + + def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None: + # Create a room and join the remote server so it's allowed to query + user = self.register_user("u1", "pass") + tok = self.login("u1", "pass") + self.room_id = self.helper.create_room_as(user, tok=tok) + # Send one event at time = 1000s + self.reactor.advance(1000) + self.event_at_1000 = self.helper.send_messages(self.room_id, 1, tok=tok)[0] + + # Send another event at time = 4000s + self.reactor.advance(3000) + self.event_at_4000 = self.helper.send_messages(self.room_id, 1, tok=tok)[0] + + # Send another event at time = 8000s + self.reactor.advance(4000) + self.event_at_8000 = self.helper.send_messages(self.room_id, 1, tok=tok)[0] + + super().prepare(reactor, clock, hs) + + @parameterized.expand( + [ + # Query backwards from 5000s, should find the event at 4000s + (5000000, "b"), + # Query forwards from 1100s, should find the event at 4000s + (1100000, "f"), + ] + ) + def test_happy_path(self, ts: int, dir: str) -> None: + """ + Tests that a server in the room gets 200 OK + with the closest event IDs as requested for a given timestamp, + in both forward and backward directions. + """ + # Join the remote server to the room + self.inject_room_member(self.room_id, "@user:" + self.OTHER_SERVER_NAME, "join") + + channel = self.make_signed_federation_request( + "GET", + f"/_matrix/federation/v1/timestamp_to_event/{self.room_id}?ts={ts}&dir={dir}", + ) + self.assertEqual(channel.code, HTTPStatus.OK, channel.json_body) + self.assertEqual(channel.json_body["event_id"], self.event_at_4000) + + @parameterized.expand( + [ + # Query backwards at 0s, no events to be found. + (0, "b"), + # Query forwards from 8100s, no events to be found. + (8100000, "f"), + ] + ) + def test_no_matching_event(self, ts: int, dir: str) -> None: + """ + Tests that a 404 / M_NOT_FOUND is returned when no event occurs + in the requested direction of a timestamp. + """ + # Join the remote server to the room + self.inject_room_member(self.room_id, "@user:" + self.OTHER_SERVER_NAME, "join") + + channel = self.make_signed_federation_request( + "GET", + f"/_matrix/federation/v1/timestamp_to_event/{self.room_id}?ts={ts}&dir={dir}", + ) + self.assertEqual(channel.code, HTTPStatus.NOT_FOUND, channel.json_body) + self.assertEqual(channel.json_body["errcode"], "M_NOT_FOUND") + + def test_requires_server_in_room(self) -> None: + """ + Tests that a server not in the room is rejected with 403 / M_FORBIDDEN. + """ + # Notably: _don't_ join the remote server to the room + + channel = self.make_signed_federation_request( + "GET", + f"/_matrix/federation/v1/timestamp_to_event/{self.room_id}?ts=2000000&dir=b", + ) + self.assertEqual(channel.code, HTTPStatus.FORBIDDEN, channel.json_body) + self.assertEqual(channel.json_body["errcode"], "M_FORBIDDEN") + + class UnstableGetExtremitiesTests(unittest.FederatingHomeserverTestCase): servlets = [ admin.register_servlets, @@ -474,6 +850,84 @@ class UnstableGetExtremitiesTests(unittest.FederatingHomeserverTestCase): self.assertEqual(channel.json_body["errcode"], "M_UNRECOGNIZED") +class EventAuthFederationTests(unittest.FederatingHomeserverTestCase): + servlets = [ + admin.register_servlets, + room.register_servlets, + login.register_servlets, + ] + + def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None: + # Create a local user + self.user_id = self.register_user("alice", "password") + self.user_tok = self.login("alice", "password") + + # Set up a room and join the remote server to it + self.room_id = self.helper.create_room_as( + self.user_id, + is_public=True, + room_version=RoomVersions.V10.identifier, + tok=self.user_tok, + ) + self.inject_room_member( + self.room_id, f"@remote:{self.OTHER_SERVER_NAME}", Membership.JOIN + ) + + # Create a known event whose auth chain we can request back. + self.event_id = self.helper.send_messages( + self.room_id, num_events=1, tok=self.user_tok + )[0] + + return super().prepare(reactor, clock, hs) + + def test_event_auth_unknown_event_returns_404(self) -> None: + """ + Tests that requesting the auth chain of an unknown event + returns 404 / M_NOT_FOUND. + """ + + # Request an event that doesn't exist in self.room_id. + channel = self.make_signed_federation_request( + "GET", + f"/_matrix/federation/v1/event_auth/{self.room_id}/$unknownevent", + ) + self.assertEqual(channel.code, HTTPStatus.NOT_FOUND, channel.result) + self.assertEqual( + channel.json_body["errcode"], Codes.NOT_FOUND, channel.json_body + ) + + def test_event_auth_wrong_room_returns_404(self) -> None: + """ + Tests that a request whose `room_id` is wrong for the event + acts the same as though it were an unknown event. + + Regression test for https://github.com/element-hq/synapse/security/advisories/GHSA-qcjr-46gf-7f4r + """ + + # Create a second room with its own event. + other_room_id = self.helper.create_room_as( + self.user_id, + is_public=True, + room_version=RoomVersions.V10.identifier, + tok=self.user_tok, + ) + other_room_event_id = self.helper.send_messages( + other_room_id, num_events=1, tok=self.user_tok + )[0] + + # Request the chain of other_room_id's event, but pretend it's part of the room + # we are in. + channel = self.make_signed_federation_request( + "GET", + f"/_matrix/federation/v1/event_auth/{self.room_id}/{other_room_event_id}", + ) + + self.assertEqual(channel.code, HTTPStatus.NOT_FOUND, channel.result) + self.assertEqual( + channel.json_body["errcode"], Codes.NOT_FOUND, channel.json_body + ) + + class SendJoinFederationTests(unittest.FederatingHomeserverTestCase): servlets = [ admin.register_servlets, diff --git a/tests/federation/transport/server/test_appservice_proxy.py b/tests/federation/transport/server/test_appservice_proxy.py new file mode 100644 index 0000000000..957e1c3dae --- /dev/null +++ b/tests/federation/transport/server/test_appservice_proxy.py @@ -0,0 +1,349 @@ +# +# This file is licensed under the Affero General Public License (AGPL) version 3. +# +# Copyright (C) 2026 Element Creations Ltd +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# See the GNU Affero General Public License for more details: +# . +# + +import tempfile +from unittest.mock import Mock + +import yaml + +from twisted.internet import defer +from twisted.internet.testing import MemoryReactor +from twisted.web.http_headers import Headers + +from synapse.server import HomeServer +from synapse.types import JsonDict +from synapse.util.clock import Clock + +from tests import unittest +from tests.test_utils import FakeResponse + +APPSERVICE_URL = "http://appservice.example.com" +APPSERVICE_PREFIX = "rtc/livekit" +VERSIONED_PREFIX = f"v1/{APPSERVICE_PREFIX}" + + +class ApplicationServiceFederationProxyTestCase(unittest.FederatingHomeserverTestCase): + """Tests the proxying of federation requests to application services from MSC4512.""" + + def default_config(self) -> JsonDict: + config = super().default_config() + _, path = tempfile.mkstemp(prefix="as_fed_proxy_config") + with open(path, "w") as f: + yaml.dump( + { + "id": "proxy_as", + "url": None, + "as_token": "as_token", + "hs_token": "hs_token", + "sender_localpart": "proxy_bot", + "namespaces": {}, + "io.element.msc4512.proxy_prefix": APPSERVICE_PREFIX, + "io.element.msc4512.proxy_url": APPSERVICE_URL, + }, + f, + ) + config["app_service_config_files"] = [path] + config.setdefault("experimental_features", {}).setdefault( + "msc4512_enabled", True + ) + return config + + def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None: + super().prepare(reactor, clock, hs) + self.agent = Mock() + hs.get_proxied_http_client().agent = self.agent + + def test_signed_get_is_proxied(self) -> None: + self.agent.request = Mock( + return_value=defer.succeed( + FakeResponse.json(code=200, payload={"ok": True}) + ) + ) + + channel = self.make_signed_federation_request( + "GET", f"/_matrix/federation/{VERSIONED_PREFIX}/some/path" + ) + + self.assertEqual(channel.code, 200) + self.assertEqual(channel.json_body, {"ok": True}) + + ((method, uri), kwargs) = self.agent.request.call_args + + self.assertEqual(method, b"GET") + self.assertEqual( + uri, + f"{APPSERVICE_URL}/_matrix/federation/{VERSIONED_PREFIX}/some/path".encode(), + ) + + headers: Headers = kwargs["headers"] + self.assertEqual(headers.getRawHeaders(b"Authorization"), [b"Bearer hs_token"]) + self.assertEqual( + headers.getRawHeaders(b"X-Matrix-Origin"), + [self.OTHER_SERVER_NAME.encode("ascii")], + ) + + def test_signed_get_is_proxied_at_root_path(self) -> None: + self.agent.request = Mock( + return_value=defer.succeed( + FakeResponse.json(code=200, payload={"ok": True}) + ) + ) + + channel = self.make_signed_federation_request( + "GET", f"/_matrix/federation/{VERSIONED_PREFIX}" + ) + + self.assertEqual(channel.code, 200) + self.assertEqual(channel.json_body, {"ok": True}) + + ((method, uri), kwargs) = self.agent.request.call_args + + self.assertEqual(method, b"GET") + self.assertEqual( + uri, + f"{APPSERVICE_URL}/_matrix/federation/{VERSIONED_PREFIX}".encode(), + ) + + headers: Headers = kwargs["headers"] + self.assertEqual(headers.getRawHeaders(b"Authorization"), [b"Bearer hs_token"]) + self.assertEqual( + headers.getRawHeaders(b"X-Matrix-Origin"), + [self.OTHER_SERVER_NAME.encode("ascii")], + ) + + def test_signed_post_is_proxied(self) -> None: + self.agent.request = Mock( + return_value=defer.succeed( + FakeResponse.json(code=200, payload={"ok": True}) + ) + ) + + content = {"key": "value"} + channel = self.make_signed_federation_request( + "POST", + f"/_matrix/federation/{VERSIONED_PREFIX}/some/path", + content=content, + ) + + self.assertEqual(channel.code, 200) + self.assertEqual(channel.json_body, {"ok": True}) + + ((method, uri), kwargs) = self.agent.request.call_args + + self.assertEqual(method, b"POST") + self.assertEqual( + uri, + f"{APPSERVICE_URL}/_matrix/federation/{VERSIONED_PREFIX}/some/path".encode(), + ) + + headers: Headers = kwargs["headers"] + self.assertEqual(headers.getRawHeaders(b"Authorization"), [b"Bearer hs_token"]) + self.assertEqual( + headers.getRawHeaders(b"X-Matrix-Origin"), + [self.OTHER_SERVER_NAME.encode("ascii")], + ) + self.assertEqual( + kwargs["headers"].getRawHeaders(b"Content-Type"), [b"application/json"] + ) + + body_producer = kwargs["bodyProducer"] + self.assertGreater(body_producer.length, 0) + + def test_headers_outside_the_allowlist_not_forwarded(self) -> None: + self.agent.request = Mock( + return_value=defer.succeed(FakeResponse.json(code=200, payload={})) + ) + + self.make_signed_federation_request( + "GET", + f"/_matrix/federation/{VERSIONED_PREFIX}/some/path", + custom_headers=[("Connection", "close"), ("X-Forward", "forward")], + ) + + ((_method, _uri), kwargs) = self.agent.request.call_args + + headers: Headers = kwargs["headers"] + self.assertIsNone(headers.getRawHeaders(b"Connection")) + self.assertIsNone(headers.getRawHeaders(b"X-Forward")) + + def test_allowlisted_headers_forwarded(self) -> None: + self.agent.request = Mock( + return_value=defer.succeed(FakeResponse.json(code=200, payload={})) + ) + + self.make_signed_federation_request( + "GET", + f"/_matrix/federation/{VERSIONED_PREFIX}/some/path", + custom_headers=[ + ("Accept", "application/json"), + ("Accept-Language", "en-US"), + ], + ) + + ((_method, _uri), kwargs) = self.agent.request.call_args + + headers: Headers = kwargs["headers"] + self.assertEqual(headers.getRawHeaders(b"Accept"), [b"application/json"]) + self.assertEqual(headers.getRawHeaders(b"Accept-Language"), [b"en-US"]) + + def test_host_and_content_length_headers_not_forwarded(self) -> None: + self.agent.request = Mock( + return_value=defer.succeed(FakeResponse.json(code=200, payload={})) + ) + + self.make_signed_federation_request( + "POST", + f"/_matrix/federation/{VERSIONED_PREFIX}/some/path", + content={"key": "value"}, + custom_headers=[("Host", "original-client-facing-host.example")], + ) + + ((_method, _uri), kwargs) = self.agent.request.call_args + + headers: Headers = kwargs["headers"] + self.assertIsNone(headers.getRawHeaders(b"Host")) + self.assertIsNone(headers.getRawHeaders(b"Content-Length")) + + def test_response_headers_forwarded(self) -> None: + self.agent.request = Mock( + return_value=defer.succeed( + FakeResponse( + code=200, + body=b"hello", + headers=Headers({"X-Forward": ["forward"]}), + ) + ) + ) + + channel = self.make_signed_federation_request( + "GET", f"/_matrix/federation/{VERSIONED_PREFIX}/some/path" + ) + + self.assertEqual(channel.code, 200) + self.assertEqual(channel.result["body"], b"hello") + self.assertEqual(channel.headers.getRawHeaders(b"X-Forward"), [b"forward"]) + + def test_unsigned_get_is_rejected(self) -> None: + self.agent.request = Mock( + return_value=defer.succeed(FakeResponse.json(code=200, payload={})) + ) + + channel = self.make_request( + "GET", + f"/_matrix/federation/{VERSIONED_PREFIX}/some/path", + shorthand=False, + ) + + self.assertEqual(channel.code, 401) + self.agent.request.assert_not_called() + + @unittest.override_config({"rc_federation": {"reject_limit": -1}}) + def test_rate_limited_request_is_rejected(self) -> None: + self.agent.request = Mock( + return_value=defer.succeed(FakeResponse.json(code=200, payload={})) + ) + + channel = self.make_signed_federation_request( + "GET", f"/_matrix/federation/{VERSIONED_PREFIX}/some/path" + ) + + self.assertEqual(channel.code, 429) + self.agent.request.assert_not_called() + + def test_non_existing_path_under_proxy_prefix_is_rejected(self) -> None: + self.agent.request = Mock(return_value=defer.fail(Exception("boom"))) + + channel = self.make_signed_federation_request( + "GET", f"/_matrix/federation/{VERSIONED_PREFIX}/some/path" + ) + + self.assertEqual(channel.code, 500) + self.agent.request.assert_called() + + def test_path_with_dot_segment_is_rejected(self) -> None: + self.agent.request = Mock( + return_value=defer.succeed(FakeResponse.json(code=200, payload={})) + ) + + channel = self.make_signed_federation_request( + "GET", f"/_matrix/federation/{VERSIONED_PREFIX}/some/../path" + ) + + self.assertEqual(channel.code, 400) + self.assertEqual(channel.json_body["errcode"], "M_INVALID_PARAM") + self.agent.request.assert_not_called() + + def test_path_with_encoded_dot_segment_is_rejected(self) -> None: + self.agent.request = Mock( + return_value=defer.succeed(FakeResponse.json(code=200, payload={})) + ) + + channel = self.make_signed_federation_request( + "GET", f"/_matrix/federation/{VERSIONED_PREFIX}/some/%2e%2e/path" + ) + + self.assertEqual(channel.code, 400) + self.assertEqual(channel.json_body["errcode"], "M_INVALID_PARAM") + self.agent.request.assert_not_called() + + def test_unregistered_prefix_is_rejected(self) -> None: + channel = self.make_signed_federation_request( + "GET", "/_matrix/federation/not_a_registered_prefix/some/path" + ) + + self.assertEqual(channel.code, 404) + + def test_unregistered_prefix_with_suffix_is_rejected(self) -> None: + channel = self.make_signed_federation_request( + "GET", f"/_matrix/federation/{VERSIONED_PREFIX}-2" + ) + + self.assertEqual(channel.code, 404) + + def test_missing_version_segment_is_rejected(self) -> None: + channel = self.make_signed_federation_request( + "GET", f"/_matrix/federation/{APPSERVICE_PREFIX}/some/path" + ) + + self.assertEqual(channel.code, 404) + self.agent.request.assert_not_called() + + def test_unstable_version_segment_is_proxied(self) -> None: + self.agent.request = Mock( + return_value=defer.succeed( + FakeResponse.json(code=200, payload={"ok": True}) + ) + ) + + path = ( + f"/_matrix/federation/unstable/org.example.msc9999/" + f"{APPSERVICE_PREFIX}/some/path" + ) + channel = self.make_signed_federation_request("GET", path) + + self.assertEqual(channel.code, 200) + self.assertEqual(channel.json_body, {"ok": True}) + + ((method, uri), _kwargs) = self.agent.request.call_args + self.assertEqual(method, b"GET") + self.assertEqual(uri, f"{APPSERVICE_URL}{path}".encode()) + + @unittest.override_config({"experimental_features": {"msc4512_enabled": False}}) + def test_proxy_route_not_registered_when_msc4512_disabled(self) -> None: + channel = self.make_signed_federation_request( + "GET", f"/_matrix/federation/{VERSIONED_PREFIX}/some/path" + ) + + self.assertEqual(channel.code, 404) + self.agent.request.assert_not_called() diff --git a/tests/handlers/test_appservice.py b/tests/handlers/test_appservice.py index 08a4ab624a..011d0552a5 100644 --- a/tests/handlers/test_appservice.py +++ b/tests/handlers/test_appservice.py @@ -36,7 +36,7 @@ from twisted.internet.testing import MemoryReactor import synapse.rest.admin import synapse.storage -from synapse.api.constants import EduTypes, EventTypes +from synapse.api.constants import EduTypes, EventTypes, ReceiptTypes from synapse.appservice import ( ApplicationService, TransactionOneTimeKeysCount, @@ -704,6 +704,105 @@ class ApplicationServicesHandlerSendEventsTestCase(unittest.HomeserverTestCase): latest_read_receipt["content"][event_id]["m.read"], {self.local_user: {}} ) + def test_application_services_receive_private_read_receipts_of_namespaced_users_only( + self, + ) -> None: + """Tests that private read receipts are only sent to an application + service for users within the appservice's namespaces, while public read + receipts are sent regardless of the sending user. + + See https://spec.matrix.org/v1.19/application-service-api/#pushing-ephemeral-data + """ + # Register an application service that's interested in a certain user + # and room prefix + interested_appservice = self._register_application_service( + namespaces={ + ApplicationService.NS_USERS: [ + { + "regex": "@exclusive_as_user:.+", + "exclusive": True, + } + ], + ApplicationService.NS_ROOMS: [ + { + "regex": "!fakeroom_.*", + "exclusive": True, + } + ], + }, + ) + + room_id = "!fakeroom_private:test" + event_id = "$eventid" + + # A public read receipt from a user outside the appservice's namespaces. + self.get_success( + self.hs.get_datastores().main.insert_receipt( + room_id=room_id, + receipt_type=ReceiptTypes.READ, + user_id=self.local_user, + event_ids=[event_id], + thread_id=None, + data={}, + ) + ) + # A private read receipt from a user within the appservice's namespaces. + self.get_success( + self.hs.get_datastores().main.insert_receipt( + room_id=room_id, + receipt_type=ReceiptTypes.READ_PRIVATE, + user_id=self.exclusive_as_user, + event_ids=[event_id], + thread_id=None, + data={}, + ) + ) + # A private read receipt on the same event from a user outside the + # appservice's namespaces. + self.get_success( + self.hs.get_datastores().main.insert_receipt( + room_id=room_id, + receipt_type=ReceiptTypes.READ_PRIVATE, + user_id=self.local_user, + event_ids=[event_id], + thread_id=None, + data={}, + ) + ) + + # Notify the appservice handler about the receipts in one go. + # note: stream tokens start at 2, so the three receipts above have + # stream IDs 2, 3 and 4. + self.get_success( + self.hs.get_application_service_handler()._notify_interested_services_ephemeral( + services=[interested_appservice], + stream_key=StreamKeyType.RECEIPT, + new_token=MultiWriterStreamToken(stream=4), + users=[self.local_user, self.exclusive_as_user], + ) + ) + + self.send_mock.assert_called_once() + ephemeral_events = self.send_mock.call_args[0][2] + + # All receipts for the room are batched into a single m.receipt event. + self.assertEqual(len(ephemeral_events), 1) + receipt_event = ephemeral_events[0] + self.assertEqual(receipt_event["type"], EduTypes.RECEIPT) + self.assertEqual(receipt_event["room_id"], room_id) + + # The public read receipt and the namespaced user's private read receipt + # should have been sent, but not the other user's private read receipt. + self.assertEqual( + receipt_event["content"], + { + event_id: { + ReceiptTypes.READ: {self.local_user: {}}, + ReceiptTypes.READ_PRIVATE: {self.exclusive_as_user: {}}, + }, + }, + ) + @unittest.override_config( {"experimental_features": {"msc2409_to_device_messages_enabled": True}} ) diff --git a/tests/handlers/test_deactivate_account.py b/tests/handlers/test_deactivate_account.py index f8b4098c71..fe5c3ff67f 100644 --- a/tests/handlers/test_deactivate_account.py +++ b/tests/handlers/test_deactivate_account.py @@ -21,7 +21,13 @@ from twisted.internet.testing import MemoryReactor -from synapse.api.constants import AccountDataTypes, EventTypes, JoinRules, Membership +from synapse.api.constants import ( + AccountDataTypes, + EventTypes, + JoinRules, + Membership, + ProfileFields, +) from synapse.push.rulekinds import PRIORITY_CLASS_MAP from synapse.rest import admin from synapse.rest.client import account, login, room @@ -515,8 +521,12 @@ class DeactivateAccountTestCase(HomeserverTestCase): # Setting a display name now works again. user = UserID.from_string(self.user) self.get_success( - self.hs.get_profile_handler().set_displayname( - user, create_requester(user), "Reactivated", by_admin=True + self.hs.get_profile_handler().set_field( + target_user=user, + requester=create_requester(user), + field_name=ProfileFields.DISPLAYNAME, + new_value="Reactivated", + by_admin=True, ) ) self.assertEqual( @@ -530,8 +540,12 @@ class DeactivateAccountTestCase(HomeserverTestCase): """ user = UserID.from_string(self.user) self.get_success( - self.hs.get_profile_handler().set_displayname( - user, create_requester(user), "Original", by_admin=True + self.hs.get_profile_handler().set_field( + target_user=user, + requester=create_requester(user), + field_name=ProfileFields.DISPLAYNAME, + new_value="Original", + by_admin=True, ) ) diff --git a/tests/handlers/test_device.py b/tests/handlers/test_device.py index cb047d118a..ce1bd06b6d 100644 --- a/tests/handlers/test_device.py +++ b/tests/handlers/test_device.py @@ -844,6 +844,7 @@ class DeviceUnPartialStateTestCase(unittest.HomeserverTestCase): partial_state=True, # Only REMOTE1_SERVER_NAME is known at join time. servers_in_room={self.REMOTE1_SERVER_NAME}, + state_dag=None, ) ) diff --git a/tests/handlers/test_e2e_keys.py b/tests/handlers/test_e2e_keys.py index a4f9d55a13..4790c339d9 100644 --- a/tests/handlers/test_e2e_keys.py +++ b/tests/handlers/test_e2e_keys.py @@ -817,6 +817,103 @@ class E2eKeysHandlerTestCase(unittest.HomeserverTestCase): self.assertDictEqual(devices["device_keys"][local_user]["abc"], device_key_1) self.assertDictEqual(devices["device_keys"][local_user]["def"], device_key_2) + def test_update_signature_master_key(self) -> None: + """should be able to update a signature on the Master signing key with an unknown algorithm""" + local_user = "@boris:" + self.hs.hostname + master_key: JsonDict = { + # private key: HvQBbU+hc2Zr+JP1sE0XwBe1pfZZEYtJNPJLZJtS+F8 + "user_id": local_user, + "usage": ["master"], + "keys": { + "ed25519:EmkqvokUn8p+vQAGZitOk4PWjp7Ukp3txV2TbMPEiBQ": "EmkqvokUn8p+vQAGZitOk4PWjp7Ukp3txV2TbMPEiBQ" + }, + "signatures": {local_user: {"unknown:abcdefg": "abcdefg"}}, + } + self_signing_key = { + # private key: 2lonYOM6xYKdEsO+6KrC766xBcHnYnim1x/4LFGF8B0 + "user_id": local_user, + "usage": ["self_signing"], + "keys": { + "ed25519:nqOvzeuGWT/sRx3h7+MHoInYj3Uk2LD/unI9kDYcHwk": "nqOvzeuGWT/sRx3h7+MHoInYj3Uk2LD/unI9kDYcHwk" + }, + } + master_signing_key = key.decode_signing_key_base64( + "ed25519", + "EmkqvokUn8p+vQAGZitOk4PWjp7Ukp3txV2TbMPEiBQ", + "HvQBbU+hc2Zr+JP1sE0XwBe1pfZZEYtJNPJLZJtS+F8", + ) + sign.sign_json(self_signing_key, local_user, master_signing_key) + self.get_success( + self.handler.upload_signing_keys_for_user( + local_user, + {"master_key": master_key, "self_signing_key": self_signing_key}, + ) + ) + + device_key: JsonDict = { + "user_id": local_user, + "device_id": "abc", + "algorithms": [ + "m.olm.curve25519-aes-sha2", + RoomEncryptionAlgorithms.MEGOLM_V1_AES_SHA2, + ], + "keys": { + "ed25519:abc": "base64+ed25519+key", + "curve25519:abc": "base64+curve25519+key", + }, + "signatures": {local_user: {"ed25519:abc": "base64+signature"}}, + } + self.get_success( + self.handler.upload_keys_for_user( + local_user, "abc", {"device_keys": device_key} + ) + ) + + # Update the signature and upload it. + master_key["signatures"][local_user]["unknown:abcdefg"] = "ABCDEFG" + self.get_success( + self.handler.upload_signatures_for_device_keys( + local_user, + { + local_user: { + "EmkqvokUn8p+vQAGZitOk4PWjp7Ukp3txV2TbMPEiBQ": master_key + } + }, + ) + ) + + devices = self.get_success( + self.handler.query_devices( + {"device_keys": {local_user: []}}, 0, local_user, "device123" + ) + ) + if "unsigned" in devices["master_keys"][local_user]: + del devices["master_keys"][local_user]["unsigned"] + self.assertDictEqual(devices["master_keys"][local_user], master_key) + + # Update the signature again and upload it. + master_key["signatures"][local_user]["unknown:abcdefg"] = "AbCdEfG" + self.get_success( + self.handler.upload_signatures_for_device_keys( + local_user, + { + local_user: { + "EmkqvokUn8p+vQAGZitOk4PWjp7Ukp3txV2TbMPEiBQ": master_key + } + }, + ) + ) + + # Assert that we were able to update the signature. + devices = self.get_success( + self.handler.query_devices( + {"device_keys": {local_user: []}}, 0, local_user, "device123" + ) + ) + if "unsigned" in devices["master_keys"][local_user]: + del devices["master_keys"][local_user]["unsigned"] + self.assertDictEqual(devices["master_keys"][local_user], master_key) + def test_self_signing_key_doesnt_show_up_as_device(self) -> None: """signing keys should be hidden when fetching a user's devices""" local_user = "@boris:" + self.hs.hostname diff --git a/tests/handlers/test_federation.py b/tests/handlers/test_federation.py index 0c7edbaa2d..71f29be2ea 100644 --- a/tests/handlers/test_federation.py +++ b/tests/handlers/test_federation.py @@ -108,6 +108,46 @@ class FederationTestCase(unittest.FederatingHomeserverTestCase): self.assertEqual(failure.errcode, Codes.FORBIDDEN, failure) self.assertEqual(failure.msg, "You are not invited to this room.") + def test_exchange_third_party_invite_forwards_to_sender_for_v12_room( + self, + ) -> None: + """When we are not resident in the room, a 3pid invite is forwarded to + a remote server for exchange. Pre-v12 room IDs encode the resident + server's domain, but v12+ room IDs are a content hash with no domain, + so the room ID must not be treated as a destination as doing so raises + an invalid-destination error and aborts the exchange. + + Regression test for 3pid invites over federation failing intermittently + in v12 rooms. + """ + sender_user_id = "@sender:remote.example.com" + # A v12-style room ID: a reference hash with no ":domain" suffix. + room_id = "!somereferencehashwithnodomain" + + # Pretend we're not in the room so we take the "forward to a remote + # server" branch. + self.handler._event_auth_handler.is_host_in_room = AsyncMock( # type: ignore[method-assign] + return_value=False + ) + forward = AsyncMock(return_value=None) + self.handler.federation_client.forward_third_party_invite = forward # type: ignore[method-assign] + + self.get_success( + self.handler.exchange_third_party_invite( + sender_user_id=sender_user_id, + target_user_id="@target:localhost", + room_id=room_id, + signed={"mxid": "@target:localhost", "token": "sometoken"}, + ) + ) + + forward.assert_called_once() + destinations = forward.call_args.args[0] + # Only the sender's server is a valid destination; the domainless room + # ID must not be misinterpreted as one. + self.assertEqual(destinations, {"remote.example.com"}) + self.assertNotIn(room_id, destinations) + def test_rejected_message_event_state(self) -> None: """ Check that we store the state group correctly for rejected non-state events. @@ -534,6 +574,7 @@ class PartialJoinTestCase(unittest.FederatingHomeserverTestCase): ], partial_state=True, servers_in_room={"example.com"}, + state_dag=None, ) ) diff --git a/tests/handlers/test_oauth_delegation.py b/tests/handlers/test_oauth_delegation.py index 995a1134b2..71f83607ff 100644 --- a/tests/handlers/test_oauth_delegation.py +++ b/tests/handlers/test_oauth_delegation.py @@ -587,6 +587,34 @@ class DisabledEndpointsTestCase(HomeserverTestCase): "POST", "/_matrix/client/v3/register/msisdn/requestToken" ) + def test_appservice_registration_without_inhibit_login(self) -> None: + """Test that appservice registration without `inhibit_login` is rejected + with the `M_APPSERVICE_LOGIN_UNSUPPORTED` errcode.""" + appservice = ApplicationService( + token="i_am_an_app_service", + id="1234", + namespaces={"users": [{"regex": r"@alice:.+", "exclusive": True}]}, + sender=UserID.from_string("@as_main:test"), + ) + + self.hs.get_datastores().main.services_cache = [appservice] + channel = self.make_request( + "POST", + "/_matrix/client/v3/register", + { + "username": "alice", + "type": "m.login.application_service", + }, + shorthand=False, + access_token="i_am_an_app_service", + ) + self.assertEqual(channel.code, 400, channel.json_body) + self.assertEqual( + channel.json_body.get("errcode"), + "M_APPSERVICE_LOGIN_UNSUPPORTED", + channel.json_body, + ) + def test_session_management_endpoints_removed(self) -> None: """Test that session management endpoints that were removed in MSC2964 are no longer available.""" self.expect_unrecognized("GET", "/_matrix/client/v3/login") diff --git a/tests/handlers/test_presence.py b/tests/handlers/test_presence.py index 88d9d2aebb..a562050842 100644 --- a/tests/handlers/test_presence.py +++ b/tests/handlers/test_presence.py @@ -51,6 +51,8 @@ from synapse.handlers.presence import ( FEDERATION_TIMEOUT, PresenceHandler, WorkerPresenceHandler, + get_interested_parties, + get_interested_remotes, handle_timeout, handle_update, ) @@ -1000,6 +1002,125 @@ class PresenceHandlerInitTestCase(unittest.HomeserverTestCase): ) self.assertEqual(state.state, sync_state) + @unittest.override_config({"presence": {"enabled": False}}) + def test_restored_presence_flushed_offline_when_presence_disabled(self) -> None: + """If presence is disabled, any non-offline presence states left in the + database from when presence was enabled should be marked as offline at + startup, and the updates streamed out to clients. + """ + main_store = self.hs.get_datastores().main + before_token = main_store.get_current_presence_token() + + # Get the handler, which schedules the startup flush. + presence_handler = self.hs.get_presence_handler() + + # Fire pending `call_when_running` hooks and let the flush complete. + self.reactor.run() + self.reactor.advance(0) + + # The user should now be offline, both in memory and in the database. + state = self.get_success( + presence_handler.get_state(UserID.from_string(self.user_id)) + ) + self.assertEqual(state.state, PresenceState.OFFLINE) + + db_state = self.get_success(main_store.get_presence_for_users([self.user_id]))[ + self.user_id + ] + self.assertEqual(db_state.state, PresenceState.OFFLINE) + + # The flush must advance the presence stream so that syncing clients + # are sent the offline updates. + self.assertGreater(main_store.get_current_presence_token(), before_token) + + +class PresenceDisabledSyncTestCase(unittest.HomeserverTestCase): + """Tests that stale presence states left over from when presence was + enabled reach clients over /sync, and that the startup flush marks them as + offline and sends the offline updates down /sync too. + """ + + servlets = [ + admin.register_servlets, + login.register_servlets, + room.register_servlets, + sync.register_servlets, + ] + + @unittest.override_config({"presence": {"enabled": False}}) + def test_stale_presence_flushed_offline_and_sent_on_sync(self) -> None: + user1 = self.register_user("alice", "pass") + user1_tok = self.login(user1, "pass") + user2 = self.register_user("bob", "pass") + user2_tok = self.login(user2, "pass") + + room_id = self.helper.create_room_as(user1, tok=user1_tok) + self.helper.join(room_id, user2, tok=user2_tok) + + channel = self.make_request("GET", "/sync", access_token=user2_tok) + self.assertEqual(channel.code, 200, channel.json_body) + next_batch = channel.json_body["next_batch"] + + # Seed a stale online presence state for user1, left over from when + # presence was enabled: in the database, and in the presence handler's + # in-memory state (which at startup is preloaded from the database). + now = self.clock.time_msec() + stale_state = UserPresenceState( + user_id=user1, + state=PresenceState.ONLINE, + last_active_ts=now, + last_federation_update_ts=now, + last_user_sync_ts=now, + status_msg=None, + currently_active=True, + ) + main_store = self.hs.get_datastores().main + self.get_success(main_store.update_presence([stale_state])) + + presence_handler = self.hs.get_presence_handler() + assert isinstance(presence_handler, PresenceHandler) + presence_handler.user_to_current_state[user1] = stale_state + + # The stale state comes down user2's incremental sync, even though + # presence is disabled. + channel = self.make_request( + "GET", f"/sync?since={next_batch}", access_token=user2_tok + ) + self.assertEqual(channel.code, 200, channel.json_body) + presence_events = channel.json_body["presence"]["events"] + self.assertEqual( + [(e["sender"], e["content"]["presence"]) for e in presence_events], + [(user1, PresenceState.ONLINE)], + ) + next_batch = channel.json_body["next_batch"] + + # Run the startup flush, as scheduled when the presence writer starts + # up with presence disabled. + self.get_success(presence_handler._mark_stale_presence_as_offline()) + + # The stale state should have been marked offline in the database... + db_state = self.get_success(main_store.get_presence_for_users([user1]))[user1] + self.assertEqual(db_state.state, PresenceState.OFFLINE) + + # ... and the offline update also comes down user2's sync. + channel = self.make_request( + "GET", f"/sync?since={next_batch}", access_token=user2_tok + ) + self.assertEqual(channel.code, 200, channel.json_body) + presence_events = channel.json_body["presence"]["events"] + self.assertEqual( + [(e["sender"], e["content"]["presence"]) for e in presence_events], + [(user1, PresenceState.OFFLINE)], + ) + + # Once caught up, further syncs include no presence. + next_batch = channel.json_body["next_batch"] + channel = self.make_request( + "GET", f"/sync?since={next_batch}", access_token=user2_tok + ) + self.assertEqual(channel.code, 200, channel.json_body) + self.assertEqual(channel.json_body.get("presence", {}).get("events", []), []) + # Timer values used by `PresenceConfigurableTimersTestCase`, all larger than # the corresponding defaults. @@ -2323,6 +2444,217 @@ class PresenceJoinTestCase(unittest.HomeserverTestCase): return event +class PresenceExcludeRoomsTestCase(unittest.HomeserverTestCase): + """Tests that `exclude_rooms_from_presence` stops presence being routed + between users solely because they share an excluded room.""" + + servlets = [ + admin.register_servlets, + login.register_servlets, + room.register_servlets, + ] + + def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None: + self.hs = hs + self.store = hs.get_datastores().main + self.presence_router = hs.get_presence_router() + self.presence_handler = hs.get_presence_handler() + + self.user1 = self.register_user("user1", "pass") + self.token1 = self.login("user1", "pass") + self.user2 = self.register_user("user2", "pass") + self.token2 = self.login("user2", "pass") + + def test_excluded_rooms_not_routed(self) -> None: + # Two rooms that user1 is joined to. + excluded_room = self.helper.create_room_as(self.user1, tok=self.token1) + shared_room = self.helper.create_room_as(self.user1, tok=self.token1) + + state = UserPresenceState.default(self.user1) + + # Without any exclusions both rooms are interested in user1's presence. + room_ids_to_states, users_to_states = self.get_success( + get_interested_parties(self.store, self.presence_router, [state]) + ) + self.assertIn(excluded_room, room_ids_to_states) + self.assertIn(shared_room, room_ids_to_states) + + # Excluding one room drops it as an interested party, but the other + # (non-excluded) room still routes presence... + room_ids_to_states, users_to_states = self.get_success( + get_interested_parties( + self.store, + self.presence_router, + [state], + frozenset({excluded_room}), + ) + ) + self.assertNotIn(excluded_room, room_ids_to_states) + self.assertIn(shared_room, room_ids_to_states) + + # ...and the user always receives their own presence, even when all of + # their rooms are excluded. + room_ids_to_states, users_to_states = self.get_success( + get_interested_parties( + self.store, + self.presence_router, + [state], + frozenset({excluded_room, shared_room}), + ) + ) + self.assertNotIn(excluded_room, room_ids_to_states) + self.assertNotIn(shared_room, room_ids_to_states) + self.assertIn(self.user1, users_to_states) + + @override_config({"exclude_rooms_from_presence": ["!excluded:test"]}) + def test_config_populates_handler(self) -> None: + """The config option should be plumbed through to the presence handler + and the presence event source as a frozenset.""" + self.assertEqual( + self.presence_handler._rooms_to_exclude_from_presence, + frozenset({"!excluded:test"}), + ) + + event_source = self.hs.get_event_sources().sources.presence + self.assertEqual( + event_source._rooms_to_exclude_from_presence, + frozenset({"!excluded:test"}), + ) + + def test_is_visible_respects_excluded_rooms(self) -> None: + """`is_visible` (which drives the read side of /sync) should not + consider two users to share presence solely via an excluded room.""" + user1 = UserID.from_string(self.user1) + user2 = UserID.from_string(self.user2) + + # A single shared room: the two users can see each other's presence. + excluded_room = self.helper.create_room_as(self.user1, tok=self.token1) + self.helper.join(excluded_room, self.user2, tok=self.token2) + + self.assertTrue( + self.get_success(self.presence_handler.is_visible(user2, user1)) + ) + + # Excluding the only shared room hides presence between them. + self.presence_handler._rooms_to_exclude_from_presence = frozenset( + {excluded_room} + ) + self.assertFalse( + self.get_success(self.presence_handler.is_visible(user2, user1)) + ) + + # But a second, non-excluded shared room restores visibility. + shared_room = self.helper.create_room_as(self.user1, tok=self.token1) + self.helper.join(shared_room, self.user2, tok=self.token2) + self.assertTrue( + self.get_success(self.presence_handler.is_visible(user2, user1)) + ) + + def test_get_interested_remotes_respects_excluded_rooms(self) -> None: + """The federation fan-out side (`get_interested_remotes`) must not route + presence to servers reached solely via an excluded room.""" + excluded_room = self.helper.create_room_as(self.user1, tok=self.token1) + state = UserPresenceState.default(self.user1) + + def hosts_for(excluded: frozenset) -> set: + result = self.get_success( + get_interested_remotes( + self.store, self.presence_router, [state], excluded + ) + ) + hosts: set[str] = set() + for room_hosts, _ in result: + hosts.update(room_hosts) + return hosts + + # The local server is a host in the room (all members are local here), + # so presence would be routed there... + self.assertIn("test", hosts_for(frozenset())) + # ...but excluding the only room removes it as a source of destinations. + self.assertNotIn("test", hosts_for(frozenset({excluded_room}))) + + +class PresenceGetNewEventsStreamTestCase(unittest.HomeserverTestCase): + """Tests the incremental (`from_key`) branch of + `PresenceEventSource.get_new_events`, which decides which updated users are + interesting to the syncing user by intersecting their cached room sets. + """ + + servlets = [ + admin.register_servlets, + login.register_servlets, + room.register_servlets, + ] + + def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None: + self.presence_handler = hs.get_presence_handler() + self.event_source = hs.get_event_sources().sources.presence + + self.user1 = self.register_user("user1", "pass") + self.token1 = self.login("user1", "pass") + self.user2 = self.register_user("user2", "pass") + self.token2 = self.login("user2", "pass") + self.user3 = self.register_user("user3", "pass") + self.token3 = self.login("user3", "pass") + + def _set_presence(self, user_id: str, state: str = "online") -> None: + self.get_success( + self.presence_handler.set_state( + UserID.from_string(user_id), "dev", {"presence": state} + ) + ) + + def _updated_users_seen_by(self, user_id: str, from_key: int) -> set[str]: + states, _ = self.get_success( + self.event_source.get_new_events( + user=UserID.from_string(user_id), from_key=from_key + ) + ) + return {state.user_id for state in states} + + def test_incremental_interest(self) -> None: + """A syncing user sees updates from users they share a room with (and + themselves), but not from strangers.""" + shared_room = self.helper.create_room_as(self.user1, tok=self.token1) + self.helper.join(shared_room, self.user2, tok=self.token2) + # user3 is in an unrelated room. + self.helper.create_room_as(self.user3, tok=self.token3) + + from_key = self.event_source.get_current_key() + self._set_presence(self.user1) + self._set_presence(self.user2) + self._set_presence(self.user3) + + seen = self._updated_users_seen_by(self.user2, from_key) + self.assertIn(self.user1, seen) + self.assertIn(self.user2, seen) # always sees own updates + self.assertNotIn(self.user3, seen) + + def test_incremental_interest_excluded_room(self) -> None: + """Sharing only an excluded room does not make an updated user + interesting; sharing an additional normal room does.""" + excluded_room = self.helper.create_room_as(self.user1, tok=self.token1) + self.helper.join(excluded_room, self.user2, tok=self.token2) + + self.event_source._rooms_to_exclude_from_presence = frozenset({excluded_room}) + + from_key = self.event_source.get_current_key() + self._set_presence(self.user1, "online") + seen = self._updated_users_seen_by(self.user2, from_key) + self.assertNotIn(self.user1, seen) + + # A second, non-excluded shared room restores interest. (Use a + # different presence state, as repeating the same one would not + # generate a new update.) + shared_room = self.helper.create_room_as(self.user1, tok=self.token1) + self.helper.join(shared_room, self.user2, tok=self.token2) + + from_key = self.event_source.get_current_key() + self._set_presence(self.user1, "unavailable") + seen = self._updated_users_seen_by(self.user2, from_key) + self.assertIn(self.user1, seen) + + class WorkerPresenceThrottleTestCase(BaseMultiWorkerStreamTestCase): """Tests that sync workers suppress the per-sync-request presence updates that the presence writer would discard anyway, while relaying genuine diff --git a/tests/handlers/test_profile.py b/tests/handlers/test_profile.py index 561b45827f..7af0345c87 100644 --- a/tests/handlers/test_profile.py +++ b/tests/handlers/test_profile.py @@ -26,12 +26,17 @@ from parameterized import parameterized from twisted.internet.testing import MemoryReactor import synapse.types -from synapse.api.constants import EventTypes -from synapse.api.errors import AuthError, SynapseError +from synapse.api.constants import ( + EventTypes, + ProfileFields, + ProfileUpdateAction, +) +from synapse.api.errors import AuthError, Codes, SynapseError from synapse.rest import admin -from synapse.rest.client import login, room +from synapse.rest.client import knock, login, room from synapse.server import HomeServer -from synapse.types import JsonDict, UserID +from synapse.storage.databases.main.profile import ProfileUpdate +from synapse.types import JsonDict, StreamKeyType, UserID from synapse.types.state import StateFilter from synapse.util.clock import Clock from synapse.util.duration import Duration @@ -48,6 +53,7 @@ class ProfileTestCase(unittest.HomeserverTestCase): admin.register_servlets, login.register_servlets, room.register_servlets, + knock.register_servlets, ] def make_homeserver(self, reactor: MemoryReactor, clock: Clock) -> HomeServer: @@ -62,8 +68,10 @@ class ProfileTestCase(unittest.HomeserverTestCase): self.query_handlers[query_type] = handler self.mock_registry.register_query_handler = register_query_handler + self.mock_hs_notifier = Mock() hs = self.setup_test_homeserver( + notifier=self.mock_hs_notifier, federation_client=self.mock_federation, federation_server=Mock(), federation_registry=self.mock_registry, @@ -83,9 +91,17 @@ class ProfileTestCase(unittest.HomeserverTestCase): self.frank_token = self.login(self.frank.localpart, "frankpassword") self.handler = hs.get_profile_handler() + self.on_new_event = self.mock_hs_notifier.on_new_event def test_get_my_name(self) -> None: - self.get_success(self.store.set_profile_displayname(self.frank, "Frank")) + self.get_success( + self.handler.set_field( + target_user=self.frank, + requester=synapse.types.create_requester(self.frank), + field_name=ProfileFields.DISPLAYNAME, + new_value="Frank", + ) + ) displayname = self.get_success(self.handler.get_displayname(self.frank)) @@ -93,8 +109,11 @@ class ProfileTestCase(unittest.HomeserverTestCase): def test_set_my_name(self) -> None: self.get_success( - self.handler.set_displayname( - self.frank, synapse.types.create_requester(self.frank), "Frank Jr." + self.handler.set_field( + target_user=self.frank, + requester=synapse.types.create_requester(self.frank), + field_name=ProfileFields.DISPLAYNAME, + new_value="Frank Jr.", ) ) @@ -105,8 +124,11 @@ class ProfileTestCase(unittest.HomeserverTestCase): # Set displayname again self.get_success( - self.handler.set_displayname( - self.frank, synapse.types.create_requester(self.frank), "Frank" + self.handler.set_field( + target_user=self.frank, + requester=synapse.types.create_requester(self.frank), + field_name=ProfileFields.DISPLAYNAME, + new_value="Frank", ) ) @@ -117,8 +139,11 @@ class ProfileTestCase(unittest.HomeserverTestCase): # Set displayname to an empty string self.get_success( - self.handler.set_displayname( - self.frank, synapse.types.create_requester(self.frank), "" + self.handler.set_field( + target_user=self.frank, + requester=synapse.types.create_requester(self.frank), + field_name=ProfileFields.DISPLAYNAME, + new_value="", ) ) @@ -130,8 +155,11 @@ class ProfileTestCase(unittest.HomeserverTestCase): """Test that `set_displayname` updates membership events in rooms.""" self.get_success( - self.handler.set_displayname( - self.frank, synapse.types.create_requester(self.frank), "Frank" + self.handler.set_field( + target_user=self.frank, + requester=synapse.types.create_requester(self.frank), + field_name=ProfileFields.DISPLAYNAME, + new_value="Frank", ) ) @@ -149,8 +177,11 @@ class ProfileTestCase(unittest.HomeserverTestCase): self.assertEqual(membership[state_tuple].content["displayname"], "Frank") self.get_success( - self.handler.set_displayname( - self.frank, synapse.types.create_requester(self.frank), "Frank Jr." + self.handler.set_field( + target_user=self.frank, + requester=synapse.types.create_requester(self.frank), + field_name=ProfileFields.DISPLAYNAME, + new_value="Frank Jr.", ) ) @@ -161,12 +192,545 @@ class ProfileTestCase(unittest.HomeserverTestCase): ) self.assertEqual(membership[state_tuple].content["displayname"], "Frank Jr.") + @parameterized.expand( + [ + ["displayname", "Frank"], + ["avatar_url", "mxc://foobar"], + ["m.status", '{"text": "Holiday", "emoji": "🏖"}'], + ] + ) + def test_update_profile_does_not_update_stream_on_set_field_if_include_profile_updates_in_sync_not_enabled( + self, + field_name: str, + new_value: str, + ) -> None: + """Test that profile updates don't get recorded in the profile updates stream + if `include_profile_updates_in_sync` is not enabled.""" + self.get_success( + self.handler.set_field( + target_user=self.frank, + requester=synapse.types.create_requester(self.frank), + field_name=field_name, + new_value=new_value, + ) + ) + updates = self.get_success( + self.store.get_updated_profile_updates( + from_id=1, + to_id=2, + limit=1, + ) + ) + self.assertEqual(len(updates), 0) + + @parameterized.expand( + [ + ["displayname", "Frank"], + ["avatar_url", "mxc://foobar"], + ["m.status", '{"text": "Holiday", "emoji": "🏖"}'], + ] + ) + def test_update_profile_does_not_notify_notifier_on_set_field_if_include_profile_updates_in_sync_not_enabled( + self, + field_name: str, + new_value: str, + ) -> None: + """Test that profile updates do not cause the profile updates stream notifier + to wake up if `include_profile_updates_in_sync` is not enabled.""" + self.get_success( + self.handler.set_field( + target_user=self.frank, + requester=synapse.types.create_requester(self.frank), + field_name=field_name, + new_value=new_value, + ) + ) + + calls_found = [ + call + for call in self.on_new_event.mock_calls + if call.args[0] == StreamKeyType.PROFILE_UPDATES + ] + self.assertEqual(len(calls_found), 0) + + @parameterized.expand( + [ + ["displayname", "Frank"], + ["avatar_url", "mxc://foobar"], + ["m.status", '{"text": "Holiday", "emoji": "🏖"}'], + ] + ) + @override_config({"include_profile_updates_in_sync": True}) + def test_update_profile_does_notify_notifier_on_set_field_if_user_not_in_rooms( + self, field_name: str, new_value: str + ) -> None: + """Test that profile updates does cause the profile updates stream notifier + to wake up if the user is not in any rooms, if `include_profile_updates_in_sync` + is enabled.""" + self.get_success( + self.handler.set_field( + target_user=self.frank, + requester=synapse.types.create_requester(self.frank), + field_name=field_name, + new_value=new_value, + ) + ) + calls_found = [ + call + for call in self.on_new_event.mock_calls + if call.args[0] == StreamKeyType.PROFILE_UPDATES + ] + self.assertEqual(len(calls_found), 1) + + @override_config({"include_profile_updates_in_sync": True}) + def test_update_profile_does_notify_notifier_on_delete_profile_field_if_user_not_in_rooms( + self, + ) -> None: + """Test that profile updates does cause the profile updates stream notifier + to wake up if the user is not in any rooms, if `include_profile_updates_in_sync` + is enabled.""" + self.get_success( + self.handler.set_field( + target_user=self.frank, + requester=synapse.types.create_requester(self.frank), + field_name="field", + new_value="value", + ) + ) + self.on_new_event.reset_mock() + self.get_success( + self.handler.delete_profile_field( + target_user=self.frank, + requester=synapse.types.create_requester(self.frank), + field_name="field", + ) + ) + calls_found = [ + call + for call in self.on_new_event.mock_calls + if call.args[0] == StreamKeyType.PROFILE_UPDATES + ] + self.assertEqual(len(calls_found), 1) + + @parameterized.expand( + [ + ["displayname", "Frank"], + ["avatar_url", "mxc://foobar"], + ["m.status", '{"text": "Holiday", "emoji": "🏖"}'], + ] + ) + @override_config({"include_profile_updates_in_sync": True}) + def test_update_profile_updates_stream_on_set_field( + self, field_name: str, new_value: str + ) -> None: + """Test that profile updates get recorded in the profile updates stream if + `include_profile_updates_in_sync` is enabled.""" + self.get_success( + self.handler.set_field( + target_user=self.frank, + requester=synapse.types.create_requester(self.frank), + field_name=field_name, + new_value=new_value, + ) + ) + updates = self.get_success( + self.store.get_updated_profile_updates( + from_id=1, + to_id=2, + limit=1, + ) + ) + self.assertEqual( + updates[0], + ( + 2, + "@1234abcd:test", + ProfileUpdateAction.UPDATE.value, + {field_name}, + ), + ) + + fields_updates = self.get_success( + # FIXME this function should be deleted, it's not used. + # Adapt this test to use the right one. + self.store.get_profile_updates_for_fields( + from_id=1, + to_id=2, + field_names={field_name}, + ) + ) + self.assertEqual( + fields_updates[0], + ProfileUpdate( + stream_id=2, + user_id="@1234abcd:test", + action=ProfileUpdateAction.UPDATE.value, + affected_fields=frozenset({field_name}), + ), + ) + + self.get_success( + self.handler.set_field( + target_user=self.frank, + requester=synapse.types.create_requester(self.frank), + field_name=field_name, + new_value="", + ) + ) + delete_updates = self.get_success( + self.store.get_updated_profile_updates( + from_id=2, + to_id=3, + limit=1, + ) + ) + self.assertEqual( + delete_updates[0], + (3, "@1234abcd:test", ProfileUpdateAction.UPDATE.value, {field_name}), + ) + + @override_config({"include_profile_updates_in_sync": True}) + def test_update_profile_set_field_writes_to_per_user_profile_tracking_table( + self, + ) -> None: + """Test that profiles updates get recorded in the 'per user' profile updates + stream tracking table, if `include_profile_updates_in_sync` is enabled.""" + self.register_user("roger", "password") + roger_token = self.login("roger", "password") + self.register_user("millie", "password") + millie_token = self.login("millie", "password") + room_id = self.helper.create_room_as( + room_creator=self.frank.to_string(), + tok=self.frank_token, + ) + self.helper.join(room_id, "@roger:test", tok=roger_token) + self.helper.join(room_id, "@millie:test", tok=millie_token) + self.get_success( + self.handler.set_field( + target_user=self.frank, + requester=synapse.types.create_requester(self.frank), + field_name="m.status", + new_value='{"text": "Holiday"}', + ) + ) + per_user_updates = self.get_success( + self.store.get_profile_updates_for_user_and_fields( + from_id=0, + to_id=10, + user_id="@roger:test", + field_names={"m.status"}, + ) + ) + self.assertEqual( + per_user_updates, + [ + ProfileUpdate( + stream_id=3, + user_id="@millie:test", + action="joined_room", + affected_fields=None, + ), + ProfileUpdate( + stream_id=4, + user_id=self.frank.to_string(), + action="update", + affected_fields=frozenset({"m.status"}), + ), + ], + ) + per_user_updates = self.get_success( + self.store.get_profile_updates_for_user_and_fields( + from_id=0, + to_id=10, + user_id="@millie:test", + field_names={"m.status"}, + ) + ) + self.assertEqual( + per_user_updates, + [ + ProfileUpdate( + stream_id=4, + user_id=self.frank.to_string(), + action="update", + affected_fields=frozenset({"m.status"}), + ), + ], + ) + per_user_updates = self.get_success( + self.store.get_profile_updates_for_user_and_fields( + from_id=0, + to_id=10, + user_id=self.frank.to_string(), + field_names={"m.status"}, + ) + ) + self.assertEqual( + per_user_updates, + [ + ProfileUpdate( + stream_id=2, + user_id="@roger:test", + action="joined_room", + affected_fields=None, + ), + ProfileUpdate( + stream_id=3, + user_id="@millie:test", + action="joined_room", + affected_fields=None, + ), + ProfileUpdate( + stream_id=4, + user_id=self.frank.to_string(), + action="update", + affected_fields=frozenset({"m.status"}), + ), + ], + ) + + @override_config({"include_profile_updates_in_sync": True}) + def test_membership_addition_to_room_adds_the_right_join_action_to_profile_streams( + self, + ) -> None: + """Test that a membership event, which adds a user as joined to a room, + adds the relevant joined action to the profile update stream tables. + + Here we consider join, knock and invite to all be additions to the room + list of members for answering the question "which profiles should we send + information about to clients based on memberships appearing". + """ + self.register_user("roger", "password") + roger_token = self.login("roger", "password") + room_id = self.helper.create_room_as( + room_creator=self.frank.to_string(), + tok=self.frank_token, + ) + + self.helper.join(room_id, "@roger:test", tok=roger_token) + + per_user_updates = self.get_success( + self.store.get_profile_updates_for_user_and_fields( + from_id=0, + to_id=10, + user_id=self.frank.to_string(), + field_names={"m.status"}, + ) + ) + self.assertEqual( + per_user_updates, + [ + ProfileUpdate( + stream_id=2, + user_id="@roger:test", + action="joined_room", + affected_fields=None, + ), + ], + ) + + @override_config({"include_profile_updates_in_sync": True}) + def test_previous_profile_updates_stream_rows_cleared_if_no_longer_sharing_a_room( + self, + ) -> None: + """Test that previous profile update stream rows are removed for a user if + the user no longer shares rooms with another user, if + `include_profile_updates_in_sync` is enabled. + + This test ensures that when a user leaves a room, we clear all old profile + update rows of users who the user no longer shares rooms with, to avoid + leaking any further profile field updates from those users. + """ + self.register_user("roger", "password") + roger_token = self.login("roger", "password") + self.register_user("millie", "password") + millie_token = self.login("millie", "password") + self.register_user("gracie", "password") + gracie_token = self.login("gracie", "password") + room_id = self.helper.create_room_as( + room_creator=self.frank.to_string(), + tok=self.frank_token, + ) + room_with_millie_id = self.helper.create_room_as( + room_creator=self.frank.to_string(), + tok=self.frank_token, + ) + self.helper.join(room_id, "@roger:test", tok=roger_token) + self.helper.join(room_with_millie_id, "@millie:test", tok=millie_token) + self.helper.join(room_id, "@gracie:test", tok=gracie_token) + self.get_success( + self.handler.set_field( + target_user=self.frank, + requester=synapse.types.create_requester(self.frank), + field_name="m.status", + new_value='{"text": "Holiday"}', + ) + ) + per_user_updates = self.get_success( + self.store.get_profile_updates_for_user_and_fields( + from_id=0, + to_id=10, + user_id="@roger:test", + field_names={"m.status"}, + ) + ) + self.assertEqual( + per_user_updates, + [ + ProfileUpdate( + stream_id=4, + user_id="@gracie:test", + action="joined_room", + affected_fields=None, + ), + ProfileUpdate( + stream_id=5, + user_id=self.frank.to_string(), + action="update", + affected_fields=frozenset({"m.status"}), + ), + ], + ) + per_user_updates = self.get_success( + self.store.get_profile_updates_for_user_and_fields( + from_id=0, + to_id=10, + user_id="@millie:test", + field_names={"m.status"}, + ) + ) + self.assertEqual( + per_user_updates, + [ + ProfileUpdate( + stream_id=5, + user_id=self.frank.to_string(), + action="update", + affected_fields=frozenset({"m.status"}), + ), + ], + ) + + # Make frank leave room and verify only the "left room" + gracies join exists + # for roger + self.helper.leave(room_id, self.frank.to_string(), tok=self.frank_token) + per_user_updates = self.get_success( + self.store.get_profile_updates_for_user_and_fields( + from_id=0, + to_id=10, + user_id="@roger:test", + field_names={"m.status"}, + ) + ) + self.assertEqual( + per_user_updates, + [ + ProfileUpdate( + stream_id=4, + user_id="@gracie:test", + action="joined_room", + affected_fields=None, + ), + ProfileUpdate( + stream_id=6, + user_id=self.frank.to_string(), + action="left_room", + affected_fields=None, + ), + ], + ) + # Make gracie leave room and verify only the "left room"'s + self.helper.leave(room_id, "@gracie:test", tok=gracie_token) + per_user_updates = self.get_success( + self.store.get_profile_updates_for_user_and_fields( + from_id=0, + to_id=10, + user_id="@roger:test", + field_names={"m.status"}, + ) + ) + self.assertEqual( + per_user_updates, + [ + ProfileUpdate( + stream_id=6, + user_id=self.frank.to_string(), + action="left_room", + affected_fields=None, + ), + ProfileUpdate( + stream_id=7, + user_id="@gracie:test", + action="left_room", + affected_fields=None, + ), + ], + ) + + # Sanity check we didn't clear any rows for millie + per_user_updates = self.get_success( + self.store.get_profile_updates_for_user_and_fields( + from_id=0, + to_id=10, + user_id="@millie:test", + field_names={"m.status"}, + ) + ) + self.assertEqual( + per_user_updates, + [ + ProfileUpdate( + stream_id=5, + user_id=self.frank.to_string(), + action="update", + affected_fields=frozenset({"m.status"}), + ), + ], + ) + + @parameterized.expand( + [ + ["displayname", "Frank"], + ["avatar_url", "mxc://foobar"], + ["m.status", '{"text": "Holiday", "emoji": "🏖"}'], + ] + ) + @override_config({"include_profile_updates_in_sync": True}) + def test_update_profile_notifies_notifier_on_set_field( + self, + field_name: str, + new_value: str, + ) -> None: + """Test that profile updates wake up the profile updates stream on profile + field updates, if `include_profile_updates_in_sync` is enabled.""" + self.helper.create_room_as( + room_creator=self.frank.to_string(), + tok=self.frank_token, + ) + self.get_success( + self.handler.set_field( + target_user=self.frank, + requester=synapse.types.create_requester(self.frank), + field_name=field_name, + new_value=new_value, + ) + ) + calls_found = [ + call + for call in self.on_new_event.mock_calls + if call.args[0] == StreamKeyType.PROFILE_UPDATES + ] + self.assertEqual(len(calls_found), 1) + def test_background_update_room_membership_on_set_displayname(self) -> None: """Test that `set_displayname` returns immediately and that room membership updates are still done in background.""" self.get_success( - self.handler.set_displayname( - self.frank, synapse.types.create_requester(self.frank), "Frank" + self.handler.set_field( + target_user=self.frank, + requester=synapse.types.create_requester(self.frank), + field_name=ProfileFields.DISPLAYNAME, + new_value="Frank", ) ) @@ -187,8 +751,11 @@ class ProfileTestCase(unittest.HomeserverTestCase): ): state_tuple = (EventTypes.Member, self.frank.to_string()) self.get_success( - self.handler.set_displayname( - self.frank, synapse.types.create_requester(self.frank), "Frank Jr." + self.handler.set_field( + target_user=self.frank, + requester=synapse.types.create_requester(self.frank), + field_name=ProfileFields.DISPLAYNAME, + new_value="Frank Jr.", ) ) @@ -215,8 +782,11 @@ class ProfileTestCase(unittest.HomeserverTestCase): """Test that room membership updates triggered by changing the avatar or the display name are resumed after a restart.""" self.get_success( - self.handler.set_displayname( - self.frank, synapse.types.create_requester(self.frank), "Frank" + self.handler.set_field( + target_user=self.frank, + requester=synapse.types.create_requester(self.frank), + field_name=ProfileFields.DISPLAYNAME, + new_value="Frank", ) ) @@ -253,8 +823,11 @@ class ProfileTestCase(unittest.HomeserverTestCase): ): state_tuple = (EventTypes.Member, self.frank.to_string()) self.get_success( - self.handler.set_displayname( - self.frank, synapse.types.create_requester(self.frank), "Frank Jr." + self.handler.set_field( + target_user=self.frank, + requester=synapse.types.create_requester(self.frank), + field_name=ProfileFields.DISPLAYNAME, + new_value="Frank Jr.", ) ) @@ -320,7 +893,13 @@ class ProfileTestCase(unittest.HomeserverTestCase): @override_config({"enable_set_displayname": False}) def test_set_my_name_if_disabled(self) -> None: # Setting displayname for the first time is allowed - self.get_success(self.store.set_profile_displayname(self.frank, "Frank")) + self.get_success( + self.store.set_profile_field( + user_id=self.frank, + field_name=ProfileFields.DISPLAYNAME, + new_value="Frank", + ) + ) self.assertEqual( (self.get_success(self.store.get_profile_displayname(self.frank))), @@ -328,17 +907,25 @@ class ProfileTestCase(unittest.HomeserverTestCase): ) # Setting displayname a second time is forbidden - self.get_failure( - self.handler.set_displayname( - self.frank, synapse.types.create_requester(self.frank), "Frank Jr." + f = self.get_failure( + self.handler.set_field( + target_user=self.frank, + requester=synapse.types.create_requester(self.frank), + field_name=ProfileFields.DISPLAYNAME, + new_value="Frank Jr.", ), SynapseError, ) + self.assertEqual(f.value.code, 403) + self.assertEqual(f.value.errcode, Codes.FORBIDDEN) def test_set_my_name_noauth(self) -> None: self.get_failure( - self.handler.set_displayname( - self.frank, synapse.types.create_requester(self.bob), "Frank Jr." + self.handler.set_field( + target_user=self.frank, + requester=synapse.types.create_requester(self.bob), + field_name=ProfileFields.DISPLAYNAME, + new_value="Frank Jr.", ), AuthError, ) @@ -361,8 +948,11 @@ class ProfileTestCase(unittest.HomeserverTestCase): self.store.create_profile(UserID.from_string("@caroline:test")) ) self.get_success( - self.store.set_profile_displayname( - UserID.from_string("@caroline:test"), "Caroline" + self.handler.set_field( + target_user=UserID.from_string("@caroline:test"), + requester=synapse.types.create_requester("@caroline:test"), + field_name=ProfileFields.DISPLAYNAME, + new_value="Caroline", ) ) @@ -380,16 +970,31 @@ class ProfileTestCase(unittest.HomeserverTestCase): def test_get_my_avatar(self) -> None: self.get_success( - self.store.set_profile_avatar_url(self.frank, "http://my.server/me.png") + self.handler.set_field( + target_user=self.frank, + requester=synapse.types.create_requester(self.frank), + field_name=ProfileFields.AVATAR_URL, + new_value="http://my.server/me.png", + ) ) avatar_url = self.get_success(self.handler.get_avatar_url(self.frank)) self.assertEqual("http://my.server/me.png", avatar_url) def test_get_profile_empty_displayname(self) -> None: - self.get_success(self.store.set_profile_displayname(self.frank, None)) self.get_success( - self.store.set_profile_avatar_url(self.frank, "http://my.server/me.png") + self.store.set_profile_field( + user_id=self.frank, + field_name=ProfileFields.DISPLAYNAME, + new_value=None, + ) + ) + self.get_success( + self.store.set_profile_field( + user_id=self.frank, + field_name=ProfileFields.AVATAR_URL, + new_value="http://my.server/me.png", + ) ) profile = self.get_success(self.handler.get_profile(self.frank.to_string())) @@ -398,10 +1003,11 @@ class ProfileTestCase(unittest.HomeserverTestCase): def test_set_my_avatar(self) -> None: self.get_success( - self.handler.set_avatar_url( - self.frank, - synapse.types.create_requester(self.frank), - "http://my.server/pic.gif", + self.handler.set_field( + target_user=self.frank, + requester=synapse.types.create_requester(self.frank), + field_name=ProfileFields.AVATAR_URL, + new_value="http://my.server/pic.gif", ) ) @@ -412,10 +1018,11 @@ class ProfileTestCase(unittest.HomeserverTestCase): # Set avatar again self.get_success( - self.handler.set_avatar_url( - self.frank, - synapse.types.create_requester(self.frank), - "http://my.server/me.png", + self.handler.set_field( + target_user=self.frank, + requester=synapse.types.create_requester(self.frank), + field_name=ProfileFields.AVATAR_URL, + new_value="http://my.server/me.png", ) ) @@ -426,10 +1033,11 @@ class ProfileTestCase(unittest.HomeserverTestCase): # Set avatar to an empty string self.get_success( - self.handler.set_avatar_url( - self.frank, - synapse.types.create_requester(self.frank), - "", + self.handler.set_field( + target_user=self.frank, + requester=synapse.types.create_requester(self.frank), + field_name=ProfileFields.AVATAR_URL, + new_value="", ) ) @@ -441,7 +1049,12 @@ class ProfileTestCase(unittest.HomeserverTestCase): def test_set_my_avatar_if_disabled(self) -> None: # Setting displayname for the first time is allowed self.get_success( - self.store.set_profile_avatar_url(self.frank, "http://my.server/me.png") + self.handler.set_field( + target_user=self.frank, + requester=synapse.types.create_requester(self.frank), + field_name=ProfileFields.AVATAR_URL, + new_value="http://my.server/me.png", + ) ) self.assertEqual( @@ -450,14 +1063,17 @@ class ProfileTestCase(unittest.HomeserverTestCase): ) # Set avatar a second time is forbidden - self.get_failure( - self.handler.set_avatar_url( - self.frank, - synapse.types.create_requester(self.frank), - "http://my.server/pic.gif", + f = self.get_failure( + self.handler.set_field( + target_user=self.frank, + requester=synapse.types.create_requester(self.frank), + field_name=ProfileFields.AVATAR_URL, + new_value="http://my.server/pic.gif", ), SynapseError, ) + self.assertEqual(f.value.code, 403) + self.assertEqual(f.value.errcode, Codes.FORBIDDEN) def test_avatar_constraints_no_config(self) -> None: """Tests that the method to check an avatar against configured constraints skips diff --git a/tests/handlers/test_register.py b/tests/handlers/test_register.py index 0db7f30b1f..182ff7a8fc 100644 --- a/tests/handlers/test_register.py +++ b/tests/handlers/test_register.py @@ -25,7 +25,7 @@ from unittest.mock import AsyncMock, Mock from twisted.internet.testing import MemoryReactor from synapse.api.auth.internal import InternalAuth -from synapse.api.constants import UserTypes +from synapse.api.constants import ProfileFields, UserTypes from synapse.api.errors import ( CodeMessageException, Codes, @@ -824,8 +824,12 @@ class RegistrationTestCase(unittest.HomeserverTestCase): if displayname is not None: # logger.info("setting user display name: %s -> %s", user_id, displayname) - await self.hs.get_profile_handler().set_displayname( - user, requester, displayname, by_admin=True + await self.hs.get_profile_handler().set_field( + target_user=user, + requester=requester, + field_name=ProfileFields.DISPLAYNAME, + new_value=displayname, + by_admin=True, ) return user_id, token diff --git a/tests/handlers/test_room.py b/tests/handlers/test_room.py index df95490d3b..2816ae9424 100644 --- a/tests/handlers/test_room.py +++ b/tests/handlers/test_room.py @@ -1,6 +1,9 @@ +from unittest.mock import patch + import synapse from synapse.api.constants import EventTypes, RoomEncryptionAlgorithms from synapse.rest.client import login, room +from synapse.types import create_requester from tests import unittest from tests.unittest import override_config @@ -106,3 +109,150 @@ class EncryptedByDefaultTestCase(unittest.HomeserverTestCase): tok=user_token, expect_code=404, ) + + @override_config({"encryption_enabled_by_default_for_room_type": "all"}) + def test_user_supplied_encryption_event_is_not_overwritten(self) -> None: + """Tests that an m.room.encryption event supplied by the user in the + initial state takes precedence over the one that would otherwise be forced + by encryption_enabled_by_default_for_room_type, rather than a duplicate + default event being sent. + """ + # Create a user + user = self.register_user("user", "pass") + user_token = self.login(user, "pass") + + # Create a room, supplying our own encryption event with a non-default + # algorithm in the initial state. + custom_content = {"algorithm": "some.custom.invalid.algorithm"} + room_id = self.helper.create_room_as( + user, + is_public=False, + tok=user_token, + extra_content={ + "initial_state": [ + { + "type": EventTypes.RoomEncryption, + "state_key": "", + "content": custom_content, + } + ] + }, + ) + + # Check that the room's encryption event is the one we supplied, not the + # default that the config option would otherwise force. + event_content = self.helper.get_state( + room_id=room_id, + event_type=EventTypes.RoomEncryption, + tok=user_token, + ) + self.assertEqual(event_content, custom_content) + + @override_config({"encryption_enabled_by_default_for_room_type": "all"}) + def test_empty_encryption_event_does_not_bypass_forced_encryption(self) -> None: + """Tests that a user cannot bypass encryption_enabled_by_default_for_room_type + by supplying an empty m.room.encryption event in the initial state. Since + such an event is not valid (it lacks the required `algorithm` key), the + forced default must still be applied. + """ + # Create a user + user = self.register_user("user", "pass") + user_token = self.login(user, "pass") + + # Create a room, supplying an empty encryption event in the initial state. + room_id = self.helper.create_room_as( + user, + is_public=False, + tok=user_token, + extra_content={ + "initial_state": [ + { + "type": EventTypes.RoomEncryption, + "state_key": "", + "content": {}, + } + ] + }, + ) + + # Check that the forced default encryption was still applied on top of the + # empty event, rather than the bypass succeeding. + event_content = self.helper.get_state( + room_id=room_id, + event_type=EventTypes.RoomEncryption, + tok=user_token, + ) + self.assertEqual(event_content, {"algorithm": RoomEncryptionAlgorithms.DEFAULT}) + + @override_config({"encryption_enabled_by_default_for_room_type": "all"}) + def test_non_string_algorithm_does_not_bypass_forced_encryption(self) -> None: + """Tests that a user cannot bypass encryption_enabled_by_default_for_room_type + by supplying an m.room.encryption event whose `algorithm` is not a string. + The forced default must still be applied. + """ + # Create a user + user = self.register_user("user", "pass") + user_token = self.login(user, "pass") + + # Create a room, supplying an encryption event with a malformed + # (non-string) algorithm in the initial state. + room_id = self.helper.create_room_as( + user, + is_public=False, + tok=user_token, + extra_content={ + "initial_state": [ + { + "type": EventTypes.RoomEncryption, + "state_key": "", + "content": {"algorithm": 42}, + } + ] + }, + ) + + # Check that the forced default encryption was still applied. + event_content = self.helper.get_state( + room_id=room_id, + event_type=EventTypes.RoomEncryption, + tok=user_token, + ) + self.assertEqual(event_content, {"algorithm": RoomEncryptionAlgorithms.DEFAULT}) + + +class RoomIDCollisionTestCase(unittest.HomeserverTestCase): + servlets = [ + login.register_servlets, + synapse.rest.admin.register_servlets_for_client_rest_resource, + room.register_servlets, + ] + + def test_colliding_v12_room_ids_are_retried(self) -> None: + """In v12+ rooms the room ID is the reference hash of the + create event, so two rooms whose create events have identical content + collide on the same room ID. This happens when the same user creates + several rooms at once (e.g. concurrent /createRoom requests with the + same config within the same millisecond). + + Regression test: the collision must be retried transparently and both + rooms created with distinct IDs. + """ + handler = self.hs.get_room_creation_handler() + user_id = self.register_user("alice", "pass") + requester = create_requester(user_id) + + # Freeze the clock so both create events carry the same + # `origin_server_ts`; with identical config this forces the two room + # IDs to hash to the same value, reproducing the collision. + with patch.object(self.hs.get_clock(), "time_msec", return_value=1234567890000): + room_id1, _, _ = self.get_success( + handler.create_room(requester, {"room_version": "12"}, ratelimit=False) + ) + room_id2, _, _ = self.get_success( + handler.create_room(requester, {"room_version": "12"}, ratelimit=False) + ) + + self.assertNotEqual(room_id1, room_id2) + # v12 room IDs are content hashes with no domain component. + self.assertNotIn(":", room_id1) + self.assertNotIn(":", room_id2) diff --git a/tests/handlers/test_room_member.py b/tests/handlers/test_room_member.py index 0a7475856a..a4a57b64d9 100644 --- a/tests/handlers/test_room_member.py +++ b/tests/handlers/test_room_member.py @@ -169,6 +169,7 @@ class TestJoinsLimitedByPerRoomRateLimiter(FederatingHomeserverTestCase): auth_chain=[create_event], partial_state=False, servers_in_room=frozenset(), + state_dag=None, ) ) diff --git a/tests/handlers/test_room_policy.py b/tests/handlers/test_room_policy.py index c67ea9b0e0..6d912fd79d 100644 --- a/tests/handlers/test_room_policy.py +++ b/tests/handlers/test_room_policy.py @@ -12,17 +12,22 @@ # . # # +from http import HTTPStatus from unittest import mock import signedjson +from parameterized import parameterized from signedjson.key import encode_verify_key_base64, get_verify_key +from twisted.internet import defer from twisted.internet.testing import MemoryReactor +from twisted.web.client import Agent from synapse.api.constants import EventTypes from synapse.api.errors import HttpResponseException, SynapseError from synapse.crypto.event_signing import compute_event_signature from synapse.events import EventBase +from synapse.federation.transport.client import TransportLayerClient from synapse.handlers.room_policy import POLICY_SERVER_KEY_ID from synapse.rest import admin from synapse.rest.client import filter, login, room, sync @@ -31,7 +36,7 @@ from synapse.types import JsonDict, UserID from synapse.util.clock import Clock from tests import unittest -from tests.test_utils import event_injection +from tests.test_utils import FakeResponse, event_injection from tests.test_utils.event_builders import make_test_event @@ -546,3 +551,92 @@ class RoomPolicyTestCase(unittest.FederatingHomeserverTestCase): if ev["event_id"] == event_id: return ev return None + + def _mock_policy_server_response_with_http_error( + self, + status: HTTPStatus, + error_body: JsonDict, + ) -> None: + """ + Make the policy server reply to its `/sign` endpoint with an error. + + Args: + status: the HTTP status to return + error_body: the JSON error body to return + """ + + def request( + method: bytes, + uri: bytes, + headers: object = None, + bodyProducer: object = None, + ) -> "defer.Deferred": + # For our test, we don't expect any other outbound request + assert b"/_matrix/policy/v1/sign" in uri, ( + f"unexpected outbound request to {uri!r}" + ) + return defer.succeed( + FakeResponse.json( + code=status, + payload=error_body, + ) + ) + + fake_agent = mock.create_autospec(Agent, spec_set=True) + fake_agent.request.side_effect = request + self.handler._federation_client.transport_layer = TransportLayerClient(self.hs) + self.hs.get_federation_http_client().agent = fake_agent + + @parameterized.expand( + ( + ( + HTTPStatus.IM_A_TEAPOT, + {"errcode": "M_FORBIDDEN", "error": "No coffee here"}, + HTTPStatus.IM_A_TEAPOT, + {"errcode": "M_FORBIDDEN", "error": "No coffee here"}, + ), + # This case is https://github.com/element-hq/synapse/security/advisories/GHSA-95fh-hv8c-chvq + # The error is rewritten for safety. + ( + HTTPStatus.UNAUTHORIZED, + {"errcode": "M_UNKNOWN_TOKEN", "error": "unknown token"}, + HTTPStatus.BAD_REQUEST, + { + "errcode": "M_UNKNOWN", + "error": "unknown token", + }, + ), + ) + ) + def test_policy_server_error_bubbling_to_client( + self, + policy_server_error_status: HTTPStatus, + policy_server_error_body: JsonDict, + expected_client_facing_error_status: HTTPStatus, + expected_client_facing_error_body: JsonDict, + ) -> None: + """ + Tests how errors from the policy server are forwarded back to clients. + + Regression test for https://github.com/element-hq/synapse/security/advisories/GHSA-95fh-hv8c-chvq + """ + + verify_key_str = encode_verify_key_base64(get_verify_key(self.signing_key)) + self._add_policy_server_to_room(public_key=verify_key_str) + + # Mock the policy server (at the HTTP level) to return + # the configured error + self._mock_policy_server_response_with_http_error( + policy_server_error_status, + policy_server_error_body, + ) + + response_body = self.helper.send_event( + self.room_id, + "m.room.message", + {"body": "honk", "msgtype": "m.text"}, + tok=self.creator_token, + expect_code=expected_client_facing_error_status, + ) + + self.assertEqual(response_body, expected_client_facing_error_body) diff --git a/tests/handlers/test_sync.py b/tests/handlers/test_sync.py index d2b2523321..e100943413 100644 --- a/tests/handlers/test_sync.py +++ b/tests/handlers/test_sync.py @@ -18,7 +18,7 @@ # # from http import HTTPStatus -from typing import Collection, ContextManager +from typing import Collection, ContextManager, cast from unittest.mock import AsyncMock, Mock, patch from parameterized import parameterized, parameterized_class @@ -26,13 +26,14 @@ from parameterized import parameterized, parameterized_class from twisted.internet import defer from twisted.internet.testing import MemoryReactor -from synapse.api.constants import AccountDataTypes, EventTypes, JoinRules +from synapse.api.constants import AccountDataTypes, EventTypes, JoinRules, ProfileFields from synapse.api.errors import Codes, ResourceLimitError from synapse.api.filtering import FilterCollection, Filtering from synapse.api.room_versions import RoomVersion, RoomVersions from synapse.events import EventBase from synapse.events.snapshot import EventContext from synapse.handlers.sync import ( + LAZY_LOADED_PROFILE_FIELDS_CACHE_MAX_AGE, SyncConfig, SyncRequestKey, SyncResult, @@ -43,6 +44,7 @@ from synapse.rest.client import knock, login, room from synapse.server import HomeServer from synapse.types import ( JsonDict, + JsonValue, MultiWriterStreamToken, RoomStreamToken, StreamKeyType, @@ -55,6 +57,8 @@ from synapse.util.duration import Duration import tests.unittest import tests.utils from tests.test_utils.event_builders import make_test_pdu_event +from tests.test_utils.event_injection import inject_member_event +from tests.unittest import override_config _request_key = 0 @@ -1152,6 +1156,1642 @@ def generate_sync_config( ) +class SyncProfileUpdatesTestCase(tests.unittest.HomeserverTestCase): + """Tests Sync Handler for profile updates.""" + + servlets = [ + admin.register_servlets, + login.register_servlets, + room.register_servlets, + ] + + def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None: + super().prepare(reactor, clock, hs) + self.sync_handler = self.hs.get_sync_handler() + self.profile_handler = self.hs.get_profile_handler() + self.store = self.hs.get_datastores().main + self.user = self.register_user("user", "password") + self.tok = self.login("user", "password") + self.other_user = self.register_user("other_user", "password") + self.other_tok = self.login("other_user", "password") + self.joined_room = self.helper.create_room_as(self.user, tok=self.tok) + self.get_success( + self.store.set_profile_field( + user_id=UserID.from_string(self.user), + field_name="m.status", + new_value={"text": "Swimming in the Great Lakes!", "emoji": "🏊"}, + ) + ) + self.helper.join( + room=self.joined_room, user=self.other_user, tok=self.other_tok + ) + + def test_initial_sync_no_profile_updates_if_not_enabled(self) -> None: + """Test that without `include_profile_updates_in_sync` enabled the initial sync + response does not contain any profile updates.""" + self.get_success( + self.profile_handler.set_field( + target_user=UserID.from_string(self.other_user), + requester=create_requester(self.other_user), + field_name="m.status", + new_value={"text": "On holiday", "emoji": "🏖"}, + ) + ) + + requester = create_requester(self.user) + initial_result = self.get_success( + self.sync_handler.wait_for_sync_for_user( + requester, + sync_config=generate_sync_config( + self.user, + ), + request_key=generate_request_key(), + ) + ) + self.assertEqual(initial_result.profile_updates, {}) + + @override_config({"include_profile_updates_in_sync": True}) + def test_initial_sync_no_profile_updates_if_not_filtered_for(self) -> None: + """Test that with `include_profile_updates_in_sync` enabled the initial sync + response does not contain any profile updates, if fields are not filtered for.""" + self.get_success( + self.profile_handler.set_field( + target_user=UserID.from_string(self.other_user), + requester=create_requester(self.other_user), + field_name="m.status", + new_value={"text": "On holiday", "emoji": "🏖"}, + ) + ) + + requester = create_requester(self.user) + initial_result = self.get_success( + self.sync_handler.wait_for_sync_for_user( + requester, + sync_config=generate_sync_config( + user_id=self.user, + ), + request_key=generate_request_key(), + ) + ) + self.assertEqual( + initial_result.profile_updates, + {}, + ) + + @override_config({"include_profile_updates_in_sync": True}) + def test_initial_sync_responds_with_tracked_profile_updates(self) -> None: + """Test that with `include_profile_updates_in_sync` enabled the initial sync + response does contain profile updates for users who share rooms, for the fields + the client requests. This response should include our syncing user.""" + self.get_success( + self.profile_handler.set_field( + target_user=UserID.from_string(self.other_user), + requester=create_requester(self.other_user), + field_name="m.status", + new_value={"text": "On holiday", "emoji": "🏖"}, + ) + ) + # Also set a field the client doesn't want + self.get_success( + self.profile_handler.set_field( + target_user=UserID.from_string(self.other_user), + requester=create_requester(self.other_user), + field_name="displayname", + new_value="New displayname", + ) + ) + + requester = create_requester(self.user) + initial_result = self.get_success( + self.sync_handler.wait_for_sync_for_user( + requester, + sync_config=generate_sync_config( + user_id=self.user, + filter_collection=FilterCollection( + hs=self.hs, + filter_json={ + "org.matrix.msc4429.profile_fields": {"ids": ["m.status"]} + }, + ), + ), + request_key=generate_request_key(), + ) + ) + assert initial_result.profile_updates[self.user] is not None + assert initial_result.profile_updates["@other_user:test"] is not None + self.assertEqual( + initial_result.profile_updates["@other_user:test"]["m.status"], + {"text": "On holiday", "emoji": "🏖"}, + ) + self.assertFalse( + "displayname" in initial_result.profile_updates["@other_user:test"].keys(), + ) + self.assertCountEqual( + initial_result.profile_updates.keys(), + [ + self.user, + "@other_user:test", + ], + ) + + @parameterized.expand( + [ + True, + False, + ] + ) + @override_config({"include_profile_updates_in_sync": True}) + def test_initial_sync_does_not_include_untracked_users_profile_updates( + self, is_lazy: bool + ) -> None: + """Test that with `include_profile_updates_in_sync` enabled the initial sync + response does not contain profile updates for users who do not share rooms.""" + third_user = self.register_user("third_user", "password") + self.get_success( + self.profile_handler.set_field( + target_user=UserID.from_string(third_user), + requester=create_requester(third_user), + field_name="m.status", + new_value={"text": "On holiday", "emoji": "🏖"}, + ) + ) + + requester = create_requester(self.user) + filter_json: dict[str, dict] = { + "org.matrix.msc4429.profile_fields": { + "ids": ["m.status", "displayname", "avatar_url"] + } + } + if is_lazy: + filter_json["room"] = { + "state": { + "lazy_load_members": True, + }, + } + initial_result = self.get_success( + self.sync_handler.wait_for_sync_for_user( + requester, + sync_config=generate_sync_config( + user_id=self.user, + filter_collection=FilterCollection( + hs=self.hs, + filter_json=filter_json, + ), + ), + request_key=generate_request_key(), + ) + ) + self.assertIsNone(initial_result.profile_updates.get(third_user)) + + @override_config({"include_profile_updates_in_sync": True}) + def test_initial_sync_lazy_loading_responds_with_only_profiles_with_events( + self, + ) -> None: + """Test that with `include_profile_updates_in_sync` enabled the initial sync + lazy loading response does contain profile updates for events in the timeline. + + This test ensures lazy loading sync only returns profiles that we also have + events for in the sync response. The second room in this test has the most + recent events from "third_user" and thus we don't get the profile of + "other_user" down the line, who is in the the same rooms as the syncer, + but not in the second room. + """ + third_user = self.register_user("third_user", "password") + third_tok = self.login("third_user", "password") + self.helper.join( + room=self.joined_room, + user=third_user, + tok=third_tok, + ) + + requester = create_requester(self.user) + + self.get_success( + self.profile_handler.set_field( + target_user=UserID.from_string(self.other_user), + requester=create_requester(self.other_user), + field_name="m.status", + new_value={"text": "On holiday", "emoji": "🏖"}, + ) + ) + # Check that lazy-loading filters out profile updates as well on initial sync. + self.get_success( + self.profile_handler.set_field( + target_user=UserID.from_string(third_user), + requester=create_requester(third_user), + field_name="m.status", + new_value={"text": "On fire", "emoji": "🔥"}, + ) + ) + self.helper.send_messages( + room_id=self.joined_room, num_events=1, tok=self.other_tok + ) + self.helper.send_messages( + room_id=self.joined_room, num_events=10, tok=third_tok + ) + initial_result = self.get_success( + self.sync_handler.wait_for_sync_for_user( + requester, + sync_config=generate_sync_config( + user_id=self.user, + filter_collection=FilterCollection( + hs=self.hs, + filter_json={ + "org.matrix.msc4429.profile_fields": { + "ids": ["m.status", "displayname", "avatar_url"] + }, + "room": { + "state": { + "lazy_load_members": True, + }, + }, + }, + ), + ), + request_key=generate_request_key(), + ) + ) + # Only third_user is returned, as lazy loading filters out the events from + # the other users + self.assertCountEqual( + initial_result.profile_updates.keys(), + [ + "@third_user:test", + ], + ) + + @override_config({"include_profile_updates_in_sync": True}) + def test_incremental_sync_sends_down_profile_update_diffs( + self, + ) -> None: + """Test that with `include_profile_updates_in_sync` enabled the incremental + sync response does contain profile update diffs.""" + requester = create_requester(self.user) + initial_result = self.get_success( + self.sync_handler.wait_for_sync_for_user( + requester, + sync_config=generate_sync_config( + user_id=self.user, + filter_collection=FilterCollection( + hs=self.hs, + filter_json={ + "org.matrix.msc4429.profile_fields": { + "ids": ["m.status", "displayname", "avatar_url"] + } + }, + ), + ), + request_key=generate_request_key(), + ) + ) + self.get_success( + self.profile_handler.set_field( + target_user=UserID.from_string(self.other_user), + requester=create_requester(self.other_user), + field_name="m.status", + new_value={"text": "On holiday", "emoji": "🏖"}, + ) + ) + # Set a field the client didn't ask for + self.get_success( + self.profile_handler.set_field( + target_user=UserID.from_string(self.other_user), + requester=create_requester(self.other_user), + field_name="uninterestingfield", + new_value="Content", + ) + ) + incremental_result = self.get_success( + self.sync_handler.wait_for_sync_for_user( + requester, + since_token=initial_result.next_batch, + sync_config=generate_sync_config( + user_id=self.user, + filter_collection=FilterCollection( + hs=self.hs, + filter_json={ + "org.matrix.msc4429.profile_fields": { + "ids": ["m.status", "displayname", "avatar_url"] + } + }, + ), + ), + request_key=generate_request_key(), + ) + ) + assert incremental_result.profile_updates["@other_user:test"] is not None + self.assertEqual( + incremental_result.profile_updates["@other_user:test"]["m.status"], + {"text": "On holiday", "emoji": "🏖"}, + ) + # We only send diffs in incremental sync for profile field updates + self.assertFalse( + "displayname" + in incremental_result.profile_updates["@other_user:test"].keys(), + ) + # The client didn't ask for this field + self.assertFalse( + "uninterestingfield" + in incremental_result.profile_updates["@other_user:test"].keys(), + ) + + @override_config({"include_profile_updates_in_sync": True}) + def test_incremental_sync_does_not_filter_profile_updates_when_lazy_loading( + self, + ) -> None: + """Test that with `include_profile_updates_in_sync` enabled the incremental + sync lazy loading response does contain profile updates even if the user would + be filtered out by lazy loading. + """ + third_user = self.register_user("third_user", "password") + third_tok = self.login("third_user", "password") + self.helper.join( + room=self.joined_room, + user=third_user, + tok=third_tok, + ) + + requester = create_requester(self.user) + initial_result = self.get_success( + self.sync_handler.wait_for_sync_for_user( + requester, + sync_config=generate_sync_config( + user_id=self.user, + filter_collection=FilterCollection( + hs=self.hs, + filter_json={ + "org.matrix.msc4429.profile_fields": { + "ids": ["m.status", "displayname"] + } + }, + ), + ), + request_key=generate_request_key(), + ) + ) + self.get_success( + self.profile_handler.set_field( + target_user=UserID.from_string(self.other_user), + requester=create_requester(self.other_user), + field_name="m.status", + new_value={"text": "On holiday", "emoji": "🏖"}, + ) + ) + self.get_success( + self.profile_handler.set_field( + target_user=UserID.from_string(third_user), + requester=create_requester(third_user), + field_name="m.status", + new_value={"text": "On fire", "emoji": "🔥"}, + ) + ) + self.helper.send_messages( + room_id=self.joined_room, num_events=1, tok=self.other_tok + ) + self.helper.send_messages( + room_id=self.joined_room, num_events=10, tok=third_tok + ) + self.get_success( + self.profile_handler.set_field( + target_user=UserID.from_string(third_user), + requester=create_requester(third_user), + field_name="uninterestingfield", + new_value="Content", + ) + ) + # Join a federated user to the room + self.get_success( + inject_member_event( + self.hs, + self.joined_room, + "@federateduser:federatedhs", + "join", + ) + ) + incremental_result = self.get_success( + self.sync_handler.wait_for_sync_for_user( + requester, + since_token=initial_result.next_batch, + sync_config=generate_sync_config( + user_id=self.user, + filter_collection=FilterCollection( + hs=self.hs, + filter_json={ + "org.matrix.msc4429.profile_fields": { + "ids": ["m.status", "displayname"] + }, + "room": { + "state": { + "lazy_load_members": True, + }, + }, + }, + ), + ), + request_key=generate_request_key(), + ) + ) + + # Ensure our federated user is filtered out, even though they have an + # event in the joined room timeline + self.assertFalse( + "@federateduser:federatedhs" in incremental_result.profile_updates.keys() + ) + + # Lazy loading only filters initial sync profile updates. Incremental syncs + # should include all tracked profile updates for the syncing user. + self.assertCountEqual( + incremental_result.profile_updates.keys(), + [ + "@other_user:test", + "@third_user:test", + ], + ) + assert incremental_result.profile_updates["@other_user:test"] is not None + + # This is a field update, so should be here + self.assertEqual( + incremental_result.profile_updates["@other_user:test"]["m.status"], + {"text": "On holiday", "emoji": "🏖"}, + ) + + # We don't have events for this user in this response, so their full profile + # is not included + self.assertFalse( + "displayname" + in incremental_result.profile_updates["@other_user:test"].keys(), + ) + assert incremental_result.profile_updates["@third_user:test"] is not None + + # This user has events in the timeline, thus the fields the client asked for + # are included + self.assertEqual( + incremental_result.profile_updates["@third_user:test"]["m.status"], + {"text": "On fire", "emoji": "🔥"}, + ) + self.assertFalse( + "uninterestingfield" + in incremental_result.profile_updates["@third_user:test"].keys(), + ) + self.assertEqual( + incremental_result.profile_updates["@third_user:test"]["displayname"], + "third_user", + ) + + @parameterized.expand( + [ + [True, True], + [False, False], + [True, False], + [False, True], + ] + ) + @override_config({"include_profile_updates_in_sync": True}) + def test_sync_filters_out_profile_updates_from_federated_users( + self, + is_initial: bool, + is_lazy: bool, + ) -> None: + """Test that with `include_profile_updates_in_sync` enabled any sync response + doesn't contain federated users even if there are timeline events from them. + """ + # Join a federated user to the room, causing a membership event into + # the joined rooms sync response + self.get_success( + inject_member_event( + self.hs, + self.joined_room, + "@federateduser1:federatedhs", + "join", + ) + ) + requester = create_requester(self.user) + filter_json: dict[str, dict] = { + "org.matrix.msc4429.profile_fields": { + "ids": ["m.status", "displayname", "avatar_url"] + }, + } + if is_lazy: + filter_json["room"] = { + "state": { + "lazy_load_members": True, + }, + } + initial_result = self.get_success( + self.sync_handler.wait_for_sync_for_user( + requester, + sync_config=generate_sync_config( + user_id=self.user, + filter_collection=FilterCollection( + hs=self.hs, + filter_json=filter_json, + ), + ), + request_key=generate_request_key(), + ) + ) + # Ensure our federated user is filtered out, even though they have an + # event in the joined room timeline + self.assertFalse( + "@federateduser1:federatedhs" in initial_result.profile_updates.keys() + ) + if not is_initial: + # Join another federated user to the room, causing a membership event into + # the joined rooms sync response + self.get_success( + inject_member_event( + self.hs, + self.joined_room, + "@federateduser2:federatedhs", + "join", + ) + ) + incremental_result = self.get_success( + self.sync_handler.wait_for_sync_for_user( + requester, + since_token=initial_result.next_batch, + sync_config=generate_sync_config( + user_id=self.user, + filter_collection=FilterCollection( + hs=self.hs, + filter_json=filter_json, + ), + ), + request_key=generate_request_key(), + ) + ) + + # Ensure our federated user is filtered out, even though they have an + # event in the joined room timeline + self.assertFalse( + "@federateduser2:federatedhs" + in incremental_result.profile_updates.keys() + ) + + @parameterized.expand( + [ + [True, True], + [False, False], + [True, False], + [False, True], + ] + ) + @override_config({"include_profile_updates_in_sync": True}) + def test_sync_response_always_includes_the_user_themselves( + self, + is_initial: bool, + is_lazy: bool, + ) -> None: + """Test that with `include_profile_updates_in_sync` enabled any sync response + always contains the users own updates. + + This test is made with a user that is not in any rooms, to prove our code + to collect interested users from the profile updates always collect the user. + """ + third_user = self.register_user("third_user", "password") + requester = create_requester(third_user) + filter_json: dict[str, dict] = { + "org.matrix.msc4429.profile_fields": {"ids": ["field"]}, + } + if is_lazy: + filter_json["room"] = { + "state": { + "lazy_load_members": True, + }, + } + if is_initial: + self.get_success( + self.profile_handler.set_field( + target_user=UserID.from_string(third_user), + requester=create_requester(third_user), + field_name="field", + new_value="Content", + ) + ) + initial_result = self.get_success( + self.sync_handler.wait_for_sync_for_user( + requester, + sync_config=generate_sync_config( + user_id=third_user, + filter_collection=FilterCollection( + hs=self.hs, + filter_json=filter_json, + ), + ), + request_key=generate_request_key(), + ) + ) + if is_initial: + assert initial_result.profile_updates["@third_user:test"] is not None + self.assertEqual( + initial_result.profile_updates["@third_user:test"]["field"], + "Content", + ) + else: + self.get_success( + self.profile_handler.set_field( + target_user=UserID.from_string(third_user), + requester=create_requester(third_user), + field_name="field", + new_value="Content", + ) + ) + incremental_result = self.get_success( + self.sync_handler.wait_for_sync_for_user( + requester, + since_token=initial_result.next_batch, + sync_config=generate_sync_config( + user_id=third_user, + filter_collection=FilterCollection( + hs=self.hs, + filter_json=filter_json, + ), + ), + request_key=generate_request_key(), + ) + ) + assert incremental_result.profile_updates["@third_user:test"] is not None + self.assertEqual( + incremental_result.profile_updates["@third_user:test"]["field"], + "Content", + ) + + @parameterized.expand( + [ + [True, True], + [False, False], + [True, False], + [False, True], + ] + ) + @override_config({"include_profile_updates_in_sync": True}) + def test_sync_profile_updates_works_correctly_with_falsey_values( + self, + is_initial: bool, + is_lazy: bool, + ) -> None: + """Test that with `include_profile_updates_in_sync` enabled a sync response + correctly includes falsey profile field values. + """ + requester = create_requester(self.user) + filter_json: dict[str, dict] = { + "org.matrix.msc4429.profile_fields": {"ids": ["falseyvaluefield"]}, + } + if is_lazy: + filter_json["room"] = { + "state": { + "lazy_load_members": True, + }, + } + for value in [False, 0, "", [], {}, None]: + if is_initial: + self.get_success( + self.profile_handler.set_field( + target_user=UserID.from_string(self.other_user), + requester=create_requester(self.other_user), + field_name="falseyvaluefield", + new_value=cast(JsonValue | dict[str, JsonValue], value), + ) + ) + initial_result = self.get_success( + self.sync_handler.wait_for_sync_for_user( + requester, + sync_config=generate_sync_config( + user_id=self.user, + filter_collection=FilterCollection( + hs=self.hs, + filter_json=filter_json, + ), + ), + request_key=generate_request_key(), + ) + ) + if is_initial: + assert initial_result.profile_updates["@other_user:test"] is not None + self.assertEqual( + initial_result.profile_updates["@other_user:test"][ + "falseyvaluefield" + ], + value, + ) + else: + self.get_success( + self.profile_handler.set_field( + target_user=UserID.from_string(self.other_user), + requester=create_requester(self.other_user), + field_name="falseyvaluefield", + new_value=cast(JsonValue | dict[str, JsonValue], value), + ) + ) + incremental_result = self.get_success( + self.sync_handler.wait_for_sync_for_user( + requester, + since_token=initial_result.next_batch, + sync_config=generate_sync_config( + user_id=self.user, + filter_collection=FilterCollection( + hs=self.hs, + filter_json=filter_json, + ), + ), + request_key=generate_request_key(), + ) + ) + assert ( + incremental_result.profile_updates["@other_user:test"] is not None + ) + self.assertEqual( + incremental_result.profile_updates["@other_user:test"][ + "falseyvaluefield" + ], + value, + ) + + @override_config({"include_profile_updates_in_sync": True}) + def test_incremental_sync_lazy_loading_cache_filters_recently_sent_profiles_and_fields( + self, + ) -> None: + """Test that with `include_profile_updates_in_sync` enabled the incremental + sync lazy loading response filters out unchanged profiles or fields we have + recently sent to the client. + """ + requester = create_requester(self.user) + self.get_success( + self.profile_handler.set_field( + target_user=UserID.from_string(self.other_user), + requester=create_requester(self.other_user), + field_name="sooninterestingfield", + new_value="Content", + ) + ) + initial_result = self.get_success( + self.sync_handler.wait_for_sync_for_user( + requester, + sync_config=generate_sync_config( + user_id=self.user, + filter_collection=FilterCollection( + hs=self.hs, + filter_json={ + "org.matrix.msc4429.profile_fields": { + "ids": ["m.status", "displayname", "avatar_url"] + }, + }, + ), + ), + request_key=generate_request_key(), + ) + ) + self.helper.send_messages( + room_id=self.joined_room, + num_events=1, + tok=self.other_tok, + ) + incremental_result = self.get_success( + self.sync_handler.wait_for_sync_for_user( + requester, + since_token=initial_result.next_batch, + sync_config=generate_sync_config( + user_id=self.user, + filter_collection=FilterCollection( + hs=self.hs, + filter_json={ + "org.matrix.msc4429.profile_fields": { + "ids": ["m.status", "displayname", "avatar_url"] + }, + "room": { + "state": { + "lazy_load_members": True, + }, + }, + }, + ), + ), + request_key=generate_request_key(), + ) + ) + # Lazy loading incremental sync should include profiles from events + self.assertCountEqual( + incremental_result.profile_updates.keys(), + [ + "@other_user:test", + ], + ) + assert incremental_result.profile_updates["@other_user:test"] is not None + self.assertEqual( + set(incremental_result.profile_updates["@other_user:test"].keys()), + {"avatar_url", "displayname"}, + ) + + # If we have more events from the other_user, and do another lazy sync, + # we don't expect the full profile to be sent again due to our cache. + self.helper.send_messages( + room_id=self.joined_room, num_events=1, tok=self.other_tok + ) + incremental_result = self.get_success( + self.sync_handler.wait_for_sync_for_user( + requester, + since_token=incremental_result.next_batch, + sync_config=generate_sync_config( + user_id=self.user, + filter_collection=FilterCollection( + hs=self.hs, + filter_json={ + "org.matrix.msc4429.profile_fields": { + "ids": ["m.status", "displayname", "avatar_url"] + }, + "room": { + "state": { + "lazy_load_members": True, + }, + }, + }, + ), + ), + request_key=generate_request_key(), + ) + ) + self.assertCountEqual( + incremental_result.profile_updates.keys(), + [], + ) + # However, if we again add an event, we do expect any fields the client didn't + # previously ask for to be there. + self.helper.send_messages( + room_id=self.joined_room, num_events=1, tok=self.other_tok + ) + incremental_result = self.get_success( + self.sync_handler.wait_for_sync_for_user( + requester, + since_token=incremental_result.next_batch, + sync_config=generate_sync_config( + user_id=self.user, + filter_collection=FilterCollection( + hs=self.hs, + filter_json={ + "org.matrix.msc4429.profile_fields": { + "ids": [ + "m.status", + "displayname", + "avatar_url", + "sooninterestingfield", + ] + }, + "room": { + "state": { + "lazy_load_members": True, + }, + }, + }, + ), + ), + request_key=generate_request_key(), + ) + ) + self.assertCountEqual( + incremental_result.profile_updates.keys(), + [ + "@other_user:test", + ], + ) + assert incremental_result.profile_updates["@other_user:test"] is not None + self.assertEqual( + set(incremental_result.profile_updates["@other_user:test"].keys()), + {"sooninterestingfield"}, + ) + + @override_config({"include_profile_updates_in_sync": True}) + def test_incremental_sync_sends_down_null_profile_if_user_no_longer_sharing_rooms( + self, + ) -> None: + """Test that with `include_profile_updates_in_sync` enabled the incremental + sync response includes a 'null' for users who are no longer sharing rooms. + """ + requester = create_requester(self.user) + initial_result = self.get_success( + self.sync_handler.wait_for_sync_for_user( + requester, + sync_config=generate_sync_config( + user_id=self.user, + filter_collection=FilterCollection( + hs=self.hs, + filter_json={ + "org.matrix.msc4429.profile_fields": { + "ids": ["m.status", "displayname", "avatar_url"] + } + }, + ), + ), + request_key=generate_request_key(), + ) + ) + self.helper.leave( + room=self.joined_room, user=self.other_user, tok=self.other_tok + ) + incremental_result = self.get_success( + self.sync_handler.wait_for_sync_for_user( + requester, + since_token=initial_result.next_batch, + sync_config=generate_sync_config( + user_id=self.user, + filter_collection=FilterCollection( + hs=self.hs, + filter_json={ + "org.matrix.msc4429.profile_fields": { + "ids": ["m.status", "displayname", "avatar_url"] + } + }, + ), + ), + request_key=generate_request_key(), + ) + ) + self.assertIsNone( + incremental_result.profile_updates["@other_user:test"], + ) + + @parameterized.expand( + [ + True, + False, + ] + ) + @override_config({"include_profile_updates_in_sync": True}) + def test_incremental_sync_sends_down_deleted_fields(self, is_lazy: bool) -> None: + """ + Tests that, with `include_profile_updates_in_sync` enabled, + an incremental sync returns deleted fields as a `null` value, both for + `displayname` (stored as its own column) and for + generic custom fields (stored as JSON). + """ + # Set up a user with `displayname` and `m.status` profile fields + requester = create_requester(self.user) + other_requester = create_requester(self.other_user) + other_user = UserID.from_string(self.other_user) + filter_json: dict = { + "org.matrix.msc4429.profile_fields": {"ids": ["displayname", "m.status"]}, + } + if is_lazy: + filter_json["room"] = { + "state": { + "lazy_load_members": True, + }, + } + sync_config = generate_sync_config( + user_id=self.user, + filter_collection=FilterCollection( + hs=self.hs, + filter_json=filter_json, + ), + ) + self.get_success( + self.profile_handler.set_field( + target_user=other_user, + requester=other_requester, + field_name=ProfileFields.DISPLAYNAME, + new_value="Bob", + ) + ) + self.get_success( + self.profile_handler.set_field( + target_user=other_user, + requester=other_requester, + field_name="m.status", + new_value={"text": "On holiday"}, + ) + ) + + # Do an initial sync after the point of those fields being set + initial_result = self.get_success( + self.sync_handler.wait_for_sync_for_user( + requester, + sync_config=sync_config, + request_key=generate_request_key(), + ) + ) + self.assertEqual( + initial_result.profile_updates["@other_user:test"], + {"displayname": "Bob", "m.status": {"text": "On holiday"}}, + ) + + # Delete the displayname and the `m.status` profile fields + self.get_success( + self.profile_handler.set_field( + target_user=other_user, + requester=other_requester, + field_name=ProfileFields.DISPLAYNAME, + new_value="", + ) + ) + self.get_success( + self.profile_handler.delete_profile_field( + target_user=other_user, + requester=other_requester, + field_name="m.status", + ) + ) + + # Do an incremental sync. + # Expect the deletion of both fields to be communicated in it. + incremental_result = self.get_success( + self.sync_handler.wait_for_sync_for_user( + requester, + since_token=initial_result.next_batch, + sync_config=sync_config, + request_key=generate_request_key(), + ) + ) + self.assertEqual( + incremental_result.profile_updates, + # We currently represent deleted fields as `null`, even though + # it's ambiguous (TODO MSC change) + {"@other_user:test": {"displayname": None, "m.status": None}}, + ) + + @override_config({"include_profile_updates_in_sync": True}) + def test_incremental_sync_sends_down_all_requested_fields_for_users_who_have_joined( + self, + ) -> None: + """Test that with `include_profile_updates_in_sync` enabled the incremental + sync response includes all the requested fields of a user who has joined a room + with the syncing user. + """ + requester = create_requester(self.user) + initial_result = self.get_success( + self.sync_handler.wait_for_sync_for_user( + requester, + sync_config=generate_sync_config( + user_id=self.user, + filter_collection=FilterCollection( + hs=self.hs, + filter_json={ + "org.matrix.msc4429.profile_fields": { + "ids": ["displayname", "avatar_url"] + }, + }, + ), + ), + request_key=generate_request_key(), + ) + ) + incremental_result = self.get_success( + self.sync_handler.wait_for_sync_for_user( + requester, + since_token=initial_result.next_batch, + sync_config=generate_sync_config( + user_id=self.user, + filter_collection=FilterCollection( + hs=self.hs, + filter_json={ + "org.matrix.msc4429.profile_fields": { + "ids": ["displayname", "avatar_url"] + }, + }, + ), + ), + request_key=generate_request_key(), + ) + ) + + third_user = self.register_user("third_user", "password") + third_tok = self.login("third_user", "password") + self.helper.join( + room=self.joined_room, + user=third_user, + tok=third_tok, + ) + # Set a status field we don't except to see in sync + self.get_success( + self.profile_handler.set_field( + target_user=UserID.from_string(third_user), + requester=create_requester(third_user), + field_name="m.status", + new_value={"text": "On fire", "emoji": "🔥"}, + ) + ) + incremental_result = self.get_success( + self.sync_handler.wait_for_sync_for_user( + requester, + since_token=incremental_result.next_batch, + sync_config=generate_sync_config( + user_id=self.user, + filter_collection=FilterCollection( + hs=self.hs, + filter_json={ + "org.matrix.msc4429.profile_fields": { + "ids": ["displayname", "avatar_url"] + }, + }, + ), + ), + request_key=generate_request_key(), + ) + ) + assert incremental_result.profile_updates["@third_user:test"] is not None + self.assertCountEqual( + incremental_result.profile_updates.keys(), + [third_user], + ) + self.assertEqual( + incremental_result.profile_updates["@third_user:test"]["displayname"], + "third_user", + ) + self.assertIsNone( + incremental_result.profile_updates["@third_user:test"]["avatar_url"], + ) + self.assertFalse( + "m.status" in incremental_result.profile_updates["@third_user:test"].keys(), + ) + + @parameterized.expand( + [ + True, + False, + ] + ) + @override_config({"include_profile_updates_in_sync": True}) + def test_incremental_sync_includes_own_profile_updates(self, is_lazy: bool) -> None: + """Test that with `include_profile_updates_in_sync` enabled the incremental + sync response includes ones own profile updates.""" + requester = create_requester(self.user) + filter_json: dict[str, dict] = { + "org.matrix.msc4429.profile_fields": {"ids": ["m.status", "avatar_url"]} + } + if is_lazy: + filter_json["room"] = { + "state": { + "lazy_load_members": True, + }, + } + initial_result = self.get_success( + self.sync_handler.wait_for_sync_for_user( + requester, + sync_config=generate_sync_config( + user_id=self.user, + filter_collection=FilterCollection( + hs=self.hs, + filter_json=filter_json, + ), + ), + request_key=generate_request_key(), + ) + ) + self.get_success( + self.profile_handler.set_field( + target_user=UserID.from_string(self.user), + requester=requester, + field_name="m.status", + new_value={"text": "On holiday", "emoji": "🏖"}, + ) + ) + incremental_result = self.get_success( + self.sync_handler.wait_for_sync_for_user( + requester, + since_token=initial_result.next_batch, + sync_config=generate_sync_config( + user_id=self.user, + filter_collection=FilterCollection( + hs=self.hs, + filter_json=filter_json, + ), + ), + request_key=generate_request_key(), + ) + ) + assert incremental_result.profile_updates["@user:test"] is not None + self.assertEqual( + incremental_result.profile_updates["@user:test"]["m.status"], + {"text": "On holiday", "emoji": "🏖"}, + ) + # We didn't ask for displayname + self.assertFalse( + "displayname" in incremental_result.profile_updates["@user:test"].keys(), + ) + + @parameterized.expand([[True, False], [True, True], [False, False], [False, True]]) + @override_config({"include_profile_updates_in_sync": True}) + def test_incremental_sync_join_leave_join_leave_includes_user_joining_and_leaving( + self, + eager_sync: bool, + is_lazy: bool, + ) -> None: + """Test that with `include_profile_updates_in_sync` enabled the incremental + sync response correctly handles multiple join / leave / join / leave in a row. + + In the first variant we sync and check after each iteration of join/leave. + In the second variant we only sync at the end of all the join/leaves. + We do both of these as lazy and not-lazy variants. + + This test checks that for a syncing user that is joining and leaving, a member + of the room gets the right profile information down the line. + """ + # Use third_user for this test as other_user is already joined + third_user = self.register_user("third_user", "password") + third_tok = self.login("third_user", "password") + + filter_json: dict[str, dict] = { + "org.matrix.msc4429.profile_fields": {"ids": ["displayname", "avatar_url"]} + } + if is_lazy: + filter_json["room"] = { + "state": { + "lazy_load_members": True, + }, + } + + requester = create_requester(self.user) + initial_result = self.get_success( + self.sync_handler.wait_for_sync_for_user( + requester, + sync_config=generate_sync_config( + user_id=self.user, + filter_collection=FilterCollection( + hs=self.hs, + filter_json=filter_json, + ), + ), + request_key=generate_request_key(), + ) + ) + # Join the room + self.helper.join( + room=self.joined_room, + user=third_user, + tok=third_tok, + ) + next_token = initial_result.next_batch + if eager_sync: + incremental_result = self.get_success( + self.sync_handler.wait_for_sync_for_user( + requester, + since_token=next_token, + sync_config=generate_sync_config( + user_id=self.user, + filter_collection=FilterCollection( + hs=self.hs, + filter_json=filter_json, + ), + ), + request_key=generate_request_key(), + ) + ) + # We expect there to be the users profile + self.assertIsNotNone( + incremental_result.profile_updates["@third_user:test"], + ) + next_token = incremental_result.next_batch + # Leave the room + self.helper.leave( + room=self.joined_room, + user=third_user, + tok=third_tok, + ) + if eager_sync: + # Ensure we don't get caught by the cache + self.reactor.advance((LAZY_LOADED_PROFILE_FIELDS_CACHE_MAX_AGE / 1000) + 1) + incremental_result = self.get_success( + self.sync_handler.wait_for_sync_for_user( + requester, + since_token=next_token, + sync_config=generate_sync_config( + user_id=self.user, + filter_collection=FilterCollection( + hs=self.hs, + filter_json=filter_json, + ), + ), + request_key=generate_request_key(), + ) + ) + # We expect there to be a null profile + self.assertIsNone( + incremental_result.profile_updates["@third_user:test"], + ) + next_token = incremental_result.next_batch + # Join the room + self.helper.join( + room=self.joined_room, + user=third_user, + tok=third_tok, + ) + if eager_sync: + # Ensure we don't get caught by the cache + self.reactor.advance((LAZY_LOADED_PROFILE_FIELDS_CACHE_MAX_AGE / 1000) + 1) + incremental_result = self.get_success( + self.sync_handler.wait_for_sync_for_user( + requester, + since_token=next_token, + sync_config=generate_sync_config( + user_id=self.user, + filter_collection=FilterCollection( + hs=self.hs, + filter_json=filter_json, + ), + ), + request_key=generate_request_key(), + ) + ) + # We expect there to be the users profile + self.assertIsNotNone( + incremental_result.profile_updates["@third_user:test"], + ) + next_token = incremental_result.next_batch + # Leave the room + self.helper.leave( + room=self.joined_room, + user=third_user, + tok=third_tok, + ) + # Ensure we don't get caught by the cache + self.reactor.advance((LAZY_LOADED_PROFILE_FIELDS_CACHE_MAX_AGE / 1000) + 1) + incremental_result = self.get_success( + self.sync_handler.wait_for_sync_for_user( + requester, + since_token=next_token, + sync_config=generate_sync_config( + user_id=self.user, + filter_collection=FilterCollection( + hs=self.hs, + filter_json=filter_json, + ), + ), + request_key=generate_request_key(), + ) + ) + # The end result should be null profile + self.assertIsNone( + incremental_result.profile_updates["@third_user:test"], + ) + + @parameterized.expand( + [ + ["string value", "new string value"], + [True, False], + [None, "not None"], + [[], ["with item"]], + [{}, {"key": "value"}], + [{"foo": "bar"}, {"bar": "foo"}], + [42, 42.24], + ] + ) + @override_config({"include_profile_updates_in_sync": True}) + def test_profile_updates_dont_get_silenced_by_cache( + self, + value: str | bool | list | dict | int | float | None, + new_value: str | bool | list | dict | int | float | None, + ) -> None: + """Test that with `include_profile_updates_in_sync` enabled the incremental + lazy sync response includes all the profile update changes for the user, even + if the profile field has been recently sent and is in our lazy loading cache. + + Parameterize across different types of potential value types that profile + field updates could have to ensure robustness. + """ + requester = create_requester(self.user) + initial_result = self.get_success( + self.sync_handler.wait_for_sync_for_user( + requester, + sync_config=generate_sync_config( + user_id=self.user, + filter_collection=FilterCollection( + hs=self.hs, + filter_json={ + "org.matrix.msc4429.profile_fields": {"ids": ["field"]}, + "room": { + "state": { + "lazy_load_members": True, + }, + }, + }, + ), + ), + request_key=generate_request_key(), + ) + ) + self.assertFalse( + "@other_user:test" in initial_result.profile_updates, + ) + + # Update the field + self.get_success( + self.profile_handler.set_field( + target_user=UserID.from_string(self.other_user), + requester=create_requester(self.other_user), + field_name="field", + new_value=cast(JsonValue | dict[str, JsonValue], value), + ) + ) + # Also send an event + self.helper.send(self.joined_room, "Foo", tok=self.other_tok) + # Sync + incremental_result = self.get_success( + self.sync_handler.wait_for_sync_for_user( + requester, + since_token=initial_result.next_batch, + sync_config=generate_sync_config( + user_id=self.user, + filter_collection=FilterCollection( + hs=self.hs, + filter_json={ + "org.matrix.msc4429.profile_fields": {"ids": ["field"]}, + "room": { + "state": { + "lazy_load_members": True, + }, + }, + }, + ), + ), + request_key=generate_request_key(), + ) + ) + # We should have the field change in our sync response. + # It will also be added to the lazy loading cache, so the same field value + # isn't sent again immediately. + assert incremental_result.profile_updates["@other_user:test"] is not None + self.assertEqual( + incremental_result.profile_updates["@other_user:test"]["field"], + value, + ) + + # Update the field again, busting our cache + self.get_success( + self.profile_handler.set_field( + target_user=UserID.from_string(self.other_user), + requester=create_requester(self.other_user), + field_name="field", + new_value=cast(JsonValue | dict[str, JsonValue], new_value), + ) + ) + # Also send an event + self.helper.send(self.joined_room, "Foo", tok=self.other_tok) + # Sync + incremental_result = self.get_success( + self.sync_handler.wait_for_sync_for_user( + requester, + since_token=incremental_result.next_batch, + sync_config=generate_sync_config( + user_id=self.user, + filter_collection=FilterCollection( + hs=self.hs, + filter_json={ + "org.matrix.msc4429.profile_fields": {"ids": ["field"]}, + "room": { + "state": { + "lazy_load_members": True, + }, + }, + }, + ), + ), + request_key=generate_request_key(), + ) + ) + # Even though the field was added to the lazy loading members cache, + # it should come through as an update, as the field value changed. + assert incremental_result.profile_updates["@other_user:test"] is not None + self.assertEqual( + incremental_result.profile_updates["@other_user:test"]["field"], + new_value, + ) + + @override_config({"include_profile_updates_in_sync": True}) + def test_lazy_loading_cache_and_multiple_updates_to_the_same_field( + self, + ) -> None: + """Test that with `include_profile_updates_in_sync` enabled the incremental + lazy sync response includes an update to a field, even when the value changes + back to a value set and cached previously. + """ + requester = create_requester(self.user) + filter_json = { + "org.matrix.msc4429.profile_fields": {"ids": ["field"]}, + "room": { + "state": { + "lazy_load_members": True, + }, + }, + } + initial_result = self.get_success( + self.sync_handler.wait_for_sync_for_user( + requester, + sync_config=generate_sync_config( + user_id=self.user, + filter_collection=FilterCollection( + hs=self.hs, + filter_json=filter_json, + ), + ), + request_key=generate_request_key(), + ) + ) + self.assertFalse( + "@other_user:test" in initial_result.profile_updates, + ) + + # Update the field + self.get_success( + self.profile_handler.set_field( + target_user=UserID.from_string(self.other_user), + requester=create_requester(self.other_user), + field_name="field", + new_value="value", + ) + ) + # Also send an event + self.helper.send(self.joined_room, "Foo", tok=self.other_tok) + # Sync + incremental_result = self.get_success( + self.sync_handler.wait_for_sync_for_user( + requester, + since_token=initial_result.next_batch, + sync_config=generate_sync_config( + user_id=self.user, + filter_collection=FilterCollection( + hs=self.hs, + filter_json=filter_json, + ), + ), + request_key=generate_request_key(), + ) + ) + # We should have the field change in our sync response. + # It will also be added to the lazy loading cache, so the same field value + # isn't sent again immediately. + assert incremental_result.profile_updates["@other_user:test"] is not None + self.assertEqual( + incremental_result.profile_updates["@other_user:test"]["field"], + "value", + ) + + # Update the field again, busting our cache + self.get_success( + self.profile_handler.set_field( + target_user=UserID.from_string(self.other_user), + requester=create_requester(self.other_user), + field_name="field", + new_value="new value", + ) + ) + # Also send an event + self.helper.send(self.joined_room, "Foo", tok=self.other_tok) + # Sync + incremental_result = self.get_success( + self.sync_handler.wait_for_sync_for_user( + requester, + since_token=incremental_result.next_batch, + sync_config=generate_sync_config( + user_id=self.user, + filter_collection=FilterCollection( + hs=self.hs, + filter_json=filter_json, + ), + ), + request_key=generate_request_key(), + ) + ) + # Even though the field was added to the lazy loading members cache, + # it should come through as an update, as the field value changed. + assert incremental_result.profile_updates["@other_user:test"] is not None + self.assertEqual( + incremental_result.profile_updates["@other_user:test"]["field"], + "new value", + ) + + # Update the field again, but to the previous value + self.get_success( + self.profile_handler.set_field( + target_user=UserID.from_string(self.other_user), + requester=create_requester(self.other_user), + field_name="field", + new_value="value", + ) + ) + # Also send an event + self.helper.send(self.joined_room, "Foo", tok=self.other_tok) + # Sync + incremental_result = self.get_success( + self.sync_handler.wait_for_sync_for_user( + requester, + since_token=incremental_result.next_batch, + sync_config=generate_sync_config( + user_id=self.user, + filter_collection=FilterCollection( + hs=self.hs, + filter_json=filter_json, + ), + ), + request_key=generate_request_key(), + ) + ) + # Even though we've quite recently sent down this value, we should still + # see it again as it is a change + assert incremental_result.profile_updates["@other_user:test"] is not None + self.assertEqual( + incremental_result.profile_updates["@other_user:test"]["field"], + "value", + ) + + class SyncStateAfterTestCase(tests.unittest.HomeserverTestCase): """Tests Sync Handler state behavior when using `use_state_after.""" @@ -1248,6 +2888,7 @@ class SyncStateAfterTestCase(tests.unittest.HomeserverTestCase): end_token=end_stream_token, members_to_fetch=None, timeline_state={}, + joined=True, ) ) self.assertEqual(state[("m.test_event", "")], second_state["event_id"]) @@ -1279,6 +2920,7 @@ class SyncStateAfterTestCase(tests.unittest.HomeserverTestCase): end_token=end_stream_token, members_to_fetch=set(), timeline_state={}, + joined=True, ) ) diff --git a/tests/http/test_appservice_proxy.py b/tests/http/test_appservice_proxy.py new file mode 100644 index 0000000000..de1ba3cae9 --- /dev/null +++ b/tests/http/test_appservice_proxy.py @@ -0,0 +1,47 @@ +# +# This file is licensed under the Affero General Public License (AGPL) version 3. +# +# Copyright (C) 2026 Element Creations Ltd +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# See the GNU Affero General Public License for more details: +# . +# + +from synapse.http.appservice_proxy import has_dot_segments + +from tests import unittest + + +class HasDotSegmentsTestCase(unittest.TestCase): + def test_plain_path_has_no_dot_segments(self) -> None: + self.assertFalse(has_dot_segments(b"/some/path")) + self.assertFalse(has_dot_segments(b"/some/path.txt")) + self.assertFalse(has_dot_segments(b"/some/...path")) + + def test_dot_segment_is_detected(self) -> None: + self.assertTrue(has_dot_segments(b"/some/./path")) + self.assertTrue(has_dot_segments(b"/./some/path")) + self.assertTrue(has_dot_segments(b"/some/path/.")) + + def test_dot_dot_segment_is_detected(self) -> None: + self.assertTrue(has_dot_segments(b"/some/../path")) + self.assertTrue(has_dot_segments(b"/../some/path")) + self.assertTrue(has_dot_segments(b"/some/path/..")) + + def test_percent_encoded_dot_segments_are_detected(self) -> None: + self.assertTrue(has_dot_segments(b"/some/%2e%2e/path")) + self.assertTrue(has_dot_segments(b"/some/%2e/path")) + self.assertTrue(has_dot_segments(b"/some/%2E%2E/path")) + + def test_percent_encoded_separator_is_detected(self) -> None: + self.assertTrue(has_dot_segments(b"/some%2f../path")) + + def test_double_encoded_dot_segments_are_not_detected(self) -> None: + # Only a single decode is performed, matching the single decode that route + # arguments get elsewhere, so a double-encoded segment is left alone. + self.assertFalse(has_dot_segments(b"/some/%252e%252e/path")) diff --git a/tests/http/test_site.py b/tests/http/test_site.py index 654ec3190b..c93aadfee1 100644 --- a/tests/http/test_site.py +++ b/tests/http/test_site.py @@ -19,6 +19,8 @@ # # +from parameterized import parameterized + from twisted.internet.address import IPv6Address from twisted.internet.testing import MemoryReactor, StringTransport @@ -92,7 +94,16 @@ class SynapseRequestTestCase(HomeserverTestCase): # that. self.assertEqual(sent, 50 * 1024 * 1024 + 1024) - def test_content_type_multipart(self) -> None: + @parameterized.expand( + [ + (b"multipart/form-data",), + # Also check with a boundary + (b"multipart/form-data; boundary=abc123",), + # Headers are case-insensitive, so test that too. + (b"Multipart/Form-Data",), + ] + ) + def test_content_type_multipart(self, content_type: bytes) -> None: """HTTP POST requests with `content-type: multipart/form-data` should be rejected""" self.hs.start_listening() @@ -133,7 +144,7 @@ class SynapseRequestTestCase(HomeserverTestCase): b"POST / HTTP/1.1\r\n" b"Connection: close\r\n" b"Transfer-Encoding: chunked\r\n" - b"Content-Type: multipart/form-data\r\n" + b"Content-Type: " + content_type + b"\r\n" b"\r\n" b"0\r\n" b"\r\n" diff --git a/tests/media/test_media_storage.py b/tests/media/test_media_storage.py index 855a623ec0..da5fa8a140 100644 --- a/tests/media/test_media_storage.py +++ b/tests/media/test_media_storage.py @@ -52,13 +52,17 @@ from synapse.media.storage_provider import ( FileStorageProviderBackend, StorageProviderWrapper, ) -from synapse.media.thumbnailer import ThumbnailProvider +from synapse.media.thumbnailer import ( + ANIMATED_THUMBNAIL_TYPE, + Thumbnailer, + ThumbnailProvider, +) from synapse.module_api import ModuleApi from synapse.module_api.callbacks.spamchecker_callbacks import load_legacy_spam_checkers from synapse.rest import admin from synapse.rest.client import login, media from synapse.server import HomeServer -from synapse.types import JsonDict, RoomAlias +from synapse.types import JsonDict, RoomAlias, UserID from synapse.util.clock import Clock from tests import unittest @@ -1407,3 +1411,319 @@ class MediaRepoSizeModuleCallbackTestCase(unittest.HomeserverTestCase): self.helper.upload_media(SMALL_PNG, tok=self.tok, expect_code=413) assert self.last_user_id == self.user assert self.last_size == len(SMALL_PNG) + + +def _make_animated_gif() -> bytes: + """Build a small two-frame animated GIF.""" + frames = [Image.new("RGB", (64, 64), color) for color in ((255, 0, 0), (0, 0, 255))] + out = BytesIO() + frames[0].save( + out, + format="GIF", + save_all=True, + append_images=frames[1:], + duration=100, + loop=0, + ) + return out.getvalue() + + +def _make_mpo() -> bytes: + """Build a two-image MPO: a JPEG holding a stereo pair, not an animation.""" + frames = [Image.new("RGB", (64, 64), color) for color in ((255, 0, 0), (0, 0, 255))] + out = BytesIO() + frames[0].save(out, format="MPO", save_all=True, append_images=frames[1:]) + return out.getvalue() + + +def _make_stale_mpo() -> bytes: + """Build an MPO whose trailing image is stripped but still advertised.""" + data = _make_mpo() + with Image.open(BytesIO(data)) as image: + primary_size = image.mpinfo[0xB002][0]["Size"] # type: ignore[attr-defined] + return data[:primary_size] + + +def _make_webp(alpha: int) -> bytes: + """Build a small WebP whose pixels all have the given alpha.""" + out = BytesIO() + Image.new("RGBA", (64, 64), (255, 0, 0, alpha)).save( + out, format="WEBP", lossless=True + ) + return out.getvalue() + + +class ThumbnailerAnimatedTestCase(unittest.TestCase): + """Tests that the thumbnailer only animates when explicitly asked to.""" + + def setUp(self) -> None: + super().setUp() + self.tempdir = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, self.tempdir, ignore_errors=True) + + self.gif_path = os.path.join(self.tempdir, "animated.gif") + with open(self.gif_path, "wb") as f: + f.write(_make_animated_gif()) + + self.png_path = os.path.join(self.tempdir, "static.png") + with open(self.png_path, "wb") as f: + f.write(SMALL_PNG) + + self.mpo_path = os.path.join(self.tempdir, "stereo.jpg") + with open(self.mpo_path, "wb") as f: + f.write(_make_mpo()) + + self.stale_mpo_path = os.path.join(self.tempdir, "stale.jpg") + with open(self.stale_mpo_path, "wb") as f: + f.write(_make_stale_mpo()) + + def assert_is_first_frame(self, output: BytesIO) -> None: + """Raises an `AssertionError` unless the given image is red.""" + pixel = Image.open(output).convert("RGB").getpixel((16, 16)) + assert isinstance(pixel, tuple) + red, green, blue = pixel + # The first frame of every source here is red. + # WebP is lossy, so allow *some* green/blue to be present. + self.assertGreater(red, 200) + self.assertLess(max(green, blue), 50) + + def test_scale_static_by_default(self) -> None: + """An animated source produces a static thumbnail unless animated=True.""" + with Thumbnailer(self.gif_path) as thumbnailer: + out = thumbnailer.scale(32, 32, "image/png") + result = Image.open(out) + self.assertFalse(getattr(result, "is_animated", False)) + + def test_scale_animated_when_requested(self) -> None: + """An animated source produces an animated thumbnail when animated=True.""" + with Thumbnailer(self.gif_path) as thumbnailer: + out = thumbnailer.scale(32, 32, ANIMATED_THUMBNAIL_TYPE, animated=True) + result = Image.open(out) + self.assertEqual(result.format, "WEBP") + self.assertTrue(getattr(result, "is_animated", False)) + self.assertEqual(result.n_frames, 2) + + def test_crop_animated_when_requested(self) -> None: + with Thumbnailer(self.gif_path) as thumbnailer: + out = thumbnailer.crop(32, 32, ANIMATED_THUMBNAIL_TYPE, animated=True) + result = Image.open(out) + self.assertEqual(result.format, "WEBP") + self.assertTrue(getattr(result, "is_animated", False)) + self.assertEqual(result.size, (32, 32)) + + def test_static_source_never_animates(self) -> None: + """A non-animated source stays static even when animated=True.""" + with Thumbnailer(self.png_path) as thumbnailer: + self.assertFalse(thumbnailer.is_animated) + out = thumbnailer.scale(1, 1, ANIMATED_THUMBNAIL_TYPE, animated=True) + result = Image.open(out) + self.assertFalse(getattr(result, "is_animated", False)) + + @parameterized.expand([("GIF", "gif"), ("PNG", "apng"), ("WEBP", "webp")]) + def test_animated_formats(self, fmt: str, ext: str) -> None: + """Every animated format we accept produces an animated thumbnail.""" + frames = [ + Image.new("RGBA", (64, 64), color) + for color in ((255, 0, 0, 255), (0, 0, 255, 255)) + ] + out = BytesIO() + frames[0].save( + out, + format=fmt, + save_all=True, + append_images=frames[1:], + duration=100, + loop=0, + ) + path = os.path.join(self.tempdir, f"animated.{ext}") + with open(path, "wb") as f: + f.write(out.getvalue()) + + with Thumbnailer(path) as thumbnailer: + self.assertTrue(thumbnailer.is_animated) + thumbnail = thumbnailer.scale( + 32, 32, ANIMATED_THUMBNAIL_TYPE, animated=True + ) + result = Image.open(thumbnail) + self.assertEqual(result.format, "WEBP") + self.assertTrue(getattr(result, "is_animated", False)) + self.assertEqual(getattr(result, "n_frames", 1), 2) + + @parameterized.expand(["scale", "crop"]) + def test_mpo_is_not_animated(self, method: str) -> None: + """An MPO packs several stills into one JPEG; not an animation.""" + with Thumbnailer(self.mpo_path) as thumbnailer: + self.assertFalse(thumbnailer.is_animated) + out = getattr(thumbnailer, method)( + 32, 32, ANIMATED_THUMBNAIL_TYPE, animated=True + ) + self.assertFalse(getattr(Image.open(out), "is_animated", False)) + self.assert_is_first_frame(out) + + @parameterized.expand(["scale", "crop"]) + def test_stale_mpo_index_does_not_raise(self, method: str) -> None: + """An MPO advertising frames that are not in the file still thumbnails. + + Regression test for https://github.com/element-hq/synapse/issues/20024. + """ + with Thumbnailer(self.stale_mpo_path) as thumbnailer: + out = getattr(thumbnailer, method)( + 32, 32, ANIMATED_THUMBNAIL_TYPE, animated=True + ) + self.assertEqual(Image.open(out).format, "WEBP") + self.assert_is_first_frame(out) + + def test_fallback_thumbnails_the_first_frame(self) -> None: + """Failing after the frames are read leaves the source parked on the + last one, so the fallback has to rewind.""" + with patch.object(Thumbnailer, "_encode_animated", side_effect=ValueError): + with Thumbnailer(self.gif_path) as thumbnailer: + out = thumbnailer.scale(32, 32, ANIMATED_THUMBNAIL_TYPE, animated=True) + self.assert_is_first_frame(out) + + @parameterized.expand(["scale", "crop"]) + def test_undecodable_animation_falls_back_to_static(self, method: str) -> None: + """If the frames can't be decoded we still serve a static thumbnail of + the first frame rather than failing the request.""" + # Force the stale MPO down the animated path so decoding it fails. + with patch.object(Thumbnailer, "ANIMATED_FORMATS", frozenset({"MPO"})): + with Thumbnailer(self.stale_mpo_path) as thumbnailer: + self.assertTrue(thumbnailer.is_animated) + out = getattr(thumbnailer, method)( + 32, 32, ANIMATED_THUMBNAIL_TYPE, animated=True + ) + # A broken source isn't retried for every other thumbnail size. + self.assertFalse(thumbnailer.is_animated) + + self.assertEqual(Image.open(out).format, "WEBP") + self.assertFalse(getattr(Image.open(out), "is_animated", False)) + self.assert_is_first_frame(out) + + +class ThumbnailerTransparencyTestCase(unittest.TestCase): + """Tests the transparency detection that picks the thumbnail format.""" + + def setUp(self) -> None: + super().setUp() + self.tempdir = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, self.tempdir, ignore_errors=True) + + def _thumbnailer(self, name: str, data: bytes) -> Thumbnailer: + path = os.path.join(self.tempdir, name) + with open(path, "wb") as f: + f.write(data) + thumbnailer = Thumbnailer(path) + self.addCleanup(thumbnailer.close) + return thumbnailer + + def test_transparent_webp(self) -> None: + thumbnailer = self._thumbnailer("transparent.webp", _make_webp(0)) + self.assertTrue(thumbnailer.has_transparency) + + def test_partially_transparent_webp(self) -> None: + thumbnailer = self._thumbnailer("partial.webp", _make_webp(128)) + self.assertTrue(thumbnailer.has_transparency) + + def test_opaque_webp(self) -> None: + thumbnailer = self._thumbnailer("opaque.webp", _make_webp(255)) + self.assertFalse(thumbnailer.has_transparency) + + def test_unused_alpha_channel(self) -> None: + """An alpha channel that is fully opaque doesn't count as transparency.""" + out = BytesIO() + Image.new("RGBA", (64, 64), (255, 0, 0, 255)).save(out, format="PNG") + thumbnailer = self._thumbnailer("opaque_rgba.png", out.getvalue()) + self.assertEqual(thumbnailer.image.mode, "RGBA") + self.assertFalse(thumbnailer.has_transparency) + + def test_palette_transparency(self) -> None: + """Palette images signal transparency through an index, not a channel.""" + out = BytesIO() + Image.new("P", (64, 64)).save(out, format="PNG", transparency=0) + thumbnailer = self._thumbnailer("palette.png", out.getvalue()) + self.assertEqual(thumbnailer.image.mode, "P") + self.assertTrue(thumbnailer.has_transparency) + + def test_cmyk_jpeg(self) -> None: + thumbnailer = self._thumbnailer("opaque.jpg", SMALL_CMYK_JPEG) + self.assertFalse(thumbnailer.has_transparency) + + def test_png_thumbnail_keeps_alpha(self) -> None: + """The PNG we switch to actually retains the transparency.""" + thumbnailer = self._thumbnailer("transparent.webp", _make_webp(0)) + result = Image.open(thumbnailer.scale(32, 32, "image/png")) + self.assertEqual(result.format, "PNG") + pixel = result.convert("RGBA").getpixel((16, 16)) + assert isinstance(pixel, tuple) + self.assertEqual(pixel[3], 0) + + +class ThumbnailFormatTestCase(unittest.HomeserverTestCase): + """Tests that transparent sources aren't flattened onto a black background.""" + + servlets = [ + admin.register_servlets, + login.register_servlets, + media.register_servlets, + ] + + def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None: + self.store = hs.get_datastores().main + self.media_repo = hs.get_media_repository() + self.user = self.register_user("user", "pass") + self.tok = self.login("user", "pass") + + def create_resource_dict(self) -> dict[str, Resource]: + resources = super().create_resource_dict() + resources["/_matrix/media"] = self.hs.get_media_repository_resource() + return resources + + def _upload(self, data: bytes, media_type: str) -> str: + """Upload the given media and return its media ID.""" + mxc = self.get_success( + self.media_repo.create_or_update_content( + media_type, + "test", + BytesIO(data), + len(data), + UserID.from_string(self.user), + ) + ) + return mxc.media_id + + def _thumbnail_types(self, media_id: str) -> set[str]: + thumbnails = self.get_success(self.store.get_local_media_thumbnails(media_id)) + self.assertTrue(thumbnails, "no thumbnails were generated") + return {thumbnail.type for thumbnail in thumbnails} + + def test_transparent_webp_thumbnails_as_png(self) -> None: + media_id = self._upload(_make_webp(0), "image/webp") + self.assertEqual(self._thumbnail_types(media_id), {"image/png"}) + + def test_opaque_webp_thumbnails_as_jpeg(self) -> None: + media_id = self._upload(_make_webp(255), "image/webp") + self.assertEqual(self._thumbnail_types(media_id), {"image/jpeg"}) + + def test_animated_thumbnail_is_still_webp(self) -> None: + """Transparency detection doesn't disturb the animated thumbnails.""" + media_id = self._upload(_make_animated_gif(), "image/gif") + self.assertIn(ANIMATED_THUMBNAIL_TYPE, self._thumbnail_types(media_id)) + + def test_served_thumbnail_keeps_transparency(self) -> None: + """The thumbnail a client actually receives still has its alpha channel.""" + media_id = self._upload(_make_webp(0), "image/webp") + + channel = self.make_request( + "GET", + f"/_matrix/client/v1/media/thumbnail/test/{media_id}" + "?width=32&height=32&method=scale", + shorthand=False, + access_token=self.tok, + ) + self.assertEqual(channel.code, 200) + self.assertEqual(channel.headers.getRawHeaders(b"Content-Type"), [b"image/png"]) + + thumbnail = Image.open(BytesIO(channel.result["body"])) + pixel = thumbnail.convert("RGBA").getpixel((16, 16)) + assert isinstance(pixel, tuple) + self.assertEqual(pixel[3], 0) diff --git a/tests/module_api/test_api.py b/tests/module_api/test_api.py index 3114675052..b4b14c87b9 100644 --- a/tests/module_api/test_api.py +++ b/tests/module_api/test_api.py @@ -743,7 +743,6 @@ class ModuleApiTestCase(BaseModuleApiTestCase): # Now do the happy path. user_id = self.register_user("user", "password") - access_token = self.login(user_id, "password") room_id, room_alias = self.get_success( self.module_api.create_room( @@ -751,15 +750,6 @@ class ModuleApiTestCase(BaseModuleApiTestCase): ) ) - # Check room creator. - channel = self.make_request( - "GET", - f"/_matrix/client/v3/rooms/{room_id}/state/m.room.create", - access_token=access_token, - ) - self.assertEqual(channel.code, 200, channel.result) - self.assertEqual(channel.json_body["creator"], user_id) - # Check room alias. self.assertEqual(room_alias, f"#foo-bar:{self.module_api.server_name}") @@ -768,15 +758,6 @@ class ModuleApiTestCase(BaseModuleApiTestCase): self.module_api.create_room(user_id=user_id, config={}, ratelimit=False) ) - # Check room creator. - channel = self.make_request( - "GET", - f"/_matrix/client/v3/rooms/{room_id}/state/m.room.create", - access_token=access_token, - ) - self.assertEqual(channel.code, 200, channel.result) - self.assertEqual(channel.json_body["creator"], user_id) - # Check room alias. self.assertIsNone(room_alias) diff --git a/tests/module_api/test_federation_callbacks.py b/tests/module_api/test_federation_callbacks.py new file mode 100644 index 0000000000..b8a34a0866 --- /dev/null +++ b/tests/module_api/test_federation_callbacks.py @@ -0,0 +1,378 @@ +# +# This file is licensed under the Affero General Public License (AGPL) version 3. +# +# Copyright (C) 2026 Element Creations Ltd +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# See the GNU Affero General Public License for more details: +# . +# +from http import HTTPStatus +from typing import Callable +from unittest.mock import AsyncMock, Mock + +from twisted.internet.testing import MemoryReactor + +from synapse.api.constants import EventTypes +from synapse.api.room_versions import KNOWN_ROOM_VERSIONS +from synapse.config.server import DEFAULT_ROOM_VERSION +from synapse.federation.federation_base import event_from_pdu_json +from synapse.federation.units import Transaction +from synapse.module_api.callbacks.federation import ( + FederatedEventDeliveryMethod, + FederationEventDeliveryEvent, +) +from synapse.rest import admin +from synapse.rest.client import login, room +from synapse.server import HomeServer +from synapse.types import JsonDict +from synapse.util.clock import Clock + +from tests import unittest + + +class FederationDeliveryCallbackTests(unittest.FederatingHomeserverTestCase): + """ + Tests for `on_event_delivered_over_federation` module callbacks. + """ + + servlets = [ + admin.register_servlets, + room.register_servlets, + login.register_servlets, + ] + + def make_homeserver(self, reactor: MemoryReactor, clock: Clock) -> HomeServer: + # Mock out the calls over federation. + self.fed_transport_client = Mock(spec=["send_transaction"]) + self.fed_transport_client.send_transaction = AsyncMock(return_value={}) + + hs = self.setup_test_homeserver( + federation_transport_client=self.fed_transport_client, + ) + + return hs + + def default_config(self) -> JsonDict: + # By default, federation sending is disabled in tests. + # Re-enable it for the main process. + config = super().default_config() + config["federation_sender_instances"] = None + return config + + def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None: + super().prepare(reactor, clock, hs) + + # Record every delivery the module callback is told about. + self._deliveries: list[FederationEventDeliveryEvent] = [] + + async def record(delivery: FederationEventDeliveryEvent) -> None: + self._deliveries.append(delivery) + + hs.get_module_api().register_federation_callbacks( + on_event_delivered_over_federation=record + ) + + # Create a public room with the remote server joined + self.creator = self.register_user("creator", "pass") + self.creator_tok = self.login("creator", "pass") + self.room_id = self.helper.create_room_as( + self.creator, tok=self.creator_tok, is_public=True + ) + self.remote_user = f"@remote:{self.OTHER_SERVER_NAME}" + self.inject_room_member(self.room_id, self.remote_user, "join") + + def _assert_only_delivery( + self, + method: FederatedEventDeliveryMethod, + ) -> FederationEventDeliveryEvent: + """ + Assert that exactly one delivery, with the given `method`, is currently recorded + (since the tracker was last cleared) and return it. + + This clears the tracker. + """ + self.assertEqual( + len(self._deliveries), + 1, + f"expected exactly one delivery; saw {self._deliveries!r}", + ) + + delivery = self._deliveries[0] + self._deliveries.clear() + + self.assertEqual(delivery.method, method, delivery) + + return delivery + + def test_backfill(self) -> None: + """ + Tests that the callback is triggered for incoming `/backfill` requests. + """ + ( + message_event_id1, + message_event_id2, + message_event_id3, + ) = self.helper.send_messages(self.room_id, 3, tok=self.creator_tok) + + # Call the endpoint twice to make sure that it doesn't forget to + # trigger the callback a second time, for example because it has + # a `ResponseCache` that bypasses the logic that triggers the + # callback. + for _ in range(2): + channel = self.make_signed_federation_request( + "GET", + f"/_matrix/federation/v1/backfill/{self.room_id}" + f"?v={message_event_id3}&limit=3", + ) + self.assertEqual(channel.code, HTTPStatus.OK, channel.result) + delivery = self._assert_only_delivery(FederatedEventDeliveryMethod.BACKFILL) + self.assertEqual(delivery.server_name, self.OTHER_SERVER_NAME) + self.assertEqual( + {e.event_id for e in delivery.events}, + {message_event_id1, message_event_id2, message_event_id3}, + ) + + def test_event(self) -> None: + """ + Tests that the callback is triggered for incoming `/event` requests. + """ + (message_event_id,) = self.helper.send_messages( + self.room_id, 1, tok=self.creator_tok + ) + + # Call the endpoint twice to make sure that it doesn't forget to + # trigger the callback a second time, for example because it has + # a `ResponseCache` that bypasses the logic that triggers the + # callback. + for _ in range(2): + channel = self.make_signed_federation_request( + "GET", f"/_matrix/federation/v1/event/{message_event_id}" + ) + self.assertEqual(channel.code, HTTPStatus.OK, channel.result) + delivery = self._assert_only_delivery(FederatedEventDeliveryMethod.EVENT) + self.assertIncludes( + {e.event_id for e in delivery.events}, + {message_event_id}, + exact=True, + ) + + def test_event_auth(self) -> None: + """ + Tests that the callback is triggered for incoming `/event_auth` requests. + """ + (message_event_id,) = self.helper.send_messages( + self.room_id, 1, tok=self.creator_tok + ) + + # Call the endpoint twice to make sure that it doesn't forget to + # trigger the callback a second time, for example because it has + # a `ResponseCache` that bypasses the logic that triggers the + # callback. + for _ in range(2): + channel = self.make_signed_federation_request( + "GET", + f"/_matrix/federation/v1/event_auth/{self.room_id}/{message_event_id}", + ) + self.assertEqual(channel.code, HTTPStatus.OK, channel.result) + delivery = self._assert_only_delivery( + FederatedEventDeliveryMethod.EVENT_AUTH + ) + + state_key_pairs_included = {(e.type, e.state_key) for e in delivery.events} + self.assertEqual( + state_key_pairs_included, + { + (EventTypes.Create, ""), + (EventTypes.PowerLevels, ""), + (EventTypes.Member, self.creator), + }, + ) + + def test_state(self) -> None: + """ + Tests that the callback is triggered for incoming `/state` requests. + """ + (message_event_id,) = self.helper.send_messages( + self.room_id, 1, tok=self.creator_tok + ) + + # Call the endpoint twice to make sure that it doesn't forget to + # trigger the callback a second time, for example because it has + # a `ResponseCache` that bypasses the logic that triggers the + # callback. + for _ in range(2): + channel = self.make_signed_federation_request( + "GET", + f"/_matrix/federation/v1/state/{self.room_id}?event_id={message_event_id}", + ) + self.assertEqual(channel.code, HTTPStatus.OK, channel.result) + delivery = self._assert_only_delivery(FederatedEventDeliveryMethod.STATE) + + # Check that we got notified about delivery for all the expected state events + # included in a `/state` response (including `pdus` and `auth_chain`) + state_key_pairs_included = {(e.type, e.state_key) for e in delivery.events} + self.assertEqual( + state_key_pairs_included, + { + (EventTypes.Create, ""), + (EventTypes.JoinRules, ""), + (EventTypes.PowerLevels, ""), + (EventTypes.RoomHistoryVisibility, ""), + (EventTypes.Member, self.creator), + (EventTypes.Member, self.remote_user), + }, + ) + + def test_get_missing_events(self) -> None: + """ + Tests that the callback is triggered for incoming `/get_missing_events` requests. + """ + ( + message_event_id1, + message_event_id2, + message_event_id3, + ) = self.helper.send_messages(self.room_id, 3, tok=self.creator_tok) + + # Call the endpoint twice to make sure that it doesn't forget to + # trigger the callback a second time, for example because it has + # a `ResponseCache` that bypasses the logic that triggers the + # callback. + for _ in range(2): + channel = self.make_signed_federation_request( + "POST", + f"/_matrix/federation/v1/get_missing_events/{self.room_id}", + { + "earliest_events": [message_event_id1], + "latest_events": [message_event_id3], + "limit": 10, + }, + ) + self.assertEqual(channel.code, HTTPStatus.OK, channel.result) + delivery = self._assert_only_delivery( + FederatedEventDeliveryMethod.GET_MISSING_EVENTS + ) + self.assertIncludes( + {e.event_id for e in delivery.events}, + {message_event_id2}, + exact=True, + ) + + def test_send_join(self) -> None: + """ + Tests that the callback is triggered for incoming `/send_join` requests, + including both the state events and the newly-created join event. + """ + joining_user = f"@joiner:{self.OTHER_SERVER_NAME}" + make_join = self.make_signed_federation_request( + "GET", + f"/_matrix/federation/v1/make_join/{self.room_id}/{joining_user}" + f"?ver={DEFAULT_ROOM_VERSION}", + ) + self.assertEqual(make_join.code, HTTPStatus.OK, make_join.json_body) + + join_event_dict = make_join.json_body["event"] + self.add_hashes_and_signatures_from_other_server( + join_event_dict, KNOWN_ROOM_VERSIONS[DEFAULT_ROOM_VERSION] + ) + channel = self.make_signed_federation_request( + "PUT", + f"/_matrix/federation/v2/send_join/{self.room_id}/x", + content=join_event_dict, + ) + self.assertEqual(channel.code, HTTPStatus.OK, channel.json_body) + + delivery = self._assert_only_delivery(FederatedEventDeliveryMethod.SEND_JOIN) + + # Check that we got notified about delivery for all the expected state events + # included in a `/send_join` response (the join itself, the room state + # and the auth chain events) + state_key_pairs_included = {(e.type, e.state_key) for e in delivery.events} + self.assertIncludes( + state_key_pairs_included, + { + (EventTypes.Create, ""), + (EventTypes.JoinRules, ""), + (EventTypes.PowerLevels, ""), + (EventTypes.RoomHistoryVisibility, ""), + (EventTypes.Member, self.creator), + (EventTypes.Member, self.remote_user), + (EventTypes.Member, joining_user), + }, + exact=True, + ) + + def test_send_outbound_transaction(self) -> None: + """ + Tests that the callback is triggered for outgoing `/send` transactions + when the remote acknowledges the PDU. + """ + + async def _acknowledge_pdus( + transaction: Transaction, + json_data_cb: Callable[[], JsonDict], + ) -> JsonDict: + """ + Acknowledge the PDUs. + """ + body = json_data_cb() + pdu_responses: JsonDict = {} + for pdu_json in body.get("pdus", []): + # We have to construct the event to calculate its event ID + event = event_from_pdu_json( + pdu_json, KNOWN_ROOM_VERSIONS[DEFAULT_ROOM_VERSION] + ) + # Empty dict means 'OK' + pdu_responses[event.event_id] = {} + return {"pdus": pdu_responses} + + self.fed_transport_client.send_transaction.side_effect = _acknowledge_pdus + + # After sending, the event propagates to the federation transmission queue + # and gets fired as a `/send` request + (message_event_id,) = self.helper.send_messages( + self.room_id, 1, tok=self.creator_tok + ) + + delivery = self._assert_only_delivery(FederatedEventDeliveryMethod.SEND) + self.assertEqual(delivery.server_name, self.OTHER_SERVER_NAME) + self.assertIncludes( + {e.event_id for e in delivery.events}, {message_event_id}, exact=True + ) + + def test_send_outbound_excludes_rejected_pdus(self) -> None: + """ + Tests that the event is NOT triggered for outgoing `/send` transactions + when the remote marks the PDU as failed. + """ + + async def _error_pdus( + transaction: Transaction, + json_data_cb: Callable[[], JsonDict], + ) -> JsonDict: + """ + Return an error for the PDUs. + """ + body = json_data_cb() + pdu_responses: JsonDict = {} + for pdu_json in body.get("pdus", []): + # We have to construct the event to calculate its event ID + event = event_from_pdu_json( + pdu_json, KNOWN_ROOM_VERSIONS[DEFAULT_ROOM_VERSION] + ) + pdu_responses[event.event_id] = {"error": "failed"} + return {"pdus": pdu_responses} + + self.fed_transport_client.send_transaction.side_effect = _error_pdus + + # After sending, the event propagates to the federation transmission queue + # and gets fired as a `/send` request + (_message_event_id,) = self.helper.send_messages( + self.room_id, 1, tok=self.creator_tok + ) + + self.assertIncludes(set(self._deliveries), set(), exact=True) diff --git a/tests/replication/tcp/streams/test_quarantined_media.py b/tests/replication/tcp/streams/test_quarantined_media.py new file mode 100644 index 0000000000..6d8533c7a8 --- /dev/null +++ b/tests/replication/tcp/streams/test_quarantined_media.py @@ -0,0 +1,86 @@ +# +# This file is licensed under the Affero General Public License (AGPL) version 3. +# +# Copyright (C) 2026 Element Creations Ltd +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# See the GNU Affero General Public License for more details: +# . +# + +from synapse.replication.tcp.streams import QuarantinedMediaStream +from synapse.types import UserID + +from tests.replication._base import BaseMultiWorkerStreamTestCase + + +class QuarantinedMediaWorkerWriterTestCase(BaseMultiWorkerStreamTestCase): + """Checks that the quarantined_media stream is replicated when the + configured stream writer is a worker rather than the main process. + """ + + def default_config(self) -> dict: + conf = super().default_config() + conf["stream_writers"] = {"quarantined_media_changes": ["worker1"]} + conf["instance_map"] = { + "main": {"host": "testserv", "port": 8765}, + "worker1": {"host": "testserv", "port": 1001}, + } + return conf + + def test_quarantine_on_worker_writer_replicates_to_main(self) -> None: + main_store = self.hs.get_datastores().main + + worker_hs = self.make_worker_hs( + "synapse.app.generic_worker", {"worker_name": "worker1"} + ) + worker_store = worker_hs.get_datastores().main + + # The worker must consider itself a source of the stream... + self.assertIn( + QuarantinedMediaStream.NAME, + { + stream.NAME + for stream in worker_hs.get_replication_command_handler().get_streams_to_replicate() + }, + ) + # ... and the main process must not, as it isn't a writer. + self.assertNotIn( + QuarantinedMediaStream.NAME, + { + stream.NAME + for stream in self.hs.get_replication_command_handler().get_streams_to_replicate() + }, + ) + + # Quarantining only records a change for media that exists. + self.get_success( + main_store.store_local_media( + media_id="media_id1", + media_type="text/plain", + time_now_ms=self.clock.time_msec(), + upload_name=None, + media_length=100, + user_id=UserID.from_string("@user:test"), + ) + ) + + initial_token = main_store.get_current_quarantined_media_stream_id() + + # Quarantine the media on the worker, i.e. the configured writer. + self.get_success( + worker_store.quarantine_media_by_id("test", "media_id1", "@admin:test") + ) + + self.replicate() + + # The main process only learns of the new stream ID over replication, + # even though the two instances share a database. + self.assertEqual( + main_store.get_current_quarantined_media_stream_id(), + initial_token + 1, + ) diff --git a/tests/rest/admin/test_room.py b/tests/rest/admin/test_room.py index c4e4170c6f..4deb3c29f4 100644 --- a/tests/rest/admin/test_room.py +++ b/tests/rest/admin/test_room.py @@ -2549,7 +2549,7 @@ class RoomMessagesTestCase(unittest.HomeserverTestCase): def test_topo_token_is_accepted(self) -> None: """Test Topo Token is accepted.""" - token = "t1-0_0_0_0_0_0_0_0_0_0_0_0_0" + token = "t1-0_0_0_0_0_0_0_0_0_0_0_0_0_0" channel = self.make_request( "GET", "/_synapse/admin/v1/rooms/%s/messages?from=%s" % (self.room_id, token), @@ -2563,7 +2563,7 @@ class RoomMessagesTestCase(unittest.HomeserverTestCase): def test_stream_token_is_accepted_for_fwd_pagianation(self) -> None: """Test that stream token is accepted for forward pagination.""" - token = "s0_0_0_0_0_0_0_0_0_0_0_0_0" + token = "s0_0_0_0_0_0_0_0_0_0_0_0_0_0" channel = self.make_request( "GET", "/_synapse/admin/v1/rooms/%s/messages?from=%s" % (self.room_id, token), diff --git a/tests/rest/admin/test_scheduled_tasks.py b/tests/rest/admin/test_scheduled_tasks.py index 4b7adb6b89..388570df0b 100644 --- a/tests/rest/admin/test_scheduled_tasks.py +++ b/tests/rest/admin/test_scheduled_tasks.py @@ -190,3 +190,56 @@ class ScheduledTasksAdminApiTestCase(unittest.HomeserverTestCase): # only the task with the matching resource id should have been returned self.assertEqual(len(found_tasks), 1) self.assertEqual(found_tasks[0]["resource_id"], "failed_task") + + def test_filtering_scheduled_tasks_multiple_values(self) -> None: + """ + Test that the `action_name` and `status` filters can be given multiple + times, returning tasks matching any of the given values. + """ + # filter via multiple statuses + channel = self.make_request( + "GET", + "/_synapse/admin/v1/scheduled_tasks?status=active&status=failed", + content={}, + access_token=self.admin_user_tok, + ) + self.assertEqual(200, channel.code, msg=channel.json_body) + found_tasks = self.check_scheduled_tasks_response( + channel.json_body["scheduled_tasks"] + ) + + # the active and failed tasks should have been returned + self.assertEqual(len(found_tasks), 2) + self.assertEqual({task["status"] for task in found_tasks}, {"active", "failed"}) + + # filter via multiple action names + channel = self.make_request( + "GET", + "/_synapse/admin/v1/scheduled_tasks?action_name=test_task&action_name=finished_test_task", + content={}, + access_token=self.admin_user_tok, + ) + self.assertEqual(200, channel.code, msg=channel.json_body) + found_tasks = self.check_scheduled_tasks_response( + channel.json_body["scheduled_tasks"] + ) + + # only the tasks with the given action names should have been returned + self.assertEqual(len(found_tasks), 2) + self.assertEqual( + {task["action"] for task in found_tasks}, + {"test_task", "finished_test_task"}, + ) + + def test_filtering_scheduled_tasks_invalid_status(self) -> None: + """ + Test that an invalid `status` value is rejected with a 400 error. + """ + channel = self.make_request( + "GET", + "/_synapse/admin/v1/scheduled_tasks?status=unknown_status", + content={}, + access_token=self.admin_user_tok, + ) + self.assertEqual(400, channel.code, msg=channel.json_body) + self.assertEqual(Codes.INVALID_PARAM, channel.json_body["errcode"]) diff --git a/tests/rest/admin/test_user.py b/tests/rest/admin/test_user.py index bce199c564..84bb7e319b 100644 --- a/tests/rest/admin/test_user.py +++ b/tests/rest/admin/test_user.py @@ -39,6 +39,7 @@ from synapse.api.constants import ( EventContentFields, EventTypes, LoginType, + ProfileFields, UserTypes, ) from synapse.api.errors import Codes, HttpResponseException, ResourceLimitError @@ -943,18 +944,24 @@ class UsersListTestCase(unittest.HomeserverTestCase): # Set avatar URL to all users, that no user has a NULL value to avoid # different sort order between SQlite and PostreSQL self.get_success( - self.store.set_profile_avatar_url( - UserID.from_string("@user1:test"), "mxc://url3" + self.store.set_profile_field( + user_id=UserID.from_string("@user1:test"), + field_name=ProfileFields.AVATAR_URL, + new_value="mxc://url3", ) ) self.get_success( - self.store.set_profile_avatar_url( - UserID.from_string("@user2:test"), "mxc://url2" + self.store.set_profile_field( + user_id=UserID.from_string("@user2:test"), + field_name=ProfileFields.AVATAR_URL, + new_value="mxc://url2", ) ) self.get_success( - self.store.set_profile_avatar_url( - UserID.from_string("@admin:test"), "mxc://url1" + self.store.set_profile_field( + user_id=UserID.from_string("@admin:test"), + field_name=ProfileFields.AVATAR_URL, + new_value="mxc://url1", ) ) @@ -1417,6 +1424,10 @@ class UsersListTestCase(unittest.HomeserverTestCase): self.assertIn("avatar_url", u) self.assertIn("creation_ts", u) self.assertIn("last_seen_ts", u) + if self.hs.config.experimental.msc3866.enabled: + self.assertIn("approved", u) + else: + self.assertNotIn("approved", u) def _create_users(self, number_users: int) -> None: """ @@ -1546,8 +1557,10 @@ class DeactivateAccountTestCase(unittest.HomeserverTestCase): # set attributes for user self.get_success( - self.store.set_profile_avatar_url( - UserID.from_string("@user:test"), "mxc://servername/mediaid" + self.store.set_profile_field( + user_id=UserID.from_string("@user:test"), + field_name=ProfileFields.AVATAR_URL, + new_value="mxc://servername/mediaid", ) ) self.get_success( @@ -1679,7 +1692,11 @@ class DeactivateAccountTestCase(unittest.HomeserverTestCase): """ # Patch `self.other_user` to have an empty string as their avatar. self.get_success( - self.store.set_profile_avatar_url(UserID.from_string("@user:test"), "") + self.store.set_profile_field( + user_id=UserID.from_string("@user:test"), + field_name=ProfileFields.AVATAR_URL, + new_value="", + ) ) # Check we can still erase them. @@ -2758,8 +2775,10 @@ class UserRestTestCase(unittest.HomeserverTestCase): # set attributes for user self.get_success( - self.store.set_profile_avatar_url( - UserID.from_string("@user:test"), "mxc://servername/mediaid" + self.store.set_profile_field( + user_id=UserID.from_string("@user:test"), + field_name=ProfileFields.AVATAR_URL, + new_value="mxc://servername/mediaid", ) ) self.get_success( diff --git a/tests/rest/client/sliding_sync/test_extension_profiles.py b/tests/rest/client/sliding_sync/test_extension_profiles.py new file mode 100644 index 0000000000..40d426fb87 --- /dev/null +++ b/tests/rest/client/sliding_sync/test_extension_profiles.py @@ -0,0 +1,1231 @@ +# +# This file is licensed under the Affero General Public License (AGPL) version 3. +# +# Copyright (C) 2024 New Vector, Ltd +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# See the GNU Affero General Public License for more details: +# . +# +import logging + +from parameterized import parameterized, parameterized_class + +from twisted.internet.testing import MemoryReactor + +import synapse.rest.admin +from synapse.api.constants import ProfileFields +from synapse.rest.client import knock, login, profile, room, sync +from synapse.server import HomeServer +from synapse.types import UserID, create_requester +from synapse.util.clock import Clock + +from tests.rest.client.sliding_sync.test_sliding_sync import SlidingSyncBase +from tests.unittest import override_config + +logger = logging.getLogger(__name__) + + +# FIXME: This can be removed once we bump `SCHEMA_COMPAT_VERSION` and run the +# foreground update for +# `sliding_sync_joined_rooms`/`sliding_sync_membership_snapshots` (tracked by +# https://github.com/element-hq/synapse/issues/17623) +@parameterized_class( + ("use_new_tables",), + [ + (True,), + (False,), + ], + class_name_func=lambda cls, + num, + params_dict: f"{cls.__name__}_{'new' if params_dict['use_new_tables'] else 'fallback'}", +) +class SlidingSyncProfilesTestCase(SlidingSyncBase): + """Tests for the profile updates sliding sync extension""" + + servlets = [ + synapse.rest.admin.register_servlets, + knock.register_servlets, + login.register_servlets, + profile.register_servlets, + room.register_servlets, + sync.register_servlets, + ] + + def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None: + self.store = hs.get_datastores().main + self.profile_handler = self.hs.get_profile_handler() + self.user = self.register_user("user", "password") + self.tok = self.login("user", "password") + self.other_user = self.register_user("other_user", "password") + self.other_tok = self.login("other_user", "password") + self.joined_room = self.helper.create_room_as(self.user, tok=self.tok) + self.helper.join( + room=self.joined_room, user=self.other_user, tok=self.other_tok + ) + super().prepare(reactor, clock, hs) + + @parameterized.expand( + [ + True, + False, + ] + ) + def test_no_data_when_not_enabled(self, is_initial: bool) -> None: + """ + Test that no profile extension response is returned + if the feature is not enabled. + """ + if is_initial: + self.get_success( + self.profile_handler.set_field( + target_user=UserID.from_string(self.other_user), + requester=create_requester(self.other_user), + field_name="field", + new_value="value", + ) + ) + # Make an initial Sliding Sync request with the profiles extension enabled + sync_body = { + "lists": {}, + "extensions": { + "org.matrix.msc4262.profiles": { + "enabled": True, + "fields": ["field"], + }, + }, + } + response_body, from_token = self.do_sync(sync_body, tok=self.tok) + self.assertIsNone( + response_body["extensions"].get("org.matrix.msc4262.profiles") + ) + + if not is_initial: + self.get_success( + self.profile_handler.set_field( + target_user=UserID.from_string(self.other_user), + requester=create_requester(self.other_user), + field_name="field", + new_value="value", + ) + ) + # Make an incremental Sliding Sync request + response_body, _ = self.do_sync(sync_body, since=from_token, tok=self.tok) + + self.assertIsNone( + response_body["extensions"].get("org.matrix.msc4262.profiles") + ) + + @override_config({"include_profile_updates_in_sync": True}) + def test_no_data_initial_sync(self) -> None: + """ + Test that enabling the profiles extension works during an initial sync, + even if there is no-data. + """ + user1_id = self.register_user("user1", "pass") + user1_tok = self.login(user1_id, "pass") + + # Make an initial Sliding Sync request with the profiles extension enabled + sync_body = { + "lists": {}, + "extensions": { + "org.matrix.msc4262.profiles": { + "enabled": True, + "fields": ["field"], + }, + }, + } + response_body, _ = self.do_sync(sync_body, tok=user1_tok) + self.assertIsNone( + response_body["extensions"].get("org.matrix.msc4262.profiles") + ) + + @override_config({"include_profile_updates_in_sync": True}) + def test_no_data_incremental_sync(self) -> None: + """ + Test that enabling profiles extension works during an incremental sync, even + if there is no-data. + """ + user1_id = self.register_user("user1", "pass") + user1_tok = self.login(user1_id, "pass") + + sync_body = { + "lists": {}, + "extensions": { + "org.matrix.msc4262.profiles": { + "enabled": True, + "fields": ["field"], + } + }, + } + _, from_token = self.do_sync(sync_body, tok=user1_tok) + + # Make an incremental Sliding Sync request with the profiles extension enabled + response_body, _ = self.do_sync(sync_body, since=from_token, tok=user1_tok) + + self.assertIsNone( + response_body["extensions"].get("org.matrix.msc4262.profiles") + ) + + @parameterized.expand( + [ + True, + False, + ] + ) + @override_config({"include_profile_updates_in_sync": True}) + def test_updated_fields_are_sent(self, is_initial: bool) -> None: + """ + Test that profile extension response returns field updates + in incremental and initial sync. + """ + if is_initial: + self.get_success( + self.profile_handler.set_field( + target_user=UserID.from_string(self.other_user), + requester=create_requester(self.other_user), + field_name="field", + new_value="value", + ) + ) + # Make an initial Sliding Sync request with the profiles extension enabled + sync_body = { + "lists": {}, + "room_subscriptions": { + self.joined_room: { + "required_state": [], + "timeline_limit": 10, + }, + }, + "extensions": { + "org.matrix.msc4262.profiles": { + "enabled": True, + "fields": ["field"], + }, + }, + } + response_body, from_token = self.do_sync(sync_body, tok=self.tok) + if is_initial: + self.assertEqual( + response_body["extensions"]["org.matrix.msc4262.profiles"]["users"][ + "@other_user:test" + ], + { + "updated": { + "field": "value", + } + }, + ) + else: + self.get_success( + self.profile_handler.set_field( + target_user=UserID.from_string(self.other_user), + requester=create_requester(self.other_user), + field_name="field", + new_value="value", + ) + ) + # We don't include room subscriptions, as we want to see updates coming + # through even without room subscriptions + del sync_body["room_subscriptions"] + # Make an incremental Sliding Sync request + response_body, _ = self.do_sync(sync_body, since=from_token, tok=self.tok) + + self.assertEqual( + response_body["extensions"]["org.matrix.msc4262.profiles"]["users"][ + "@other_user:test" + ], + { + "updated": { + "field": "value", + } + }, + ) + + @override_config({"include_profile_updates_in_sync": True}) + def test_updated_field_then_deleted_does_not_error(self) -> None: + """ + Test that profile extension response does not crash if the user first + updates a field, then deletes it, and then the sync happens seeing both + the update and delete in the stream. + """ + self.get_success( + self.profile_handler.set_field( + target_user=UserID.from_string(self.other_user), + requester=create_requester(self.other_user), + field_name="field", + new_value="value", + ) + ) + # Make an initial Sliding Sync request with the profiles extension enabled + sync_body = { + "lists": {}, + "extensions": { + "org.matrix.msc4262.profiles": { + "enabled": True, + "fields": ["field"], + }, + }, + } + response_body, from_token = self.do_sync(sync_body, tok=self.tok) + + # Update field + self.get_success( + self.profile_handler.set_field( + target_user=UserID.from_string(self.other_user), + requester=create_requester(self.other_user), + field_name="field", + new_value="new value", + ) + ) + # Delete field + self.get_success( + self.profile_handler.delete_profile_field( + target_user=UserID.from_string(self.other_user), + requester=create_requester(self.other_user), + field_name="field", + ) + ) + + # Make an incremental Sliding Sync request + response_body, _ = self.do_sync(sync_body, since=from_token, tok=self.tok) + self.assertEqual( + response_body["extensions"]["org.matrix.msc4262.profiles"]["users"][ + "@other_user:test" + ], + { + "removed": [ + "field", + ], + }, + ) + + @parameterized.expand( + [ + True, + False, + ] + ) + @override_config({"include_profile_updates_in_sync": True}) + def test_updated_fields_are_not_sent_if_not_requested( + self, is_initial: bool + ) -> None: + """ + Test that profile extension response doesn't return field updates we didn't + request in initial and incremental sync. + """ + if is_initial: + self.get_success( + self.profile_handler.set_field( + target_user=UserID.from_string(self.other_user), + requester=create_requester(self.other_user), + field_name="anotherfield", + new_value="value", + ) + ) + # Make an initial Sliding Sync request with the profiles extension enabled + sync_body = { + "lists": {}, + "extensions": { + "org.matrix.msc4262.profiles": { + "enabled": True, + "fields": ["field"], + }, + }, + } + response_body, from_token = self.do_sync(sync_body, tok=self.tok) + if is_initial: + # Nothing returned since we didn't ask for the updated field + self.assertIsNone( + response_body["extensions"].get("org.matrix.msc4262.profiles") + ) + + if not is_initial: + self.get_success( + self.profile_handler.set_field( + target_user=UserID.from_string(self.other_user), + requester=create_requester(self.other_user), + field_name="anotherfield", + new_value="value", + ) + ) + # Make an incremental Sliding Sync request + response_body, _ = self.do_sync(sync_body, since=from_token, tok=self.tok) + # Nothing returned since we didn't ask for the updated field + self.assertIsNone( + response_body["extensions"].get("org.matrix.msc4262.profiles") + ) + + @parameterized.expand( + [ + True, + False, + ] + ) + @override_config({"include_profile_updates_in_sync": True}) + def test_updated_fields_are_not_included_if_not_in_requested_rooms( + self, is_initial: bool + ) -> None: + """ + Test that profile extension response respects the room subscriptions, by: + * for initial sync returning updates for only those users in the given rooms + * for incremental sync returning all updates in shared rooms + """ + new_room = self.helper.create_room_as(self.user, tok=self.tok) + if is_initial: + self.get_success( + self.profile_handler.set_field( + target_user=UserID.from_string(self.other_user), + requester=create_requester(self.other_user), + field_name="field", + new_value="value", + ) + ) + # Make an initial Sliding Sync request with the profiles extension enabled + sync_body = { + "lists": {}, + "room_subscriptions": { + new_room: { + "required_state": [], + "timeline_limit": 10, + }, + }, + "extensions": { + "org.matrix.msc4262.profiles": { + "enabled": True, + "fields": ["field"], + }, + }, + } + response_body, from_token = self.do_sync(sync_body, tok=self.tok) + if is_initial: + # Nothing returned since even though user and other_user share a room, + # we didn't ask for that room. + self.assertIsNone( + response_body["extensions"].get("org.matrix.msc4262.profiles") + ) + + if not is_initial: + self.get_success( + self.profile_handler.set_field( + target_user=UserID.from_string(self.other_user), + requester=create_requester(self.other_user), + field_name="field", + new_value="value", + ) + ) + # Make an incremental Sliding Sync request + response_body, _ = self.do_sync(sync_body, since=from_token, tok=self.tok) + # Even though we only asked for a room other_user is not in, + # since these users share a room, updates are always sent via incremental + # sync. + self.assertEqual( + response_body["extensions"]["org.matrix.msc4262.profiles"]["users"][ + "@other_user:test" + ], + { + "updated": { + "field": "value", + } + }, + ) + + @parameterized.expand( + [ + True, + False, + ] + ) + @override_config({"include_profile_updates_in_sync": True}) + def test_all_fields_returned_if_no_fields_specified(self, is_initial: bool) -> None: + """ + Test that profile extension response returns all profile fields if we didn't + request any particular fields in initial and incremental sync. + """ + if is_initial: + self.get_success( + self.profile_handler.set_field( + target_user=UserID.from_string(self.other_user), + requester=create_requester(self.other_user), + field_name="field", + new_value="value", + ) + ) + # Make an initial Sliding Sync request with the profiles extension enabled + sync_body = { + "lists": {}, + # We need to ensure a room is included to get things back in initial sync + "room_subscriptions": { + self.joined_room: { + "required_state": [], + "timeline_limit": 10, + }, + }, + "extensions": { + "org.matrix.msc4262.profiles": { + "enabled": True, + }, + }, + } + response_body, from_token = self.do_sync(sync_body, tok=self.tok) + if is_initial: + # As this is an initial sync, we get all profile fields + self.assertEqual( + response_body["extensions"]["org.matrix.msc4262.profiles"]["users"][ + "@other_user:test" + ], + { + "updated": { + "avatar_url": None, + "displayname": "other_user", + "field": "value", + } + }, + ) + + else: + self.get_success( + self.profile_handler.set_field( + target_user=UserID.from_string(self.other_user), + requester=create_requester(self.other_user), + field_name="field", + new_value="value", + ) + ) + # We don't include room subscriptions, as we want to see updates coming + # through even without room subscriptions + del sync_body["room_subscriptions"] + # Make an incremental Sliding Sync request + response_body, _ = self.do_sync(sync_body, since=from_token, tok=self.tok) + # As this is an incremental sync, we only get actual updates back + self.assertEqual( + response_body["extensions"]["org.matrix.msc4262.profiles"]["users"][ + "@other_user:test" + ], + { + "updated": { + "field": "value", + } + }, + ) + + @parameterized.expand( + [ + True, + False, + ] + ) + @override_config({"include_profile_updates_in_sync": True}) + def test_null_profile_returned_if_user_left_all_rooms( + self, + request_fields: bool, + ) -> None: + """ + Test that profile extension response returns a null for the user in + incremental sync. + """ + # Make an initial Sliding Sync request with the profiles extension enabled + profiles_config: dict = { + "enabled": True, + } + if request_fields: + profiles_config["fields"] = ["field"] + sync_body = { + "lists": {}, + "extensions": { + "org.matrix.msc4262.profiles": profiles_config, + }, + } + response_body, from_token = self.do_sync(sync_body, tok=self.tok) + + self.helper.leave(self.joined_room, self.other_user, tok=self.other_tok) + + # Make an incremental Sliding Sync request + response_body, _ = self.do_sync(sync_body, since=from_token, tok=self.tok) + # We should see a null profile + self.assertIsNone( + response_body["extensions"]["org.matrix.msc4262.profiles"]["users"][ + "@other_user:test" + ], + ) + + @override_config({"include_profile_updates_in_sync": True}) + def test_profile_returned_if_user_left_then_rejoined(self) -> None: + """ + Test that the profile extension response returns a profile, rather than a + null, for a user that left their last shared room and then rejoined it + within the same incremental sync window. + """ + sync_body = { + "lists": {}, + "extensions": { + "org.matrix.msc4262.profiles": {"enabled": True}, + }, + } + response_body, from_token = self.do_sync(sync_body, tok=self.tok) + + self.helper.leave(self.joined_room, self.other_user, tok=self.other_tok) + self.helper.join(self.joined_room, self.other_user, tok=self.other_tok) + + # Make an incremental Sliding Sync request + response_body, _ = self.do_sync(sync_body, since=from_token, tok=self.tok) + # The rejoin overrides the leave, so we should see the full profile rather + # than a null profile. + self.assertEqual( + response_body["extensions"]["org.matrix.msc4262.profiles"]["users"][ + "@other_user:test" + ], + { + "updated": { + "displayname": "other_user", + # FIXME: This shouldn't be returned, but currently is + "avatar_url": None, + } + }, + ) + + @parameterized.expand( + [ + True, + False, + ] + ) + @override_config({"include_profile_updates_in_sync": True}) + def test_all_fields_returned_in_incremental_non_lazy_sync_if_someone_joined( + self, request_fields: bool + ) -> None: + """ + Test that profile extension response returns all profile fields in + incremental non-lazy sync, if someone joined the room.. + """ + # Make an initial Sliding Sync request with the profiles extension enabled + profiles_config: dict = { + "enabled": True, + } + if request_fields: + profiles_config["fields"] = ["displayname"] + sync_body = { + "lists": {}, + "extensions": { + "org.matrix.msc4262.profiles": profiles_config, + }, + } + response_body, from_token = self.do_sync(sync_body, tok=self.tok) + + third_user = self.register_user("third_user", "third_user") + third_tok = self.login(third_user, "third_user") + self.helper.join(self.joined_room, third_user, tok=third_tok) + + # Make an incremental Sliding Sync request + response_body, _ = self.do_sync(sync_body, since=from_token, tok=self.tok) + + expectation = { + "updated": { + "avatar_url": None, + "displayname": "third_user", + } + } + if request_fields: + expectation = { + "updated": { + "displayname": "third_user", + }, + } + self.assertEqual( + response_body["extensions"]["org.matrix.msc4262.profiles"]["users"][ + "@third_user:test" + ], + expectation, + ) + + @parameterized.expand(["displayname", "avatar_url", "someotherfield"]) + @override_config({"include_profile_updates_in_sync": True}) + def test_removed_fields_get_sent_down_as_removed( + self, + field_name: str, + ) -> None: + """ + Test that we deliver clear/removed fields in the "removed" key in the response. + """ + self.get_success( + self.profile_handler.set_field( + target_user=UserID.from_string(self.other_user), + requester=create_requester(self.other_user), + field_name=field_name, + new_value="value", + ) + ) + # Make an initial Sliding Sync request with the profiles extension enabled + sync_body = { + "lists": {}, + "extensions": { + "org.matrix.msc4262.profiles": { + "enabled": True, + }, + }, + } + response_body, from_token = self.do_sync(sync_body, tok=self.tok) + + # Delete the field + if field_name in (ProfileFields.DISPLAYNAME, ProfileFields.AVATAR_URL): + self.get_success( + self.profile_handler.set_profile_field( + target_user=UserID.from_string(self.other_user), + requester=create_requester(self.other_user), + field_name=field_name, + new_value=None, + ) + ) + else: + self.get_success( + self.profile_handler.delete_profile_field( + target_user=UserID.from_string(self.other_user), + requester=create_requester(self.other_user), + field_name=field_name, + ) + ) + + # Make an incremental Sliding Sync request + response_body, _ = self.do_sync(sync_body, since=from_token, tok=self.tok) + # We should see the removed field + self.assertEqual( + response_body["extensions"]["org.matrix.msc4262.profiles"]["users"][ + "@other_user:test" + ], + { + "removed": [ + field_name, + ], + }, + ) + + @override_config({"include_profile_updates_in_sync": True}) + def test_updated_key_only_present_if_updates(self) -> None: + """ + > The updated field SHOULD only be present if there are changes to existing fields on a user's profile. + """ + self.skipTest("Not yet implemented") + + @override_config({"include_profile_updates_in_sync": True}) + def test_rooms_subset_changing_includes_full_profile(self) -> None: + """ + > When a room enters this subset in this connection for the first time, all requested + > fields from profiles of users in that room MAY be sent down. This gives the client + > a base set of information for which future field updates can be applied on top of. + > The homeserver MAY omit some fields and profiles if it believes that the client has + > already received them, likewise repeat profiles MAY be sent down based on homeserver + > implementation. + """ + self.skipTest("Not yet implemented") + + @override_config({"include_profile_updates_in_sync": True}) + def test_fields_subset_changing_sends_down_field_even_if_not_changed(self) -> None: + """ + > Finally, if the list of fields expands to cover a new field ID, those fields should + > be sent down for all users that are within the current room subset. Future incremental + > updates will then include changes to this field. + """ + self.skipTest("Not yet implemented") + + @parameterized.expand( + [ + [True, True], + [True, False], + [False, False], + [False, True], + ] + ) + @override_config({"include_profile_updates_in_sync": True}) + def test_lazy_loading_sends_down_full_profile_if_events_in_timeline( + self, + is_initial: bool, + use_room_subsciptions: bool, + ) -> None: + """ + Test that when lazy loading, only those members who have events in + the timeline get their profiles sent down in the sync response, for + rooms configured with lazy loading. + + Rooms without lazy loading should include all the members in initial sync, + none in incremental. + """ + # Create three users to fill the heroes + # Our heroes will thus be user, other_user and the three heroes here. + for i in range(3): + user = self.register_user(f"hero{i}", "password") + tok = self.login(f"hero{i}", "password") + self.helper.join(self.joined_room, user=user, tok=tok) + third_user = self.register_user("third_user", "password") + third_tok = self.login("third_user", "password") + fourth_user = self.register_user("fourth_user", "password") + fourth_tok = self.login("fourth_user", "password") + fifth_user = self.register_user("fifth_user", "password") + fifth_tok = self.login("fifth_user", "password") + self.helper.join( + room=self.joined_room, + user=third_user, + tok=third_tok, + ) + self.helper.join( + room=self.joined_room, + user=fifth_user, + tok=fifth_tok, + ) + new_room = self.helper.create_room_as(self.user, tok=self.tok) + self.helper.join( + room=new_room, + user=fourth_user, + tok=fourth_tok, + ) + if is_initial: + self.helper.send_messages( + room_id=self.joined_room, num_events=1, tok=self.other_tok + ) + self.helper.send_messages( + room_id=self.joined_room, num_events=10, tok=third_tok + ) + # Make an initial Sliding Sync request with the profiles extension enabled + sync_body: dict[str, dict] = { + "extensions": { + "org.matrix.msc4262.profiles": { + "enabled": True, + }, + }, + } + if use_room_subsciptions: + sync_body["room_subscriptions"] = { + self.joined_room: { + "required_state": [], + "timeline_limit": 10, + }, + new_room: { + "required_state": [], + "timeline_limit": 10, + }, + } + else: + sync_body["lists"] = { + "foo-list": { + "ranges": [[0, 0]], + "required_state": [], + "timeline_limit": 10, + } + } + # We also need to specifically request the non-lazy room, otherwise + # our test to see if non-lazy members are also included will fail + sync_body["room_subscriptions"] = { + new_room: { + "required_state": [], + "timeline_limit": 10, + }, + } + if is_initial: + if use_room_subsciptions: + sync_body["room_subscriptions"][self.joined_room]["required_state"] = [ + ["m.room.member", "$LAZY"], + # Don't request other state as we're checking timeline events + # ["*", "*"], + ] + else: + sync_body["lists"]["foo-list"]["required_state"] = [ + ["m.room.member", "$LAZY"], + # Don't request other state as we're checking timeline events + # ["*", "*"], + ] + response_body, from_token = self.do_sync(sync_body, tok=self.tok) + if is_initial: + self.assertIsNotNone( + response_body["extensions"].get("org.matrix.msc4262.profiles") + ) + # Other user is a hero so should be included. + self.assertIsNotNone( + response_body["extensions"]["org.matrix.msc4262.profiles"]["users"].get( + "@other_user:test" + ) + ) + # Third user has events in the timeline, so should be here. + self.assertIsNotNone( + response_body["extensions"]["org.matrix.msc4262.profiles"]["users"].get( + "@third_user:test" + ) + ) + # Initial sync always includes ourselves + self.assertIsNotNone( + response_body["extensions"]["org.matrix.msc4262.profiles"]["users"].get( + "@user:test" + ) + ) + # Fourth user is a member of a non-lazy configured room, so should be here. + self.assertIsNotNone( + response_body["extensions"]["org.matrix.msc4262.profiles"]["users"].get( + "@fourth_user:test" + ) + ) + # Fifth user should be filtered out as they have no events in the room. + self.assertIsNone( + response_body["extensions"]["org.matrix.msc4262.profiles"]["users"].get( + "@fifth_user:test" + ) + ) + + if not is_initial: + self.helper.send_messages( + room_id=self.joined_room, num_events=1, tok=self.other_tok + ) + self.helper.send_messages( + room_id=self.joined_room, num_events=10, tok=third_tok + ) + if use_room_subsciptions: + sync_body["room_subscriptions"][self.joined_room]["required_state"] = [ + ["m.room.member", "$LAZY"], + # Don't request other state as we're checking timeline events + # ["*", "*"], + ] + else: + sync_body["lists"]["foo-list"]["required_state"] = [ + ["m.room.member", "$LAZY"], + # Don't request other state as we're checking timeline events + # ["*", "*"], + ] + # Make an incremental Sliding Sync request + response_body, _ = self.do_sync(sync_body, since=from_token, tok=self.tok) + self.assertIsNotNone( + response_body["extensions"].get("org.matrix.msc4262.profiles") + ) + # TODO check this if it's expected that heroes come down differently + # depending on if using a room subscription or a list + if use_room_subsciptions: + # Other user should be filtered out as heroes don't come down + # in incremental sync in the same way as initial sync, if the + # room is included via a room subscription. + self.assertIsNone( + response_body["extensions"]["org.matrix.msc4262.profiles"][ + "users" + ].get("@other_user:test") + ) + else: + # Other user should be included as heroes do come down + # in incremental sync in the same way as initial sync when the + # room is included in a list + self.assertIsNotNone( + response_body["extensions"]["org.matrix.msc4262.profiles"][ + "users" + ].get("@other_user:test") + ) + # Third user has events in the timeline, so should be here. + self.assertIsNotNone( + response_body["extensions"]["org.matrix.msc4262.profiles"]["users"].get( + "@third_user:test" + ) + ) + # We are not included ourselves in incremental sync without updates. + self.assertIsNone( + response_body["extensions"]["org.matrix.msc4262.profiles"]["users"].get( + "@user:test" + ) + ) + # Fourth user is a member of a non-lazy configured room, but had no updates, + # so shouldn't be here. + self.assertIsNone( + response_body["extensions"]["org.matrix.msc4262.profiles"]["users"].get( + "@fourth_user:test" + ) + ) + # Fifth user should be excluded as they have no events. + self.assertIsNone( + response_body["extensions"]["org.matrix.msc4262.profiles"]["users"].get( + "@fifth_user:test" + ) + ) + + @parameterized.expand( + [ + True, + False, + ] + ) + @override_config({"include_profile_updates_in_sync": True}) + def test_lazy_loading_sends_full_profile_even_if_no_events_if_otherwise_included( + self, + use_room_subsciptions: bool, + ) -> None: + """ + Test that when lazy loading, if a user is in both a lazy loading room + and a non-lazy configured room, even if there are no events in the timeline, + their profile is sent down. + + This test only makes sense for initial sync, as for incremental we would + not expect to see users without timeline events if they had no profile updates. + """ + # Create some users to fill the heroes so they don't pollute the test. + for i in range(3): + user = self.register_user(f"hero{i}", "password") + tok = self.login(f"hero{i}", "password") + self.helper.join(self.joined_room, user=user, tok=tok) + new_user = self.register_user("new_user", password="password") + new_tok = self.login("new_user", password="password") + new_room = self.helper.create_room_as(self.user, tok=self.tok) + self.helper.join( + room=self.joined_room, + user=new_user, + tok=new_tok, + ) + self.helper.join( + room=new_room, + user=new_user, + tok=new_tok, + ) + # Make an initial Sliding Sync request with the profiles extension enabled + sync_body: dict[str, dict] = { + "extensions": { + "org.matrix.msc4262.profiles": { + "enabled": True, + }, + }, + } + if use_room_subsciptions: + sync_body["room_subscriptions"] = { + self.joined_room: { + "required_state": [ + ["m.room.member", "$LAZY"], + # Don't request any events for this room + # ["*", "*"], + ], + # Force zero timeline events in the response, otherwise + # this test wont work, as the timeline_events in the room + # response will contain all the create/join etc events too. + "timeline_limit": 0, + }, + new_room: { + "required_state": [], + "timeline_limit": 10, + }, + } + else: + sync_body["lists"] = { + "foo-list": { + "ranges": [[0, 0]], + "required_state": [ + ["m.room.member", "$LAZY"], + # Don't request any events for this room + # ["*", "*"], + ], + # Force zero timeline events in the response, otherwise + # this test wont work, as the timeline_events in the room + # response will contain all the create/join etc events too. + "timeline_limit": 0, + } + } + # We also need to specifically request the non-lazy room, otherwise + # our test to see if non-lazy members are also included will fail + sync_body["room_subscriptions"] = { + new_room: { + "required_state": [], + "timeline_limit": 10, + }, + } + response_body, from_token = self.do_sync(sync_body, tok=self.tok) + self.assertIsNotNone( + response_body["extensions"].get("org.matrix.msc4262.profiles") + ) + # New user should be included as they are in a non-lazy room too, + # even though the lazy configured room had no events. + self.assertIsNotNone( + response_body["extensions"]["org.matrix.msc4262.profiles"]["users"].get( + "@new_user:test" + ) + ) + # Initial sync always includes ourselves + self.assertIsNotNone( + response_body["extensions"]["org.matrix.msc4262.profiles"]["users"].get( + "@user:test" + ) + ) + + @parameterized.expand( + [ + True, + False, + ] + ) + @override_config({"include_profile_updates_in_sync": True}) + def test_lazy_loading_sends_full_profile_for_required_state_member_events( + self, + use_room_subsciptions: bool, + ) -> None: + """ + Test that when lazy loading for lazy rooms, even without timeline events, + we get profiles for users who have membership events in required_state. + + This test only makes sense for initial sync, as for incremental this would + happen via the `ProfileUpdateAction.JOINED_ROOM` events. + """ + # Create some users to fill the heroes so they don't pollute the test. + for i in range(3): + user = self.register_user(f"hero{i}", "password") + tok = self.login(f"hero{i}", "password") + self.helper.join(self.joined_room, user=user, tok=tok) + new_user = self.register_user("new_user", password="password") + new_tok = self.login("new_user", password="password") + self.helper.join( + room=self.joined_room, + user=new_user, + tok=new_tok, + ) + # Make an initial Sliding Sync request with the profiles extension enabled + sync_body: dict[str, dict] = { + "extensions": { + "org.matrix.msc4262.profiles": { + "enabled": True, + }, + }, + } + if use_room_subsciptions: + sync_body["room_subscriptions"] = { + self.joined_room: { + "required_state": [ + ["m.room.member", "$LAZY"], + ["*", "*"], + ], + # Force zero timeline events in the response, otherwise + # this test wont work, as the timeline_events in the room + # response will contain all the create/join etc events too. + "timeline_limit": 0, + }, + } + else: + sync_body["lists"] = { + "foo-list": { + "ranges": [[0, 0]], + "required_state": [ + ["m.room.member", "$LAZY"], + ["*", "*"], + ], + # Force zero timeline events in the response, otherwise + # this test wont work, as the timeline_events in the room + # response will contain all the create/join etc events too. + "timeline_limit": 0, + } + } + response_body, from_token = self.do_sync(sync_body, tok=self.tok) + self.assertIsNotNone( + response_body["extensions"].get("org.matrix.msc4262.profiles") + ) + # New user should be included as they joined the room and as such + # have membership events in required_state. + self.assertIsNotNone( + response_body["extensions"]["org.matrix.msc4262.profiles"]["users"].get( + "@new_user:test" + ) + ) + # Initial sync always includes ourselves + self.assertIsNotNone( + response_body["extensions"]["org.matrix.msc4262.profiles"]["users"].get( + "@user:test" + ) + ) + + @parameterized.expand( + [ + True, + False, + ] + ) + @override_config({"include_profile_updates_in_sync": True}) + def test_lazy_loading_sends_full_profile_for_heroes( + self, + use_room_subsciptions: bool, + ) -> None: + """ + Test that when lazy loading for lazy rooms, even without timeline events or + required_state, we get profiles for room heroes. + + This test must ensure heroes don't get included in timeline_events + or required_state. + + This test only makes sense for initial sync, as for incremental sync + Synapse doesn't generate a room response without requesting state or + timeline events, thus no heroes either. + """ + # Create some users to fill the heroes + for i in range(4): + user = self.register_user(f"hero{i}", "password") + tok = self.login(f"hero{i}", "password") + self.helper.join(self.joined_room, user=user, tok=tok) + not_hero = self.register_user("not_hero", "password") + not_hero_tok = self.login("not_hero", "password") + self.helper.join(self.joined_room, user=not_hero, tok=not_hero_tok) + + # Make an initial Sliding Sync request with the profiles extension enabled + sync_body: dict[str, dict] = { + "extensions": { + "org.matrix.msc4262.profiles": { + "enabled": True, + }, + }, + } + if use_room_subsciptions: + sync_body["room_subscriptions"] = { + self.joined_room: { + "required_state": [ + ["m.room.member", "$LAZY"], + # Don't request any events for this room + # ["*", "*"], + ], + # Force zero timeline events in the response, otherwise + # this test wont work, as the timeline_events in the room + # response will contain all the create/join etc events too. + "timeline_limit": 0, + }, + } + else: + sync_body["lists"] = { + "foo-list": { + "ranges": [[0, 0]], + "required_state": [ + ["m.room.member", "$LAZY"], + # Don't request any events for this room + # ["*", "*"], + ], + # Force zero timeline events in the response, otherwise + # this test wont work, as the timeline_events in the room + # response will contain all the create/join etc events too. + "timeline_limit": 0, + } + } + response_body, from_token = self.do_sync(sync_body, tok=self.tok) + self.assertIsNotNone( + response_body["extensions"].get("org.matrix.msc4262.profiles") + ) + # Other user should be included as they are a room hero + self.assertIsNotNone( + response_body["extensions"]["org.matrix.msc4262.profiles"]["users"].get( + "@other_user:test" + ) + ) + # Not hero user should be excluded as they're not a hero + self.assertIsNone( + response_body["extensions"]["org.matrix.msc4262.profiles"]["users"].get( + "@not_hero:test" + ) + ) + # Initial sync always includes ourselves + self.assertIsNotNone( + response_body["extensions"]["org.matrix.msc4262.profiles"]["users"].get( + "@user:test" + ) + ) + + @override_config({"include_profile_updates_in_sync": True}) + def test_repeat_of_sync_correctly_includes_profile_information_again(self) -> None: + """ + > Homeservers should only consider a profile field update "accepted" by a client + > once the client returns with a new /sync request with the next /sync token, + > NOT just after sending down the profile update. The client may never receive + > response due to network conditions, or a bug in the client implementation. + """ + self.skipTest("Not yet implemented") diff --git a/tests/rest/client/sliding_sync/test_rooms_invites.py b/tests/rest/client/sliding_sync/test_rooms_invites.py index 5a463303dd..85628833f5 100644 --- a/tests/rest/client/sliding_sync/test_rooms_invites.py +++ b/tests/rest/client/sliding_sync/test_rooms_invites.py @@ -143,7 +143,7 @@ class SlidingSyncRoomsInvitesTestCase(SlidingSyncBase): response_body["rooms"][room_id1]["invite_state"], [ { - "content": {"creator": user2_id, "room_version": "10"}, + "content": {"room_version": "11"}, "sender": user2_id, "state_key": "", "type": "m.room.create", @@ -253,7 +253,7 @@ class SlidingSyncRoomsInvitesTestCase(SlidingSyncBase): response_body["rooms"][room_id1]["invite_state"], [ { - "content": {"creator": user2_id, "room_version": "10"}, + "content": {"room_version": "11"}, "sender": user2_id, "state_key": "", "type": "m.room.create", @@ -374,7 +374,7 @@ class SlidingSyncRoomsInvitesTestCase(SlidingSyncBase): response_body["rooms"][room_id1]["invite_state"], [ { - "content": {"creator": user2_id, "room_version": "10"}, + "content": {"room_version": "11"}, "sender": user2_id, "state_key": "", "type": "m.room.create", @@ -500,7 +500,7 @@ class SlidingSyncRoomsInvitesTestCase(SlidingSyncBase): response_body["rooms"][room_id1]["invite_state"], [ { - "content": {"creator": user2_id, "room_version": "10"}, + "content": {"room_version": "11"}, "sender": user2_id, "state_key": "", "type": "m.room.create", diff --git a/tests/rest/client/sliding_sync/test_rooms_meta.py b/tests/rest/client/sliding_sync/test_rooms_meta.py index b1b771ef84..93fdfa59f9 100644 --- a/tests/rest/client/sliding_sync/test_rooms_meta.py +++ b/tests/rest/client/sliding_sync/test_rooms_meta.py @@ -13,6 +13,7 @@ # import logging from typing import Any +from unittest.mock import patch from parameterized import parameterized, parameterized_class @@ -26,7 +27,11 @@ from synapse.server import HomeServer from synapse.util.clock import Clock from tests.rest.client.sliding_sync.test_sliding_sync import SlidingSyncBase -from tests.test_utils.event_injection import create_event +from tests.test_utils.event_injection import ( + create_event, + inject_event, + inject_member_event, +) logger = logging.getLogger(__name__) @@ -1407,3 +1412,239 @@ class SlidingSyncRoomsMetaTestCase(SlidingSyncBase): } } response_body, _ = self.do_sync(sync_body, tok=user1_tok) + + @parameterized.expand(((True,), (None,), ({"a": "dict"},), (["a list"],), (42,))) + def test_rooms_meta_non_string_name(self, non_string_name: object) -> None: + """ + Test that when the room name is not a string, it gets + treated the same as if there is no room name set; + the `name` field is omitted and `heroes` are populated instead. + """ + user1_id = self.register_user("user1", "pass") + user1_tok = self.login(user1_id, "pass") + + # For heroes to be emitted, we need a second user + user2_id = self.register_user("user2", "pass") + user2_tok = self.login(user2_id, "pass") + + room_id = self.helper.create_room_as( + user1_id, + tok=user1_tok, + ) + self.helper.join(room_id, user2_id, tok=user2_tok) + + # Set the room name to a non-string + # Need to patch out our client-sent event checks to do this. + # (We don't apply these same out-of-spec checks to events + # received through federation. + # Could have instead set up the test to receive the event over federation.) + with patch("synapse.events.validator.EventValidator.validate_new"): + self.get_success( + inject_event( + self.hs, + room_id=room_id, + sender=user1_id, + type=EventTypes.Name, + state_key="", + content={"name": non_string_name}, + ) + ) + + sync_body = { + "lists": { + "wombat": { + "ranges": [[0, 1]], + "required_state": [], + "timeline_limit": 0, + } + } + } + response_body, _ = self.do_sync(sync_body, tok=user1_tok) + + # Sanity check that the room is included with an initial snapshot + self.assertEqual(response_body["rooms"][room_id]["initial"], True) + + # The name should be omitted (non-string value treated as unset). + self.assertNotIn( + "name", + response_body["rooms"][room_id], + response_body["rooms"][room_id], + ) + + # Since there is no name, heroes should be populated. + self.assertEqual( + response_body["rooms"][room_id]["heroes"], + [{"displayname": "user2", "user_id": "@user2:test"}], + ) + + @parameterized.expand(((True,), (None,), ({"a": "dict"},), (["a list"],), (42,))) + def test_rooms_meta_non_string_avatar(self, non_string_avatar: str) -> None: + """ + Test that when the room avatar is not a string, it gets + treated the same as if there is no room avatar set; + the `avatar` field is omitted. + """ + user1_id = self.register_user("user1", "pass") + user1_tok = self.login(user1_id, "pass") + + room_id = self.helper.create_room_as( + user1_id, + tok=user1_tok, + ) + + # Set the room avatar to a dict (non-string) instead of a URL string. + # Need to patch out our client-sent event checks to do this + # (We don't apply these same out-of-spec checks to events + # received through federation. + # Could have instead set up the test to receive the event over federation.) + with patch("synapse.events.validator.EventValidator.validate_new"): + self.get_success( + inject_event( + self.hs, + room_id=room_id, + sender=user1_id, + type=EventTypes.RoomAvatar, + state_key="", + content={"url": non_string_avatar}, + ) + ) + + sync_body = { + "lists": { + "wombat": { + "ranges": [[0, 1]], + "required_state": [], + "timeline_limit": 0, + } + } + } + response_body, _ = self.do_sync(sync_body, tok=user1_tok) + + # Sanity check that the room is included with an initial snapshot + self.assertEqual(response_body["rooms"][room_id]["initial"], True) + + # The avatar should be omitted (non-string value treated as unset). + self.assertNotIn( + "avatar", + response_body["rooms"][room_id], + response_body["rooms"][room_id], + ) + + @parameterized.expand(((True,), (None,), ({"a": "dict"},), (["a list"],), (42,))) + def test_rooms_meta_heroes_non_string_displayname( + self, non_string_name: str + ) -> None: + """ + Test that when a hero's displayname is not a string, it gets + treated the same as if there is no displayname set: + the `displayname` field is omitted from the hero entry. + """ + user1_id = self.register_user("user1", "pass") + user1_tok = self.login(user1_id, "pass") + user2_id = self.register_user("user2", "pass") + + # Create a room with no name so heroes are populated. + room_id = self.helper.create_room_as( + user1_id, + tok=user1_tok, + ) + + # Inject a membership event for user2 with a non-string displayname. + self.get_success( + inject_member_event( + self.hs, + room_id, + sender=user2_id, + target=user2_id, + membership=Membership.JOIN, + extra_content={ + "displayname": non_string_name, + "avatar_url": "mxc://example.org/a-real-mxc", + }, + ) + ) + + sync_body = { + "lists": { + "wombat": { + "ranges": [[0, 1]], + "required_state": [], + "timeline_limit": 0, + } + } + } + response_body, _ = self.do_sync(sync_body, tok=user1_tok) + + # Sanity check that the room is included with an initial snapshot + self.assertEqual(response_body["rooms"][room_id]["initial"], True) + self.assertNotIn( + "name", response_body["rooms"][room_id], response_body["rooms"][room_id] + ) + + # user2 should be in the heroes list, but without a displayname + self.assertEqual( + response_body["rooms"][room_id]["heroes"], + [ + { + "avatar_url": "mxc://example.org/a-real-mxc", + "user_id": "@user2:test", + } + ], + ) + + @parameterized.expand(((True,), (None,), ({"a": "dict"},), (["a list"],), (42,))) + def test_rooms_meta_heroes_non_string_avatar_url( + self, non_string_avatar: str + ) -> None: + """ + Test that when a hero's avatar URL is not a string, it gets + treated the same as if there is no avatar URL set: + the `avatar_url` field is omitted from the hero entry. + """ + user1_id = self.register_user("user1", "pass") + user1_tok = self.login(user1_id, "pass") + user2_id = self.register_user("user2", "pass") + user2_tok = self.login(user2_id, "pass") + + # Create a room with no name so heroes are populated. + room_id = self.helper.create_room_as( + user2_id, + tok=user2_tok, + ) + self.helper.join(room_id, user1_id, tok=user1_tok) + + # Inject a membership event for user2 with a non-string avatar_url. + self.get_success( + inject_member_event( + self.hs, + room_id, + sender=user2_id, + target=user2_id, + membership=Membership.JOIN, + extra_content={ + "displayname": "second user", + "avatar_url": non_string_avatar, + }, + ) + ) + + sync_body = { + "lists": { + "wombat": { + "ranges": [[0, 1]], + "required_state": [], + "timeline_limit": 0, + } + } + } + response_body, _ = self.do_sync(sync_body, tok=user1_tok) + + # Sanity check that the room is included with an initial snapshot + self.assertEqual(response_body["rooms"][room_id]["initial"], True) + self.assertNotIn("name", response_body["rooms"][room_id]) + + # user2 should be in the heroes list, but without an avatar + self.assertEqual( + response_body["rooms"][room_id]["heroes"], + [{"displayname": "second user", "user_id": "@user2:test"}], + ) diff --git a/tests/rest/client/test_account.py b/tests/rest/client/test_account.py index 158d7f33f0..074186b1e6 100644 --- a/tests/rest/client/test_account.py +++ b/tests/rest/client/test_account.py @@ -30,7 +30,7 @@ from twisted.internet.interfaces import IReactorTCP from twisted.internet.testing import MemoryReactor import synapse.rest.admin -from synapse.api.constants import LoginType, Membership +from synapse.api.constants import LoginType, Membership, ProfileFields from synapse.api.errors import Codes, HttpResponseException, SynapseError from synapse.appservice import ApplicationService from synapse.rest import admin @@ -42,7 +42,7 @@ from synapse.types import JsonDict, UserID, create_requester from synapse.util.clock import Clock from tests import unittest -from tests.server import FakeSite, make_request +from tests.server import FakeChannel, FakeSite, make_request from tests.unittest import override_config @@ -325,13 +325,8 @@ class PasswordResetTestCase(unittest.HomeserverTestCase): email = "test@example.com" client_secret = "foobar" - session_id = self._request_token( - email, - client_secret, - # The endpoint intentionally adds up to 1000ms of jitter to avoid - # leaking whether the email address is bound to an account. - timeout_ms=3000, - ) + + session_id = self._request_token(email, client_secret) self.assertIsNotNone(session_id) @@ -364,24 +359,47 @@ class PasswordResetTestCase(unittest.HomeserverTestCase): self._validate_token(link, next_link) + def test_password_reset_invalid_email(self) -> None: + """A malformed email address is reported with M_INVALID_PARAM, as on + /account/3pid/email/requestToken (the two endpoints share the request + body model). + """ + channel = self.make_request( + "POST", + b"account/password/email/requestToken", + { + "client_secret": "foobar", + "email": "address-without-at.bar", + "send_attempt": 1, + }, + ) + self.assertEqual( + HTTPStatus.BAD_REQUEST, channel.code, msg=channel.result["body"] + ) + self.assertEqual(Codes.INVALID_PARAM, channel.json_body["errcode"]) + self.assertIn("Unable to parse email address", channel.json_body["error"]) + def _request_token( self, email: str, client_secret: str, ip: str = "127.0.0.1", next_link: str | None = None, - timeout_ms: int = 1000, ) -> str: body = {"client_secret": client_secret, "email": email, "send_attempt": 1} if next_link is not None: body["next_link"] = next_link + channel = self.make_request( "POST", b"account/password/email/requestToken", body, client_ip=ip, - timeout_ms=timeout_ms, + await_result=False, ) + # Note: The endpoint intentionally adds up to 1000ms of jitter to avoid + # leaking whether the email address is bound to an account. + channel.await_result(timeout_ms=1000) if channel.code != 200: raise HttpResponseException( @@ -522,13 +540,19 @@ class DeactivateTestCase(unittest.HomeserverTestCase): # Set some profile data that can be checked for after the user is erased self.get_success( - profile_handler.set_displayname( - user_id, create_requester(user_id), "Kermit the Frog" + profile_handler.set_field( + target_user=user_id, + requester=create_requester(user_id), + field_name=ProfileFields.DISPLAYNAME, + new_value="Kermit the Frog", ) ) self.get_success( - profile_handler.set_avatar_url( - user_id, create_requester(user_id), "http://test/Kermit.jpg" + profile_handler.set_field( + target_user=user_id, + requester=create_requester(user_id), + field_name=ProfileFields.AVATAR_URL, + new_value="http://test/Kermit.jpg", ) ) # Verify it is set @@ -580,9 +604,19 @@ class DeactivateTestCase(unittest.HomeserverTestCase): # Can not use the profile handler to set a display name when it is disabled. Use # the database directly store = self.hs.get_datastores().main - self.get_success(store.set_profile_displayname(user_id, "Kermit the Frog")) self.get_success( - store.set_profile_avatar_url(user_id, "http://test/Kermit.jpg") + store.set_profile_field( + user_id=user_id, + field_name=ProfileFields.DISPLAYNAME, + new_value="Kermit the Frog", + ) + ) + self.get_success( + store.set_profile_field( + user_id=user_id, + field_name=ProfileFields.AVATAR_URL, + new_value="http://test/Kermit.jpg", + ) ) # Verify it is set @@ -996,21 +1030,21 @@ class ThreepidEmailRestTestCase(unittest.HomeserverTestCase): def test_add_email_no_at(self) -> None: self._request_token_invalid_email( "address-without-at.bar", - expected_errcode=Codes.BAD_JSON, + expected_errcode=Codes.INVALID_PARAM, expected_error="Unable to parse email address", ) def test_add_email_two_at(self) -> None: self._request_token_invalid_email( "foo@foo@test.bar", - expected_errcode=Codes.BAD_JSON, + expected_errcode=Codes.INVALID_PARAM, expected_error="Unable to parse email address", ) def test_add_email_bad_format(self) -> None: self._request_token_invalid_email( "user@bad.example.net@good.example.com", - expected_errcode=Codes.BAD_JSON, + expected_errcode=Codes.INVALID_PARAM, expected_error="Unable to parse email address", ) @@ -1421,6 +1455,69 @@ class ThreepidEmailRestTestCase(unittest.HomeserverTestCase): self.assertIn(expected_email, threepids) +class ThreepidMsisdnRestTestCase(unittest.HomeserverTestCase): + """Tests the error codes of /account/3pid/msisdn/requestToken. + + See https://spec.matrix.org/v1.19/client-server-api/#post_matrixclientv3account3pidmsisdnrequesttoken + (error codes added in Matrix v1.13 by MSC4178). + """ + + servlets = [ + account.register_servlets, + login.register_servlets, + synapse.rest.admin.register_servlets_for_client_rest_resource, + ] + + def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None: + self.store = hs.get_datastores().main + self.user_id = self.register_user("kermit", "test") + + def _request_token(self, country: str, phone_number: str) -> FakeChannel: + return self.make_request( + "POST", + b"account/3pid/msisdn/requestToken", + { + "client_secret": "foobar", + "country": country, + "phone_number": phone_number, + "send_attempt": 1, + }, + ) + + @override_config({"account_threepid_delegates": {"msisdn": "https://id_server"}}) + def test_invalid_country_code(self) -> None: + """A malformed country code is reported with M_INVALID_PARAM.""" + channel = self._request_token("gb", "07700900001") + self.assertEqual( + HTTPStatus.BAD_REQUEST, channel.code, msg=channel.result["body"] + ) + self.assertEqual(Codes.INVALID_PARAM, channel.json_body["errcode"]) + + @override_config({"account_threepid_delegates": {"msisdn": "https://id_server"}}) + def test_invalid_phone_number(self) -> None: + """An unparseable phone number is reported with M_INVALID_PARAM.""" + channel = self._request_token("GB", "not a phone number") + self.assertEqual( + HTTPStatus.BAD_REQUEST, channel.code, msg=channel.result["body"] + ) + self.assertEqual(Codes.INVALID_PARAM, channel.json_body["errcode"]) + + @override_config({"allowed_local_3pids": [{"medium": "email", "pattern": ".*"}]}) + def test_medium_not_supported_checked_before_denied(self) -> None: + """When the server cannot send validation SMSes, it reports + M_THREEPID_MEDIUM_NOT_SUPPORTED even if the phone number would be + denied: the unsupported-medium check comes first, as on the email + variant. + """ + channel = self._request_token("GB", "07700900001") + self.assertEqual( + HTTPStatus.BAD_REQUEST, channel.code, msg=channel.result["body"] + ) + self.assertEqual( + Codes.THREEPID_MEDIUM_NOT_SUPPORTED, channel.json_body["errcode"] + ) + + class AccountStatusTestCase(unittest.HomeserverTestCase): servlets = [ account.register_servlets, diff --git a/tests/rest/client/test_appservice_proxy.py b/tests/rest/client/test_appservice_proxy.py new file mode 100644 index 0000000000..d389fa7d00 --- /dev/null +++ b/tests/rest/client/test_appservice_proxy.py @@ -0,0 +1,442 @@ +# +# This file is licensed under the Affero General Public License (AGPL) version 3. +# +# Copyright (C) 2026 Element Creations Ltd +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# See the GNU Affero General Public License for more details: +# . +# + +import os +import tempfile +from unittest.mock import Mock + +import yaml + +from twisted.internet import defer +from twisted.internet.testing import MemoryReactor +from twisted.web.http_headers import Headers + +from synapse.rest import admin +from synapse.rest.client import appservice_proxy, login +from synapse.server import HomeServer +from synapse.types import JsonDict +from synapse.util.clock import Clock +from synapse.util.json import json_encoder + +from tests import unittest +from tests.test_utils import FakeResponse + +APPSERVICE_URL = "http://appservice.example.com" +APPSERVICE_PREFIX = "rtc/livekit" +VERSIONED_PREFIX = f"v1/{APPSERVICE_PREFIX}" + + +class ApplicationServiceClientProxyTestCase(unittest.HomeserverTestCase): + servlets = [ + admin.register_servlets, + login.register_servlets, + appservice_proxy.register_servlets, + ] + + def default_config(self) -> JsonDict: + config = super().default_config() + with tempfile.NamedTemporaryFile( + mode="w", prefix="as_proxy_config", delete=False + ) as f: + self.addCleanup(os.remove, f.name) + yaml.dump( + { + "id": "proxy_as", + "url": None, + "as_token": "as_token", + "hs_token": "hs_token", + "sender_localpart": "proxy_bot", + "namespaces": {}, + "io.element.msc4512.proxy_prefix": APPSERVICE_PREFIX, + "io.element.msc4512.proxy_url": APPSERVICE_URL, + }, + f, + ) + config["app_service_config_files"] = [f.name] + config.setdefault("experimental_features", {}).setdefault( + "msc4512_enabled", True + ) + return config + + def prepare(self, _reactor: MemoryReactor, _clock: Clock, hs: HomeServer) -> None: + self.agent = Mock() + hs.get_proxied_http_client().agent = self.agent + + self.user_id = self.register_user("proxy_user", "password") + self.access_token = self.login("proxy_user", "password") + + def test_get_is_proxied(self) -> None: + self.agent.request = Mock( + return_value=defer.succeed( + FakeResponse.json(code=200, payload={"hello": "world"}) + ) + ) + + channel = self.make_request( + "GET", + f"/_matrix/client/{VERSIONED_PREFIX}/some/path?foo=bar", + shorthand=False, + access_token=self.access_token, + ) + + self.assertEqual(channel.code, 200) + self.assertEqual(channel.json_body, {"hello": "world"}) + + ((method, uri), kwargs) = self.agent.request.call_args + + self.assertEqual(method, b"GET") + self.assertEqual( + uri, + f"{APPSERVICE_URL}/_matrix/client/{VERSIONED_PREFIX}/some/path?foo=bar".encode(), + ) + + headers: Headers = kwargs["headers"] + self.assertEqual(headers.getRawHeaders(b"Authorization"), [b"Bearer hs_token"]) + self.assertEqual( + headers.getRawHeaders(b"X-Matrix-User-Identifier"), + [self.user_id.encode("ascii")], + ) + + def test_access_token_query_param_is_stripped(self) -> None: + self.agent.request = Mock( + return_value=defer.succeed( + FakeResponse.json(code=200, payload={"hello": "world"}) + ) + ) + + channel = self.make_request( + "GET", + f"/_matrix/client/{VERSIONED_PREFIX}/some/path?access_token={self.access_token}&foo=bar", + shorthand=False, + ) + + self.assertEqual(channel.code, 200) + self.assertEqual(channel.json_body, {"hello": "world"}) + + ((method, uri), _kwargs) = self.agent.request.call_args + + self.assertEqual(method, b"GET") + self.assertEqual( + uri, + f"{APPSERVICE_URL}/_matrix/client/{VERSIONED_PREFIX}/some/path?foo=bar".encode(), + ) + + def test_get_is_proxied_at_root_path(self) -> None: + self.agent.request = Mock( + return_value=defer.succeed( + FakeResponse.json(code=200, payload={"hello": "world"}) + ) + ) + + channel = self.make_request( + "GET", + f"/_matrix/client/{VERSIONED_PREFIX}", + shorthand=False, + access_token=self.access_token, + ) + + self.assertEqual(channel.code, 200) + self.assertEqual(channel.json_body, {"hello": "world"}) + + ((method, uri), kwargs) = self.agent.request.call_args + + self.assertEqual(method, b"GET") + self.assertEqual( + uri, + f"{APPSERVICE_URL}/_matrix/client/{VERSIONED_PREFIX}".encode(), + ) + + headers: Headers = kwargs["headers"] + self.assertEqual(headers.getRawHeaders(b"Authorization"), [b"Bearer hs_token"]) + self.assertEqual( + headers.getRawHeaders(b"X-Matrix-User-Identifier"), + [self.user_id.encode("ascii")], + ) + + def test_post_is_proxied(self) -> None: + self.agent.request = Mock( + return_value=defer.succeed( + FakeResponse.json(code=200, payload={"hello": "world"}) + ) + ) + + channel = self.make_request( + "POST", + f"/_matrix/client/{VERSIONED_PREFIX}/some/path", + content={"key": "value"}, + shorthand=False, + access_token=self.access_token, + ) + + self.assertEqual(channel.code, 200) + self.assertEqual(channel.json_body, {"hello": "world"}) + + ((method, uri), kwargs) = self.agent.request.call_args + + self.assertEqual(method, b"POST") + self.assertEqual( + uri, + f"{APPSERVICE_URL}/_matrix/client/{VERSIONED_PREFIX}/some/path".encode(), + ) + + headers: Headers = kwargs["headers"] + self.assertEqual(headers.getRawHeaders(b"Authorization"), [b"Bearer hs_token"]) + self.assertEqual( + headers.getRawHeaders(b"X-Matrix-User-Identifier"), + [self.user_id.encode("ascii")], + ) + self.assertEqual(headers.getRawHeaders(b"Content-Type"), [b"application/json"]) + + body_producer = kwargs["bodyProducer"] + expected_body = json_encoder.encode({"key": "value"}).encode("utf8") + self.assertEqual(body_producer.length, len(expected_body)) + + def test_headers_outside_the_allowlist_not_forwarded(self) -> None: + self.agent.request = Mock( + return_value=defer.succeed(FakeResponse.json(code=200, payload={})) + ) + + self.make_request( + "GET", + f"/_matrix/client/{VERSIONED_PREFIX}/some/path", + shorthand=False, + access_token=self.access_token, + custom_headers=[("Connection", "close"), ("X-Forward", "forward")], + ) + + ((_method, _uri), kwargs) = self.agent.request.call_args + + headers: Headers = kwargs["headers"] + self.assertIsNone(headers.getRawHeaders(b"Connection")) + self.assertIsNone(headers.getRawHeaders(b"X-Forward")) + + def test_allowlisted_headers_forwarded(self) -> None: + self.agent.request = Mock( + return_value=defer.succeed(FakeResponse.json(code=200, payload={})) + ) + + self.make_request( + "GET", + f"/_matrix/client/{VERSIONED_PREFIX}/some/path", + shorthand=False, + access_token=self.access_token, + custom_headers=[ + ("Accept", "application/json"), + ("Accept-Language", "en-US"), + ], + ) + + ((_method, _uri), kwargs) = self.agent.request.call_args + + headers: Headers = kwargs["headers"] + self.assertEqual(headers.getRawHeaders(b"Accept"), [b"application/json"]) + self.assertEqual(headers.getRawHeaders(b"Accept-Language"), [b"en-US"]) + + def test_host_and_content_length_headers_not_forwarded(self) -> None: + self.agent.request = Mock( + return_value=defer.succeed(FakeResponse.json(code=200, payload={})) + ) + + self.make_request( + "POST", + f"/_matrix/client/{VERSIONED_PREFIX}/some/path", + content={"key": "value"}, + shorthand=False, + access_token=self.access_token, + custom_headers=[("Host", "original-client-facing-host.example")], + ) + + ((_method, _uri), kwargs) = self.agent.request.call_args + + headers: Headers = kwargs["headers"] + self.assertIsNone(headers.getRawHeaders(b"Host")) + self.assertIsNone(headers.getRawHeaders(b"Content-Length")) + + def test_response_headers_forwarded(self) -> None: + self.agent.request = Mock( + return_value=defer.succeed( + FakeResponse( + code=200, + body=b"hello", + headers=Headers({"X-Forward": ["forward"]}), + ) + ) + ) + + channel = self.make_request( + "GET", + f"/_matrix/client/{VERSIONED_PREFIX}/some/path", + shorthand=False, + access_token=self.access_token, + ) + + self.assertEqual(channel.code, 200) + self.assertEqual(channel.result["body"], b"hello") + self.assertEqual(channel.headers.getRawHeaders(b"X-Forward"), [b"forward"]) + + def test_response_cors_headers_set(self) -> None: + self.agent.request = Mock( + return_value=defer.succeed(FakeResponse.json(code=200, payload={})) + ) + + channel = self.make_request( + "GET", + f"/_matrix/client/{VERSIONED_PREFIX}/some/path", + shorthand=False, + access_token=self.access_token, + ) + + self.assertEqual(channel.code, 200) + self.assertEqual( + channel.headers.getRawHeaders(b"Access-Control-Allow-Origin"), [b"*"] + ) + + def test_non_existing_path_under_proxy_prefix_is_rejected(self) -> None: + self.agent.request = Mock(return_value=defer.fail(Exception("boom"))) + + channel = self.make_request( + "GET", + f"/_matrix/client/{VERSIONED_PREFIX}/does/not/exist", + shorthand=False, + access_token=self.access_token, + ) + + self.assertEqual(channel.code, 500) + self.assertEqual(channel.json_body["errcode"], "M_UNKNOWN") + self.agent.request.assert_called() + + def test_unauthenticated_get_is_rejected(self) -> None: + self.agent.request = Mock( + return_value=defer.succeed(FakeResponse.json(code=200, payload={})) + ) + + channel = self.make_request( + "GET", + f"/_matrix/client/{VERSIONED_PREFIX}/some/path", + shorthand=False, + ) + + self.assertEqual(channel.code, 401) + self.agent.request.assert_not_called() + + @unittest.override_config({"rc_message": {"burst_count": 0}}) + def test_rate_limited_request_is_rejected(self) -> None: + self.agent.request = Mock( + return_value=defer.succeed(FakeResponse.json(code=200, payload={})) + ) + + channel = self.make_request( + "GET", + f"/_matrix/client/{VERSIONED_PREFIX}/some/path", + shorthand=False, + access_token=self.access_token, + ) + + self.assertEqual(channel.code, 429) + self.agent.request.assert_not_called() + + def test_path_with_dot_segment_is_rejected(self) -> None: + self.agent.request = Mock( + return_value=defer.succeed(FakeResponse.json(code=200, payload={})) + ) + + channel = self.make_request( + "GET", + f"/_matrix/client/{VERSIONED_PREFIX}/some/../path", + shorthand=False, + access_token=self.access_token, + ) + + self.assertEqual(channel.code, 400) + self.assertEqual(channel.json_body["errcode"], "M_INVALID_PARAM") + self.agent.request.assert_not_called() + + def test_path_with_encoded_dot_segment_is_rejected(self) -> None: + self.agent.request = Mock( + return_value=defer.succeed(FakeResponse.json(code=200, payload={})) + ) + + channel = self.make_request( + "GET", + f"/_matrix/client/{VERSIONED_PREFIX}/some/%2e%2e/path", + shorthand=False, + access_token=self.access_token, + ) + + self.assertEqual(channel.code, 400) + self.assertEqual(channel.json_body["errcode"], "M_INVALID_PARAM") + self.agent.request.assert_not_called() + + def test_unregistered_prefix_is_rejected(self) -> None: + channel = self.make_request( + "GET", + "/_matrix/client/not-a-prefix", + shorthand=False, + ) + + self.assertEqual(channel.code, 404) + + def test_unregistered_prefix_with_suffix_is_rejected(self) -> None: + channel = self.make_request( + "GET", + f"/_matrix/client/{VERSIONED_PREFIX}-2", + shorthand=False, + ) + + self.assertEqual(channel.code, 404) + + def test_missing_version_segment_is_rejected(self) -> None: + channel = self.make_request( + "GET", + f"/_matrix/client/{APPSERVICE_PREFIX}/some/path", + shorthand=False, + access_token=self.access_token, + ) + + self.assertEqual(channel.code, 404) + self.agent.request.assert_not_called() + + def test_unstable_msc_version_segment_is_proxied(self) -> None: + self.agent.request = Mock( + return_value=defer.succeed( + FakeResponse.json(code=200, payload={"hello": "world"}) + ) + ) + + path = f"/_matrix/client/unstable/org.example.msc9999/{APPSERVICE_PREFIX}/some/path" + channel = self.make_request( + "GET", + path, + shorthand=False, + access_token=self.access_token, + ) + + self.assertEqual(channel.code, 200) + self.assertEqual(channel.json_body, {"hello": "world"}) + + ((method, uri), _kwargs) = self.agent.request.call_args + self.assertEqual(method, b"GET") + self.assertEqual(uri, f"{APPSERVICE_URL}{path}".encode()) + + @unittest.override_config({"experimental_features": {"msc4512_enabled": False}}) + def test_proxy_route_not_registered_when_msc4512_disabled(self) -> None: + channel = self.make_request( + "GET", + f"/_matrix/client/{VERSIONED_PREFIX}/some/path", + shorthand=False, + access_token=self.access_token, + ) + + self.assertEqual(channel.code, 404) + self.agent.request.assert_not_called() diff --git a/tests/rest/client/test_auth_metadata.py b/tests/rest/client/test_auth_metadata.py index e9f8597c5a..683fce46e3 100644 --- a/tests/rest/client/test_auth_metadata.py +++ b/tests/rest/client/test_auth_metadata.py @@ -27,20 +27,6 @@ from synapse.rest.client import auth_metadata from tests.unittest import HomeserverTestCase -class AuthIssuerTestCase(HomeserverTestCase): - servlets = [ - auth_metadata.register_servlets, - ] - - def test_returns_404_when_mas_disabled(self) -> None: - # Make an unauthenticated request for the discovery info. - channel = self.make_request( - "GET", - "/_matrix/client/unstable/org.matrix.msc2965/auth_issuer", - ) - self.assertEqual(channel.code, HTTPStatus.NOT_FOUND) - - @parameterized_class( ("endpoint",), [ diff --git a/tests/rest/client/test_capabilities.py b/tests/rest/client/test_capabilities.py index c28e0605b5..42926c4359 100644 --- a/tests/rest/client/test_capabilities.py +++ b/tests/rest/client/test_capabilities.py @@ -26,6 +26,7 @@ from synapse.api.room_versions import KNOWN_ROOM_VERSIONS from synapse.rest.client import capabilities, login from synapse.server import HomeServer from synapse.util.clock import Clock +from synapse.util.duration import Duration from tests import unittest from tests.unittest import override_config, skip_unless @@ -203,6 +204,43 @@ class CapabilitiesTestCase(unittest.HomeserverTestCase): ["avatar_url"], ) + def test_get_delayed_events_capabilities_default_config_msc4140(self) -> None: + access_token = self.login(self.localpart, self.password) + + channel = self.make_request("GET", self.url, access_token=access_token) + capabilities = channel.json_body["capabilities"] + + self.assertEqual(channel.code, HTTPStatus.OK) + self.assertEqual( + capabilities["org.matrix.msc4140.delayed_events"]["max_delay_ms"], 0 + ) + self.assertEqual( + capabilities["org.matrix.msc4140.delayed_events"]["max_scheduled"], 100 + ) + + @override_config( + { + "max_event_delay_duration": "24h", + "experimental_features": { + "msc4140_max_delayed_events_per_user": 50, + }, + } + ) + def test_get_delayed_events_capabilities_custom_config_msc4140(self) -> None: + access_token = self.login(self.localpart, self.password) + + channel = self.make_request("GET", self.url, access_token=access_token) + capabilities = channel.json_body["capabilities"] + + self.assertEqual(channel.code, HTTPStatus.OK) + self.assertEqual( + capabilities["org.matrix.msc4140.delayed_events"]["max_delay_ms"], + Duration(days=1).as_millis(), + ) + self.assertEqual( + capabilities["org.matrix.msc4140.delayed_events"]["max_scheduled"], 50 + ) + @override_config({"enable_3pid_changes": False}) def test_get_change_3pid_capabilities_3pid_disabled(self) -> None: """Test if change 3pid is disabled that the server responds it.""" diff --git a/tests/rest/client/test_delayed_events.py b/tests/rest/client/test_delayed_events.py index 75d716244a..3af7d13858 100644 --- a/tests/rest/client/test_delayed_events.py +++ b/tests/rest/client/test_delayed_events.py @@ -127,6 +127,120 @@ class DelayedEventsTestCase(HomeserverTestCase): def test_delayed_events_empty_on_startup(self) -> None: self.assertListEqual([], self._get_delayed_events()) + def test_delayed_event_lookup(self) -> None: + # Schedule a message event + delay_ms = 100000 + content: JsonDict = {"message": "hello"} + delayed_since_ts = self.hs.get_clock().time_msec() + channel = self.make_request( + "POST", + _get_path_for_delayed_send(self.room_id, _EVENT_TYPE, delay_ms), + content, + self.user1_access_token, + ) + self.assertEqual(channel.code, HTTPStatus.OK, channel.result) + delay_id = channel.json_body["delay_id"] + + # Test that the scheduled delayed event can be retrieved + channel = self.make_request( + "GET", + f"{PATH_PREFIX}/{delay_id}", + access_token=self.user1_access_token, + ) + self.assertEqual(channel.code, HTTPStatus.OK, channel.result) + + # Assert the stored properties of the delayed event + event = channel.json_body + self.assertDictEqual( + event, + { + "delay_id": delay_id, + "room_id": self.room_id, + "type": _EVENT_TYPE, + "delay_ms": delay_ms, + "delayed_since_ts": delayed_since_ts, + "content": content, + }, + ) + + # Test that a non-existent delayed event cannot be found + channel = self.make_request( + "GET", + f"{PATH_PREFIX}/{delay_id}-fake", + access_token=self.user1_access_token, + ) + self.assertEqual(channel.code, HTTPStatus.NOT_FOUND, channel.result) + + # Test that other users cannot access this delayed event + channel = self.make_request( + "GET", + f"{PATH_PREFIX}/{delay_id}", + access_token=self.user2_access_token, + ) + self.assertEqual(channel.code, HTTPStatus.NOT_FOUND, channel.result) + + # Now schedule a state event. + # Do it in this test, as opposed to a new one, to confirm that the correct delayed event + # is retrieved when multiple delayed events have been scheduled. + delay_ms += 2000 + state_key = "" + state_event_type = _EVENT_TYPE + "_state" + content = {"state_message": "greetings"} + delayed_since_ts = self.hs.get_clock().time_msec() + channel = self.make_request( + "PUT", + _get_path_for_delayed_state( + self.room_id, state_event_type, state_key, delay_ms + ), + content, + self.user1_access_token, + ) + self.assertEqual(channel.code, HTTPStatus.OK, channel.result) + delay_id_2 = channel.json_body["delay_id"] + + # Test that the new delayed event has a different ID from the previous one + self.assertNotEqual(delay_id, delay_id_2) + + # Test that the scheduled delayed event can be retrieved + channel = self.make_request( + "GET", + f"{PATH_PREFIX}/{delay_id_2}", + access_token=self.user1_access_token, + ) + self.assertEqual(channel.code, HTTPStatus.OK, channel.result) + + # Assert the stored properties of the delayed event + state_event = channel.json_body + self.assertDictEqual( + state_event, + { + "delay_id": delay_id_2, + "room_id": self.room_id, + "type": state_event_type, + "state_key": state_key, + "delay_ms": delay_ms, + "delayed_since_ts": delayed_since_ts, + "content": content, + }, + ) + + # Test that the list lookup retrieves the same items (with legacy fields included) + self.assertEqual( + self._get_delayed_events(), + [ + event + | { + "delay": event["delay_ms"], + "running_since": event["delayed_since_ts"], + }, + state_event + | { + "delay": state_event["delay_ms"], + "running_since": state_event["delayed_since_ts"], + }, + ], + ) + def test_delayed_state_events_are_sent_on_timeout(self) -> None: state_key = "to_send_on_timeout" diff --git a/tests/rest/client/test_login.py b/tests/rest/client/test_login.py index d83604a696..f1639bfeee 100644 --- a/tests/rest/client/test_login.py +++ b/tests/rest/client/test_login.py @@ -1521,7 +1521,7 @@ class AppserviceLoginRestServletTestCase(unittest.HomeserverTestCase): self.assertEqual(channel.code, 200, msg=channel.result) def test_login_appservice_msc4190_fail(self) -> None: - """Test that an appservice user can use /login""" + """Test that an appservice with MSC4190 enabled can't use /login""" self.register_appservice_user( "as3_user_alice", self.msc4190_service.token, inhibit_login=True ) @@ -1537,7 +1537,7 @@ class AppserviceLoginRestServletTestCase(unittest.HomeserverTestCase): self.assertEqual(channel.code, 400, msg=channel.result) self.assertEqual( channel.json_body.get("errcode"), - Codes.APPSERVICE_LOGIN_UNSUPPORTED, + "M_APPSERVICE_LOGIN_UNSUPPORTED", channel.json_body, ) diff --git a/tests/rest/client/test_matrixrtc.py b/tests/rest/client/test_matrixrtc.py index f2bf1596be..63c7632a88 100644 --- a/tests/rest/client/test_matrixrtc.py +++ b/tests/rest/client/test_matrixrtc.py @@ -17,8 +17,13 @@ """Tests REST events for /rtc/endpoints path.""" +import unittest as stdlib_unittest + +from pydantic import ValidationError + from twisted.internet.testing import MemoryReactor +from synapse.config.matrixrtc import TransportConfigModel from synapse.rest import admin from synapse.rest.client import login, matrixrtc, register, room, versions from synapse.server import HomeServer @@ -30,7 +35,16 @@ from tests.unittest import HomeserverTestCase, override_config PATH_PREFIX = "/_matrix/client/unstable/org.matrix.msc4143" RTC_ENDPOINT = {"type": "focusA", "required_field": "theField"} -LIVEKIT_ENDPOINT = { +LIVEKIT_TRANSPORT = { + "type": "livekit", + "url": "wss://livekit.example.com", +} +BACKWARDS_COMPATIBLE_LIVEKIT_TRANSPORT = { + "type": "livekit", + "url": "wss://livekit.example.com", + "livekit_service_url": "https://livekit.example.com", +} +LEGACY_LIVEKIT_TRANSPORT = { "type": "livekit", "livekit_service_url": "https://livekit.example.com", } @@ -96,7 +110,7 @@ class MatrixRtcTestCase(HomeserverTestCase): @override_config( { "experimental_features": {"msc4143_enabled": True}, - "matrix_rtc": {"transports": [LIVEKIT_ENDPOINT]}, + "matrix_rtc": {"transports": [LIVEKIT_TRANSPORT]}, } ) def test_matrixrtc_endpoint_livekit_transport(self) -> None: @@ -104,7 +118,38 @@ class MatrixRtcTestCase(HomeserverTestCase): "GET", f"{PATH_PREFIX}/rtc/transports", access_token=self._alice_tok ) self.assertEqual(200, channel.code, channel.json_body) - self.assert_dict({"rtc_transports": [LIVEKIT_ENDPOINT]}, channel.json_body) + self.assert_dict({"rtc_transports": [LIVEKIT_TRANSPORT]}, channel.json_body) + + @override_config( + { + "experimental_features": {"msc4143_enabled": True}, + "matrix_rtc": {"transports": [BACKWARDS_COMPATIBLE_LIVEKIT_TRANSPORT]}, + } + ) + def test_matrixrtc_endpoint_backwards_compatible_livekit_transport(self) -> None: + channel = self.make_request( + "GET", f"{PATH_PREFIX}/rtc/transports", access_token=self._alice_tok + ) + self.assertEqual(200, channel.code, channel.json_body) + self.assert_dict( + {"rtc_transports": [BACKWARDS_COMPATIBLE_LIVEKIT_TRANSPORT]}, + channel.json_body, + ) + + @override_config( + { + "experimental_features": {"msc4143_enabled": True}, + "matrix_rtc": {"transports": [LEGACY_LIVEKIT_TRANSPORT]}, + } + ) + def test_matrixrtc_endpoint_legacy_livekit_transport(self) -> None: + channel = self.make_request( + "GET", f"{PATH_PREFIX}/rtc/transports", access_token=self._alice_tok + ) + self.assertEqual(200, channel.code, channel.json_body) + self.assert_dict( + {"rtc_transports": [LEGACY_LIVEKIT_TRANSPORT]}, channel.json_body + ) class MatrixRtcVersionsTestCase(HomeserverTestCase): @@ -150,3 +195,23 @@ class MatrixRtcVersionsTestCase(HomeserverTestCase): channel = self.make_request("GET", "/_matrix/client/versions") self.assertEqual(channel.code, 200, channel.result) self.assertTrue(channel.json_body["unstable_features"]["org.matrix.msc4143"]) + + +class TransportConfigModelTestCase(stdlib_unittest.TestCase): + """Tests validation of the `TransportConfigModel` pydantic model.""" + + def test_livekit_transport_requires_url_or_livekit_service_url(self) -> None: + with self.assertRaises(ValidationError): + TransportConfigModel(type="livekit") + + def test_livekit_transport_with_only_url(self) -> None: + TransportConfigModel(type="livekit", url="wss://livekit.example.com") + + def test_livekit_transport_with_only_livekit_service_url(self) -> None: + TransportConfigModel( + type="livekit", livekit_service_url="https://livekit.example.com" + ) + + def test_invalid_field_type(self) -> None: + with self.assertRaises(ValidationError): + TransportConfigModel(type="livekit", url=1234) # type: ignore[arg-type] diff --git a/tests/rest/client/test_media.py b/tests/rest/client/test_media.py index 3409581c5e..b20417a6ed 100644 --- a/tests/rest/client/test_media.py +++ b/tests/rest/client/test_media.py @@ -3,6 +3,7 @@ # # Copyright 2022 The Matrix.org Foundation C.I.C. # Copyright (C) 2024 New Vector, Ltd +# Copyright (C) 2026 Element Creations Ltd # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License as @@ -29,6 +30,7 @@ from unittest.mock import MagicMock, Mock, patch from urllib import parse from urllib.parse import quote, urlencode +from matrix_common.types.mxc_uri import MXCUri from parameterized import parameterized, parameterized_class from PIL import Image as Image @@ -57,6 +59,10 @@ from synapse.media.url_previewer import IMAGE_CACHE_EXPIRY_MS from synapse.module_api import MediaUploadLimit from synapse.rest import admin from synapse.rest.client import login, media +from synapse.rest.synapse.client import build_synapse_client_resource_tree +from synapse.rest.synapse.client.media_upload_limit_exceeded import ( + MEDIA_UPLOAD_LIMIT_EXCEEDED_PATH, +) from synapse.server import HomeServer from synapse.types import JsonDict, UserID from synapse.util.clock import Clock @@ -2932,11 +2938,20 @@ class MediaUploadLimits(unittest.HomeserverTestCase): config["media_storage_providers"] = [provider_config] - # These are the limits that we are testing - config["media_upload_limits"] = [ - {"time_period": "1d", "max_size": "1K"}, - {"time_period": "1w", "max_size": "3K"}, - ] + # These are the limits that we are testing unless overridden + if config.get("media_upload_limits") is None: + config["media_upload_limits"] = [ + { + "time_period": "1d", + "max_size": "1K", + "info_uri": "https://example.com/limits#daily", + }, + { + "time_period": "1w", + "max_size": "3K", + "info_uri": "https://example.com/limits#weekly", + }, + ] return self.setup_test_homeserver(config=config) @@ -2950,6 +2965,9 @@ class MediaUploadLimits(unittest.HomeserverTestCase): def create_resource_dict(self) -> dict[str, Resource]: resources = super().create_resource_dict() resources["/_matrix/media"] = self.hs.get_media_repository_resource() + # Mount the `/_synapse/client` tree, which includes the fallback page + # served at `MEDIA_UPLOAD_LIMIT_EXCEEDED_PATH` which we use in the tests. + resources.update(build_synapse_client_resource_tree(self.hs)) return resources def upload_media(self, size: int) -> FakeChannel: @@ -2974,7 +2992,8 @@ class MediaUploadLimits(unittest.HomeserverTestCase): self.assertEqual(channel.code, 200) channel = self.upload_media(800) - self.assertEqual(channel.code, 400) + self.assertEqual(channel.code, 403) + self.assertEqual(channel.json_body["errcode"], "M_USER_LIMIT_EXCEEDED") def test_under_daily_limit(self) -> None: """Test that uploading media under the daily limit fails.""" @@ -3012,7 +3031,8 @@ class MediaUploadLimits(unittest.HomeserverTestCase): # This will fail as the weekly limit has been exceeded channel = self.upload_media(900) - self.assertEqual(channel.code, 400) + self.assertEqual(channel.code, 403) + self.assertEqual(channel.json_body["errcode"], "M_USER_LIMIT_EXCEEDED") # Reset the weekly limit by advancing a week self.reactor.advance(7 * 60 * 60 * 24) # Advance by 7 days @@ -3021,6 +3041,104 @@ class MediaUploadLimits(unittest.HomeserverTestCase): channel = self.upload_media(900) self.assertEqual(channel.code, 200) + @override_config( + { + "media_upload_limits": [ + { + "time_period": "1d", + "max_size": "1K", + # No `info_uri` is configured, so the fallback resource + # should be used instead. + } + ], + } + ) + def test_falls_back_to_served_page_when_no_info_uri(self) -> None: + """When no info_uri is configured, the error should point at a page + served by Synapse, and that page should return HTML.""" + channel = self.upload_media(1300) + self.assertEqual(channel.code, 403) + self.assertEqual(channel.json_body["errcode"], "M_USER_LIMIT_EXCEEDED") + + expected_info_uri = ( + self.hs.config.server.public_baseurl + + MEDIA_UPLOAD_LIMIT_EXCEEDED_PATH.lstrip("/") + ) + self.assertEqual(channel.json_body["info_uri"], expected_info_uri) + + # The fallback page should be served by Synapse and return HTML. + page = self.make_request( + "GET", + MEDIA_UPLOAD_LIMIT_EXCEEDED_PATH, + shorthand=False, + ) + self.assertEqual(page.code, 200) + self.assertEqual( + page.headers.getRawHeaders("Content-Type"), + ["text/html; charset=utf-8"], + ) + self.assertIn(b"upload limit", page.result["body"]) + + def test_fallback_page_mounted(self) -> None: + """The fallback resource should always be mounted, even if every configured + limit has an explicit info_uri, since module callbacks can return + limits without an info_uri at any time.""" + # The default config for this test case sets an info_uri on every limit. + page = self.make_request( + "GET", + MEDIA_UPLOAD_LIMIT_EXCEEDED_PATH, + shorthand=False, + ) + self.assertEqual(page.code, 200) + self.assertEqual( + page.headers.getRawHeaders("Content-Type"), + ["text/html; charset=utf-8"], + ) + + @override_config( + { + "media_upload_limits": [ + { + "time_period": "1d", + "max_size": "1K", + "info_uri": "https://example.com", + } + ], + } + ) + def test_returns_hard_user_limit_exceeded_by_default(self) -> None: + """Test that the error is returned with can_upgrade False by default.""" + channel = self.upload_media(1300) + self.assertEqual(channel.code, 403) + self.assertEqual(channel.json_body["errcode"], "M_USER_LIMIT_EXCEEDED") + self.assertEqual(channel.json_body["info_uri"], "https://example.com/") + # the spec says that can_upgrade should not be included if it is False + self.assertIsNone(channel.json_body.get("can_upgrade")) + + @override_config( + { + "media_upload_limits": [ + { + "time_period": "1d", + "max_size": "1K", + "info_uri": "https://example.com", + "can_upgrade": True, + } + ], + } + ) + def test_returns_soft_user_limit_exceeded(self) -> None: + """Test that the M_USER_LIMIT_EXCEEDED error is returned with + can_upgrade True when specified in config.""" + channel = self.upload_media(500) + self.assertEqual(channel.code, 200) + + channel = self.upload_media(800) + self.assertEqual(channel.code, 403) + self.assertEqual(channel.json_body["errcode"], "M_USER_LIMIT_EXCEEDED") + self.assertEqual(channel.json_body["info_uri"], "https://example.com/") + self.assertEqual(channel.json_body["can_upgrade"], True) + class MediaUploadLimitsModuleOverrides(unittest.HomeserverTestCase): """ @@ -3053,10 +3171,19 @@ class MediaUploadLimitsModuleOverrides(unittest.HomeserverTestCase): config["media_storage_providers"] = [provider_config] # default limits to use - config["media_upload_limits"] = [ - {"time_period": "1d", "max_size": "1K"}, - {"time_period": "1w", "max_size": "3K"}, - ] + if config.get("media_upload_limits") is None: + config["media_upload_limits"] = [ + { + "time_period": "1d", + "max_size": "1K", + "info_uri": "https://example.com/limits#daily", + }, + { + "time_period": "1w", + "max_size": "3K", + "info_uri": "https://example.com/limits#weekly", + }, + ] return self.setup_test_homeserver(config=config) @@ -3069,10 +3196,14 @@ class MediaUploadLimitsModuleOverrides(unittest.HomeserverTestCase): # n.b. we return these in increasing duration order and Synapse will need to sort them correctly return [ MediaUploadLimit( - time_period_ms=Config.parse_duration("1d"), max_bytes=5000 + time_period_ms=Config.parse_duration("1d"), + max_bytes=5000, + info_uri="https://override.example.com/limits#daily", ), MediaUploadLimit( - time_period_ms=Config.parse_duration("1w"), max_bytes=15000 + time_period_ms=Config.parse_duration("1w"), + max_bytes=15000, + info_uri="https://override.example.com/limits#weekly", ), ] # user2 has no limits @@ -3153,13 +3284,16 @@ class MediaUploadLimitsModuleOverrides(unittest.HomeserverTestCase): # User 1 attempts to upload 4000 bytes taking it over the limit channel = self.upload_media(4000, self.tok1) - self.assertEqual(channel.code, 400) + self.assertEqual(channel.code, 403) assert self.last_media_upload_limit_exceeded is not None self.assertEqual(self.last_media_upload_limit_exceeded["user_id"], self.user1) self.assertEqual( self.last_media_upload_limit_exceeded["limit"], MediaUploadLimit( - max_bytes=5000, time_period_ms=Config.parse_duration("1d") + max_bytes=5000, + time_period_ms=Config.parse_duration("1d"), + info_uri="https://override.example.com/limits#daily", + can_upgrade=False, ), ) self.assertEqual(self.last_media_upload_limit_exceeded["sent_bytes"], 3000) @@ -3168,13 +3302,16 @@ class MediaUploadLimitsModuleOverrides(unittest.HomeserverTestCase): # User 1 attempts to upload 20000 bytes which is over the weekly limit # This tests that the limits have been sorted as expected channel = self.upload_media(20000, self.tok1) - self.assertEqual(channel.code, 400) + self.assertEqual(channel.code, 403) assert self.last_media_upload_limit_exceeded is not None self.assertEqual(self.last_media_upload_limit_exceeded["user_id"], self.user1) self.assertEqual( self.last_media_upload_limit_exceeded["limit"], MediaUploadLimit( - max_bytes=15000, time_period_ms=Config.parse_duration("1w") + max_bytes=15000, + time_period_ms=Config.parse_duration("1w"), + info_uri="https://override.example.com/limits#weekly", + can_upgrade=False, ), ) self.assertEqual(self.last_media_upload_limit_exceeded["sent_bytes"], 3000) @@ -3197,14 +3334,253 @@ class MediaUploadLimitsModuleOverrides(unittest.HomeserverTestCase): # User 3 uploads 800 bytes which is over the limit channel = self.upload_media(800, self.tok3) - self.assertEqual(channel.code, 400) + self.assertEqual(channel.code, 403) assert self.last_media_upload_limit_exceeded is not None self.assertEqual(self.last_media_upload_limit_exceeded["user_id"], self.user3) self.assertEqual( self.last_media_upload_limit_exceeded["limit"], MediaUploadLimit( - max_bytes=1024, time_period_ms=Config.parse_duration("1d") + max_bytes=1024, + time_period_ms=Config.parse_duration("1d"), + info_uri="https://example.com/limits#daily", + can_upgrade=False, ), ) self.assertEqual(self.last_media_upload_limit_exceeded["sent_bytes"], 500) self.assertEqual(self.last_media_upload_limit_exceeded["attempted_bytes"], 800) + + def test_module_limit_without_info_uri_falls_back(self) -> None: + """A limit returned by a module callback without an info_uri falls back + to the static page served by Synapse when the error is generated.""" + + async def _limits_without_info_uri( + user_id: str, + ) -> list[MediaUploadLimit] | None: + # info_uri is omitted (defaults to None). + return [ + MediaUploadLimit( + time_period_ms=Config.parse_duration("1d"), + max_bytes=123, + ) + ] + + self.hs.get_module_api().register_media_repository_callbacks( + get_media_upload_limits_for_user=_limits_without_info_uri, + ) + + # This is over the limit of 123 bytes, but below the default limit of 1k bytes + channel = self.upload_media(130, self.tok3) + self.assertEqual(channel.code, 403) + self.assertEqual(channel.json_body["errcode"], "M_USER_LIMIT_EXCEEDED") + + # Assert that the upload limit from the module callback was used (identified by + # max_bytes=123) (as opposed to the global default). + assert self.last_media_upload_limit_exceeded is not None + limit = self.last_media_upload_limit_exceeded["limit"] + assert isinstance(limit, MediaUploadLimit) + self.assertEqual(limit.max_bytes, 123) + + # Assert that the info_uri was correctly populated (even though not provided by the module) + expected_info_uri = ( + self.hs.config.server.public_baseurl + + MEDIA_UPLOAD_LIMIT_EXCEEDED_PATH.lstrip("/") + ) + self.assertEqual(channel.json_body["info_uri"], expected_info_uri) + + +class AnimatedThumbnailTestCase(unittest.HomeserverTestCase): + """End-to-end tests for the `animated` query parameter on the local + thumbnail endpoint.""" + + servlets = [ + media.register_servlets, + login.register_servlets, + admin.register_servlets, + ] + + def make_homeserver(self, reactor: MemoryReactor, clock: Clock) -> HomeServer: + self.storage_path = self.mktemp() + self.media_store_path = self.mktemp() + os.mkdir(self.storage_path) + os.mkdir(self.media_store_path) + + config = self.default_config() + config["media_store_path"] = self.media_store_path + config["media_storage_providers"] = [ + { + "module": "synapse.media.storage_provider.FileStorageProviderBackend", + "store_local": True, + "store_synchronous": False, + "store_remote": True, + "config": {"directory": self.storage_path}, + } + ] + + return self.setup_test_homeserver(config=config) + + def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None: + self.store = hs.get_datastores().main + self.repo = hs.get_media_repository() + self.user = self.register_user("user", "pass") + self.tok = self.login("user", "pass") + + self.content_uri = self._upload(self._make_animated_gif(), "image/gif") + + def _make_animated_gif(self) -> bytes: + frames = [ + Image.new("RGB", (64, 64), color) for color in ((255, 0, 0), (0, 0, 255)) + ] + out = io.BytesIO() + frames[0].save( + out, + format="GIF", + save_all=True, + append_images=frames[1:], + duration=100, + loop=0, + ) + return out.getvalue() + + def _make_mpo(self, stale: bool = False) -> bytes: + """Build a two-image MPO: a JPEG holding a stereo pair, not an + animation. If `stale` is set, the trailing image is stripped but still + advertised.""" + frames = [ + Image.new("RGB", (64, 64), color) for color in ((255, 0, 0), (0, 0, 255)) + ] + out = io.BytesIO() + frames[0].save(out, format="MPO", save_all=True, append_images=frames[1:]) + data = out.getvalue() + if not stale: + return data + + with Image.open(io.BytesIO(data)) as image: + primary_size = image.mpinfo[0xB002][0]["Size"] # type: ignore[attr-defined] + return data[:primary_size] + + def _upload(self, data: bytes, content_type: str) -> MXCUri: + return self.get_success( + self.repo.create_or_update_content( + content_type, + "test", + io.BytesIO(data), + len(data), + UserID.from_string(self.user), + ) + ) + + def _thumbnail(self, content_uri: MXCUri, animated: str | None) -> FakeChannel: + params = "?width=32&height=32&method=scale" + if animated is not None: + params += f"&animated={animated}" + channel = self.make_request( + "GET", + f"/_matrix/client/v1/media/thumbnail/{content_uri.server_name}" + f"/{content_uri.media_id}{params}", + shorthand=False, + access_token=self.tok, + ) + self.pump() + self.assertEqual(channel.code, 200, channel.result) + return channel + + def test_default_is_static(self) -> None: + """Without the `animated` parameter a static thumbnail is returned.""" + channel = self._thumbnail(self.content_uri, animated=None) + result = Image.open(io.BytesIO(channel.result["body"])) + self.assertFalse(getattr(result, "is_animated", False)) + + def test_animated_false_is_static(self) -> None: + channel = self._thumbnail(self.content_uri, animated="false") + result = Image.open(io.BytesIO(channel.result["body"])) + self.assertFalse(getattr(result, "is_animated", False)) + + def test_animated_true_is_animated(self) -> None: + """With `animated=true` an animated WebP thumbnail is returned.""" + channel = self._thumbnail(self.content_uri, animated="true") + self.assertEqual( + channel.headers.getRawHeaders(b"Content-Type"), [b"image/webp"] + ) + result = Image.open(io.BytesIO(channel.result["body"])) + self.assertEqual(result.format, "WEBP") + self.assertTrue(getattr(result, "is_animated", False)) + self.assertEqual(result.n_frames, 2) + + def test_animated_thumbnail_is_cached(self) -> None: + """Animated thumbnails are generated at upload and served from cache, + not regenerated on each request.""" + # The animated (WebP) thumbnail is stored alongside the static ones at + # upload time. + thumbnails = self.get_success( + self.store.get_local_media_thumbnails(self.content_uri.media_id) + ) + self.assertTrue( + any(info.type == "image/webp" for info in thumbnails), + thumbnails, + ) + + # Two consecutive requests return identical cached bytes. + first = self._thumbnail(self.content_uri, animated="true") + second = self._thumbnail(self.content_uri, animated="true") + self.assertEqual(first.result["body"], second.result["body"]) + + def test_non_animatable_source_falls_back_to_static(self) -> None: + """`animated=true` on a non-animatable source behaves as `animated=false`.""" + png_uri = self._upload(SMALL_PNG, "image/png") + channel = self._thumbnail(png_uri, animated="true") + self.assertEqual(channel.headers.getRawHeaders(b"Content-Type"), [b"image/png"]) + result = Image.open(io.BytesIO(channel.result["body"])) + self.assertFalse(getattr(result, "is_animated", False)) + + @parameterized.expand([("intact", False), ("stale_index", True)]) + def test_mpo_is_thumbnailed_as_a_still(self, _name: str, stale: bool) -> None: + """An MPO holds several stills rather than an animation, so it never + gets an animated thumbnail. + + Regression test for https://github.com/element-hq/synapse/issues/20024. + """ + mpo_uri = self._upload(self._make_mpo(stale=stale), "image/jpeg") + + thumbnails = self.get_success( + self.store.get_local_media_thumbnails(mpo_uri.media_id) + ) + self.assertTrue(thumbnails) + self.assertNotIn("image/webp", [info.type for info in thumbnails]) + + channel = self._thumbnail(mpo_uri, animated="true") + result = Image.open(io.BytesIO(channel.result["body"])) + self.assertFalse(getattr(result, "is_animated", False)) + + @parameterized.expand([("intact", False), ("stale_index", True)]) + @override_config({"dynamic_thumbnails": True}) + def test_dynamic_thumbnails_of_mpo_are_static( + self, _name: str, stale: bool + ) -> None: + """The same holds when the thumbnail is generated on demand.""" + mpo_uri = self._upload(self._make_mpo(stale=stale), "image/jpeg") + + channel = self._thumbnail(mpo_uri, animated="true") + result = Image.open(io.BytesIO(channel.result["body"])) + self.assertFalse(getattr(result, "is_animated", False)) + + @override_config({"dynamic_thumbnails": True}) + def test_dynamic_thumbnails_generates_and_caches_animated(self) -> None: + """With dynamic thumbnails, animated thumbnails are generated on demand + and then cached.""" + content_uri = self._upload(self._make_animated_gif(), "image/gif") + + channel = self._thumbnail(content_uri, animated="true") + self.assertEqual( + channel.headers.getRawHeaders(b"Content-Type"), [b"image/webp"] + ) + result = Image.open(io.BytesIO(channel.result["body"])) + self.assertTrue(getattr(result, "is_animated", False)) + + # The on-demand generated thumbnail is now cached. + thumbnails = self.get_success( + self.store.get_local_media_thumbnails(content_uri.media_id) + ) + self.assertTrue( + any(info.type == "image/webp" for info in thumbnails), + thumbnails, + ) diff --git a/tests/rest/client/test_models.py b/tests/rest/client/test_models.py index f297856830..e5dc922023 100644 --- a/tests/rest/client/test_models.py +++ b/tests/rest/client/test_models.py @@ -23,7 +23,7 @@ from typing import Literal from pydantic import BaseModel, ValidationError -from synapse.types.rest.client import EmailRequestTokenBody +from synapse.types.rest.client import ClientSecretStr, EmailRequestTokenBody class ThreepidMediumEnumTestCase(stdlib_unittest.TestCase): @@ -48,6 +48,44 @@ class ThreepidMediumEnumTestCase(stdlib_unittest.TestCase): self.Model.model_validate({"medium": 123}) +class ClientSecretStrTestCase(stdlib_unittest.TestCase): + class Model(BaseModel): + client_secret: ClientSecretStr + + def test_accepts_valid_client_secrets(self) -> None: + """Secrets consisting entirely of `[0-9a-zA-Z.=_-]` are accepted.""" + for client_secret in ( + "this.is-a_valid=secret", + "foobar", + "a", + "0123456789", + "a" * 255, + ): + with self.subTest(client_secret=client_secret): + model = self.Model.model_validate({"client_secret": client_secret}) + self.assertEqual(model.client_secret, client_secret) + + def test_rejects_client_secrets_with_invalid_characters(self) -> None: + for client_secret in ( + "foo bar", + "secret!", + "café", + # Little bobby tables + "Robert'; DROP TABLE students;--", + ): + with self.subTest(client_secret=client_secret): + with self.assertRaises(ValidationError): + self.Model.model_validate({"client_secret": client_secret}) + + def test_rejects_empty_client_secret(self) -> None: + with self.assertRaises(ValidationError): + self.Model.model_validate({"client_secret": ""}) + + def test_rejects_overlong_client_secret(self) -> None: + with self.assertRaises(ValidationError): + self.Model.model_validate({"client_secret": "a" * 256}) + + class EmailRequestTokenBodyTestCase(stdlib_unittest.TestCase): base_request = { "client_secret": "hunter2", diff --git a/tests/rest/client/test_profile.py b/tests/rest/client/test_profile.py index 023a376ed1..524045c751 100644 --- a/tests/rest/client/test_profile.py +++ b/tests/rest/client/test_profile.py @@ -127,6 +127,37 @@ class ProfileTestCase(unittest.HomeserverTestCase): ) self.assertEqual(channel.code, 400, channel.result) + @unittest.override_config({"enable_set_displayname": False}) + def test_set_displayname_disabled(self) -> None: + """Changing an existing displayname while `enable_set_displayname` is off + should get a 403 with M_FORBIDDEN.""" + channel = self.make_request( + "PUT", + "/profile/%s/displayname" % (self.owner,), + content={"displayname": "test"}, + access_token=self.owner_tok, + ) + self.assertEqual(channel.code, 403, channel.result) + self.assertEqual(channel.json_body["errcode"], Codes.FORBIDDEN) + + res = self._get_displayname() + self.assertEqual(res, "owner") + + @unittest.override_config({"enable_set_displayname": False}) + def test_delete_displayname_disabled(self) -> None: + """Deleting an existing displayname while `enable_set_displayname` is off + should get a 403 with M_FORBIDDEN.""" + channel = self.make_request( + "DELETE", + "/profile/%s/displayname" % (self.owner,), + access_token=self.owner_tok, + ) + self.assertEqual(channel.code, 403, channel.result) + self.assertEqual(channel.json_body["errcode"], Codes.FORBIDDEN) + + res = self._get_displayname() + self.assertEqual(res, "owner") + def test_get_avatar_url(self) -> None: res = self._get_avatar_url() self.assertIsNone(res) @@ -177,6 +208,53 @@ class ProfileTestCase(unittest.HomeserverTestCase): ) self.assertEqual(channel.code, 400, channel.result) + @unittest.override_config({"enable_set_avatar_url": False}) + def test_set_avatar_url_disabled(self) -> None: + """Changing an existing avatar while `enable_set_avatar_url` is off should + get a 403 with M_FORBIDDEN. Setting it for the first time is allowed.""" + channel = self.make_request( + "PUT", + "/profile/%s/avatar_url" % (self.owner,), + content={"avatar_url": "http://my.server/pic.gif"}, + access_token=self.owner_tok, + ) + self.assertEqual(channel.code, 200, channel.result) + + channel = self.make_request( + "PUT", + "/profile/%s/avatar_url" % (self.owner,), + content={"avatar_url": "http://my.server/me.png"}, + access_token=self.owner_tok, + ) + self.assertEqual(channel.code, 403, channel.result) + self.assertEqual(channel.json_body["errcode"], Codes.FORBIDDEN) + + res = self._get_avatar_url() + self.assertEqual(res, "http://my.server/pic.gif") + + @unittest.override_config({"enable_set_avatar_url": False}) + def test_delete_avatar_url_disabled(self) -> None: + """Deleting an existing avatar while `enable_set_avatar_url` is off should + get a 403 with M_FORBIDDEN.""" + channel = self.make_request( + "PUT", + "/profile/%s/avatar_url" % (self.owner,), + content={"avatar_url": "http://my.server/pic.gif"}, + access_token=self.owner_tok, + ) + self.assertEqual(channel.code, 200, channel.result) + + channel = self.make_request( + "DELETE", + "/profile/%s/avatar_url" % (self.owner,), + access_token=self.owner_tok, + ) + self.assertEqual(channel.code, 403, channel.result) + self.assertEqual(channel.json_body["errcode"], Codes.FORBIDDEN) + + res = self._get_avatar_url() + self.assertEqual(res, "http://my.server/pic.gif") + def _get_displayname(self, name: str | None = None) -> str | None: channel = self.make_request( "GET", "/profile/%s/displayname" % (name or self.owner,) @@ -767,6 +845,21 @@ class ProfileTestCase(unittest.HomeserverTestCase): avatar_url = self._get_avatar_url() self.assertEqual(avatar_url, "mxc://test/good") + def test_set_custom_field_never_existed_user(self) -> None: + """Setting a profile field for a user that does not exist should not + conjure up a profile row for them, even for a server admin.""" + self.register_user("admin", "pass", admin=True) + admin_tok = self.login("admin", "pass") + + channel = self.make_request( + "PUT", + "/_matrix/client/v3/profile/@never-existed:test/custom_field", + content={"custom_field": "test"}, + access_token=admin_tok, + ) + self.assertEqual(channel.code, HTTPStatus.NOT_FOUND, channel.result) + self.assertEqual(channel.json_body["errcode"], Codes.NOT_FOUND) + def test_set_custom_field_other(self) -> None: """Setting someone else's profile field should fail""" channel = self.make_request( diff --git a/tests/rest/client/test_push_rule_attrs.py b/tests/rest/client/test_push_rule_attrs.py index 53c36b7a9c..f6bbc7496e 100644 --- a/tests/rest/client/test_push_rule_attrs.py +++ b/tests/rest/client/test_push_rule_attrs.py @@ -3,6 +3,7 @@ # # Copyright 2020 The Matrix.org Foundation C.I.C. # Copyright (C) 2023 New Vector, Ltd +# Copyright (C) 2026 Element Creations Ltd # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License as @@ -20,9 +21,13 @@ # from http import HTTPStatus +import canonicaljson +from parameterized.parameterized import parameterized + import synapse from synapse.api.errors import Codes from synapse.rest.client import login, push_rule, room +from synapse.types import JsonDict from tests.unittest import HomeserverTestCase @@ -508,3 +513,191 @@ class PushRuleAttributesTestCase(HomeserverTestCase): Codes.INVALID_PARAM, channel.json_body["errcode"], ) + + +class PushRuleLimitTestCase(HomeserverTestCase): + """ + Tests for server-configured limits on push rule size. + + See: https://github.com/element-hq/synapse/security/advisories/GHSA-fp53-rw9v-hcf9 + """ + + servlets = [ + synapse.rest.admin.register_servlets_for_client_rest_resource, + room.register_servlets, + login.register_servlets, + push_rule.register_servlets, + ] + hijack_auth = False + + def default_config(self) -> JsonDict: + config = super().default_config() + # Set some small limits for push rule sizes so that + # we can easily test them. + config["push_rules"] = { + "limits": { + # Size limit of each rule in bytes (canonical JSON) + "rule_size": 64, + # Size limit of the rule ID (in bytes) + "rule_id_length": 24, + # Limit on how many rules you can have + "rule_count": 2, + } + } + return config + + @parameterized.expand( + ( + ( + { + "actions": ["notify"], + "conditions": [{"kind": "event_match", "key": "a", "pattern": "a"}], + }, + 58, + True, + ), + ( + { + "actions": ["notify"], + "conditions": [ + {"kind": "event_match", "key": "a", "pattern": "abcdefg"} + ], + }, + 64, + True, + ), + ( + { + "actions": ["notify"], + "conditions": [ + {"kind": "event_match", "key": "a", "pattern": "abcdefgH"} + ], + }, + 65, + False, + ), + ) + ) + def test_limit_on_push_rule_size( + self, body: JsonDict, expected_body_num_bytes: int, expected_allowed: bool + ) -> None: + """ + Tests that the `push_rules.limits.rule_size` applies to the size of the push rule. + """ + self.register_user("alice", "pass") + token = self.login("alice", "pass") + + # Sanity check the test data: the canonical JSON size of the push rule body + # should be exactly as we expect, otherwise our test is void. + body_bytes = canonicaljson.encode_canonical_json(body) + # We exclude the size of the wrapper, as our implementation currently only + # counts the size of the actions and conditions fragments themselves. + self.assertEqual( + len(body_bytes) - len('{"actions":,"conditions":}'.encode("utf-8")), + expected_body_num_bytes, + ) + + channel = self.make_request( + "PUT", + "/pushrules/global/underride/rule1", + body, + access_token=token, + ) + if expected_allowed: + self.assertEqual( + channel.code, + HTTPStatus.OK, + f"Push rule ({body_bytes!r}) within size limit should be accepted: {channel.json_body}", + ) + else: + self.assertEqual( + channel.code, + HTTPStatus.REQUEST_ENTITY_TOO_LARGE, + f"Push rule ({body_bytes!r}) exceeding size limit should be rejected", + ) + self.assertEqual(channel.json_body["errcode"], Codes.UNKNOWN) + + @parameterized.expand( + ( + ( + 18, + True, + ), + ( + 24, + True, + ), + ( + 25, + False, + ), + ) + ) + def test_limit_on_rule_id_length( + self, rule_id_length: int, expected_allowed: bool + ) -> None: + """ + Tests that the `push_rules.limits.rule_id_length` applies to the byte length + of the push rule ID. + """ + self.register_user("alice", "pass") + token = self.login("alice", "pass") + + PREFIX = "global/underride/" + rule_suffix_length = rule_id_length - len(PREFIX) + assert rule_suffix_length >= 1, "can't construct a rule ID that short" + channel = self.make_request( + "PUT", + f"/pushrules/{PREFIX}{rule_suffix_length * 'a'}", + {"conditions": [], "actions": ["notify"]}, + access_token=token, + ) + if expected_allowed: + self.assertEqual( + channel.code, + HTTPStatus.OK, + f"Push rule ID ({rule_id_length} B) within size limit should be accepted: {channel.json_body}", + ) + else: + self.assertEqual( + channel.code, + HTTPStatus.REQUEST_ENTITY_TOO_LARGE, + "Push rule ID ({rule_id_length} B) exceeding size limit should be rejected", + ) + self.assertEqual(channel.json_body["errcode"], Codes.UNKNOWN) + + def test_limit_on_push_rule_count(self) -> None: + """ + Tests that we are allowed to create exactly the number of push rules + specified by `push_rules.limits.rule_count`, but not a single one more. + """ + self.register_user("bob", "pass") + token = self.login("bob", "pass") + + # First 2 rules are allowable + for i in range(2): + channel = self.make_request( + "PUT", + f"/pushrules/global/underride/rule{i}", + {"actions": ["notify"], "conditions": []}, + access_token=token, + ) + self.assertEqual( + channel.code, + HTTPStatus.OK, + f"Push rule within count limit should be allowed: {channel.json_body}", + ) + + # 3rd rule gets denied as it goes over the limit + channel = self.make_request( + "PUT", + "/pushrules/global/underride/rule3", + {"actions": ["notify"], "conditions": []}, + access_token=token, + ) + self.assertEqual( + channel.code, + HTTPStatus.BAD_REQUEST, + "Push rule exceeding count limit should be rejected", + ) + self.assertEqual(channel.json_body["errcode"], Codes.UNKNOWN) diff --git a/tests/rest/client/test_read_marker.py b/tests/rest/client/test_read_marker.py index c8bb0da5e6..ad13d3607e 100644 --- a/tests/rest/client/test_read_marker.py +++ b/tests/rest/client/test_read_marker.py @@ -66,6 +66,19 @@ class ReadMarkerTestCase(unittest.HomeserverTestCase): self.store = self.hs.get_datastores().main self.clock = self.hs.get_clock() + def _get_fully_read_marker(self, room_id: str) -> str | None: + content = self.get_success( + self.store.get_account_data_for_room_and_type( + self.owner, + room_id, + "m.fully_read", + ) + ) + if content is None: + return None + + return content.get("event_id") + def test_send_read_marker(self) -> None: room_id = self.helper.create_room_as(self.owner, tok=self.owner_tok) @@ -98,6 +111,123 @@ class ReadMarkerTestCase(unittest.HomeserverTestCase): ) self.assertEqual(channel.code, 200, channel.result) + def test_send_read_marker_does_not_move_backwards_by_default(self) -> None: + room_id = self.helper.create_room_as(self.owner, tok=self.owner_tok) + + older_event_id = self.helper.send( + room_id=room_id, body="1", tok=self.owner_tok + )["event_id"] + newer_event_id = self.helper.send( + room_id=room_id, body="2", tok=self.owner_tok + )["event_id"] + + channel = self.make_request( + "POST", + f"/rooms/{room_id}/read_markers", + content={"m.fully_read": newer_event_id}, + access_token=self.owner_tok, + ) + self.assertEqual(channel.code, 200, channel.result) + self.assertEqual(self._get_fully_read_marker(room_id), newer_event_id) + + # Expected to be a no-op. + channel = self.make_request( + "POST", + f"/rooms/{room_id}/read_markers", + content={"m.fully_read": older_event_id}, + access_token=self.owner_tok, + ) + self.assertEqual(channel.code, 200, channel.result) + self.assertEqual(self._get_fully_read_marker(room_id), newer_event_id) + + @unittest.override_config({"experimental_features": {"msc4446_enabled": True}}) + def test_send_read_marker_can_move_backwards_with_opt_in(self) -> None: + room_id = self.helper.create_room_as(self.owner, tok=self.owner_tok) + + older_event_id = self.helper.send( + room_id=room_id, body="1", tok=self.owner_tok + )["event_id"] + newer_event_id = self.helper.send( + room_id=room_id, body="2", tok=self.owner_tok + )["event_id"] + + channel = self.make_request( + "POST", + f"/rooms/{room_id}/read_markers", + content={"m.fully_read": newer_event_id}, + access_token=self.owner_tok, + ) + self.assertEqual(channel.code, 200, channel.result) + + channel = self.make_request( + "POST", + f"/rooms/{room_id}/read_markers", + content={"m.fully_read": older_event_id, "com.beeper.allow_backward": True}, + access_token=self.owner_tok, + ) + self.assertEqual(channel.code, 200, channel.result) + self.assertEqual(self._get_fully_read_marker(room_id), older_event_id) + + @unittest.override_config({"experimental_features": {"msc4446_enabled": True}}) + def test_send_read_marker_does_not_move_backwards_with_explicit_opt_out( + self, + ) -> None: + room_id = self.helper.create_room_as(self.owner, tok=self.owner_tok) + + older_event_id = self.helper.send( + room_id=room_id, body="1", tok=self.owner_tok + )["event_id"] + newer_event_id = self.helper.send( + room_id=room_id, body="2", tok=self.owner_tok + )["event_id"] + + channel = self.make_request( + "POST", + f"/rooms/{room_id}/read_markers", + content={"m.fully_read": newer_event_id}, + access_token=self.owner_tok, + ) + self.assertEqual(channel.code, 200, channel.result) + + # Expected to be a no-op. + channel = self.make_request( + "POST", + f"/rooms/{room_id}/read_markers", + content={ + "m.fully_read": older_event_id, + "com.beeper.allow_backward": False, + }, + access_token=self.owner_tok, + ) + self.assertEqual(channel.code, 200, channel.result) + self.assertEqual(self._get_fully_read_marker(room_id), newer_event_id) + + def test_send_read_marker_ignores_opt_in_when_feature_disabled(self) -> None: + room_id = self.helper.create_room_as(self.owner, tok=self.owner_tok) + older_event_id = self.helper.send( + room_id=room_id, body="1", tok=self.owner_tok + )["event_id"] + newer_event_id = self.helper.send( + room_id=room_id, body="2", tok=self.owner_tok + )["event_id"] + + channel = self.make_request( + "POST", + f"/rooms/{room_id}/read_markers", + content={"m.fully_read": newer_event_id}, + access_token=self.owner_tok, + ) + self.assertEqual(channel.code, 200, channel.result) + + channel = self.make_request( + "POST", + f"/rooms/{room_id}/read_markers", + content={"m.fully_read": older_event_id, "com.beeper.allow_backward": True}, + access_token=self.owner_tok, + ) + self.assertEqual(channel.code, 200, channel.result) + self.assertEqual(self._get_fully_read_marker(room_id), newer_event_id) + def test_send_read_marker_missing_previous_event(self) -> None: """ Test moving a read marker from an event that previously existed but was diff --git a/tests/rest/client/test_receipts.py b/tests/rest/client/test_receipts.py index 3a6a869c54..0835eec6de 100644 --- a/tests/rest/client/test_receipts.py +++ b/tests/rest/client/test_receipts.py @@ -24,6 +24,7 @@ from twisted.internet.testing import MemoryReactor import synapse.rest.admin from synapse.api.constants import EduTypes, EventTypes, HistoryVisibility, ReceiptTypes +from synapse.api.errors import Codes from synapse.rest.client import login, receipts, room, sync from synapse.server import HomeServer from synapse.types import JsonDict @@ -44,6 +45,7 @@ class ReceiptsTestCase(unittest.HomeserverTestCase): def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None: self.url = "/sync?since=%s" self.next_batch = "s0" + self.store = hs.get_datastores().main # Register the first user self.user_id = self.register_user("kermit", "monkey") @@ -59,6 +61,19 @@ class ReceiptsTestCase(unittest.HomeserverTestCase): # Join the second user self.helper.join(room=self.room_id, user=self.user2, tok=self.tok2) + def _get_fully_read_marker(self) -> str | None: + content = self.get_success( + self.store.get_account_data_for_room_and_type( + self.user2, + self.room_id, + ReceiptTypes.FULLY_READ, + ) + ) + if content is None: + return None + + return content.get("event_id") + def test_send_receipt(self) -> None: # Send a message. res = self.helper.send(self.room_id, body="hello", tok=self.tok) @@ -258,6 +273,126 @@ class ReceiptsTestCase(unittest.HomeserverTestCase): self.assertEqual(channel.code, HTTPStatus.BAD_REQUEST) self.assertEqual(channel.json_body["errcode"], "M_NOT_JSON", channel.json_body) + def test_fully_read_receipt_does_not_move_backwards_by_default(self) -> None: + older_event_id = self.helper.send(self.room_id, body="1", tok=self.tok)[ + "event_id" + ] + newer_event_id = self.helper.send(self.room_id, body="2", tok=self.tok)[ + "event_id" + ] + + channel = self.make_request( + "POST", + f"/rooms/{self.room_id}/receipt/{ReceiptTypes.FULLY_READ}/{newer_event_id}", + {}, + access_token=self.tok2, + ) + self.assertEqual(channel.code, 200, channel.result) + self.assertEqual(self._get_fully_read_marker(), newer_event_id) + + # Expected to be a no-op. + channel = self.make_request( + "POST", + f"/rooms/{self.room_id}/receipt/{ReceiptTypes.FULLY_READ}/{older_event_id}", + {}, + access_token=self.tok2, + ) + self.assertEqual(channel.code, 200, channel.result) + self.assertEqual(self._get_fully_read_marker(), newer_event_id) + + @unittest.override_config({"experimental_features": {"msc4446_enabled": True}}) + def test_fully_read_receipt_can_move_backwards_with_opt_in(self) -> None: + older_event_id = self.helper.send(self.room_id, body="1", tok=self.tok)[ + "event_id" + ] + newer_event_id = self.helper.send(self.room_id, body="2", tok=self.tok)[ + "event_id" + ] + + channel = self.make_request( + "POST", + f"/rooms/{self.room_id}/receipt/{ReceiptTypes.FULLY_READ}/{newer_event_id}", + {}, + access_token=self.tok2, + ) + self.assertEqual(channel.code, 200, channel.result) + + channel = self.make_request( + "POST", + f"/rooms/{self.room_id}/receipt/{ReceiptTypes.FULLY_READ}/{older_event_id}", + {"com.beeper.allow_backward": True}, + access_token=self.tok2, + ) + self.assertEqual(channel.code, 200, channel.result) + self.assertEqual(self._get_fully_read_marker(), older_event_id) + + @unittest.override_config({"experimental_features": {"msc4446_enabled": True}}) + def test_fully_read_receipt_does_not_move_backwards_with_explicit_opt_out( + self, + ) -> None: + older_event_id = self.helper.send(self.room_id, body="1", tok=self.tok)[ + "event_id" + ] + newer_event_id = self.helper.send(self.room_id, body="2", tok=self.tok)[ + "event_id" + ] + + channel = self.make_request( + "POST", + f"/rooms/{self.room_id}/receipt/{ReceiptTypes.FULLY_READ}/{newer_event_id}", + {}, + access_token=self.tok2, + ) + self.assertEqual(channel.code, 200, channel.result) + + # Expected to be a no-op. + channel = self.make_request( + "POST", + f"/rooms/{self.room_id}/receipt/{ReceiptTypes.FULLY_READ}/{older_event_id}", + {"com.beeper.allow_backward": False}, + access_token=self.tok2, + ) + self.assertEqual(channel.code, 200, channel.result) + self.assertEqual(self._get_fully_read_marker(), newer_event_id) + + @unittest.override_config({"experimental_features": {"msc4446_enabled": True}}) + def test_allow_backward_is_rejected_for_read_receipts(self) -> None: + event_id = self.helper.send(self.room_id, body="1", tok=self.tok)["event_id"] + + channel = self.make_request( + "POST", + f"/rooms/{self.room_id}/receipt/{ReceiptTypes.READ}/{event_id}", + {"com.beeper.allow_backward": True}, + access_token=self.tok2, + ) + self.assertEqual(channel.code, HTTPStatus.BAD_REQUEST, channel.result) + self.assertEqual(channel.json_body["errcode"], Codes.INVALID_PARAM) + + def test_allow_backward_is_ignored_when_feature_disabled(self) -> None: + older_event_id = self.helper.send(self.room_id, body="1", tok=self.tok)[ + "event_id" + ] + newer_event_id = self.helper.send(self.room_id, body="2", tok=self.tok)[ + "event_id" + ] + + channel = self.make_request( + "POST", + f"/rooms/{self.room_id}/receipt/{ReceiptTypes.FULLY_READ}/{newer_event_id}", + {}, + access_token=self.tok2, + ) + self.assertEqual(channel.code, 200, channel.result) + + channel = self.make_request( + "POST", + f"/rooms/{self.room_id}/receipt/{ReceiptTypes.FULLY_READ}/{older_event_id}", + {"com.beeper.allow_backward": True}, + access_token=self.tok2, + ) + self.assertEqual(channel.code, 200, channel.result) + self.assertEqual(self._get_fully_read_marker(), newer_event_id) + def _get_read_receipt(self) -> JsonDict | None: """Syncs and returns the read receipt.""" diff --git a/tests/rest/client/test_redactions.py b/tests/rest/client/test_redactions.py index 997ca5f9ca..4cab3fc8c3 100644 --- a/tests/rest/client/test_redactions.py +++ b/tests/rest/client/test_redactions.py @@ -24,6 +24,7 @@ from parameterized import parameterized from twisted.internet.testing import MemoryReactor from synapse.api.constants import EventTypes, RelationTypes +from synapse.api.errors import Codes from synapse.api.room_versions import RoomVersion, RoomVersions from synapse.rest import admin from synapse.rest.client import login, room, sync @@ -63,7 +64,8 @@ class RedactionsTestCase(HomeserverTestCase): # Create a room self.room_id = self.helper.create_room_as( - self.mod_user_id, tok=self.mod_access_token + self.mod_user_id, + tok=self.mod_access_token, ) # Invite the other user @@ -165,6 +167,80 @@ class RedactionsTestCase(HomeserverTestCase): self.assertEqual(timeline[-3]["unsigned"]["redacted_by"], redaction_id) self.assertEqual(timeline[-3]["content"], {}) + @override_config({"redaction_allowed_period": "1h"}) + def test_can_redact_recent_event(self) -> None: + b = self.helper.send(room_id=self.room_id, tok=self.other_access_token) + self._redact_event( + self.other_access_token, self.room_id, b["event_id"], expect_code=200 + ) + + @override_config({"redaction_allowed_period": "1h"}) + def test_cannot_redact_old_event(self) -> None: + b = self.helper.send(room_id=self.room_id, tok=self.other_access_token) + msg_id = b["event_id"] + + # go past allowed period + self.reactor.advance(60 * 60 + 1) + + body = self._redact_event( + self.other_access_token, self.room_id, msg_id, expect_code=403 + ) + self.assertEqual(body["errcode"], Codes.FORBIDDEN) + + @override_config({"redaction_allowed_period": "1h"}) + def test_edit_redaction_uses_original_timestamp(self) -> None: + b = self.helper.send(room_id=self.room_id, tok=self.other_access_token) + original_id = b["event_id"] + + self.reactor.advance(60 * 60 + 1) + + edit = self.helper.send_event( + self.room_id, + EventTypes.Message, + content={ + "msgtype": "m.text", + "body": "* edited", + "m.new_content": {"msgtype": "m.text", "body": "edited"}, + "m.relates_to": { + "rel_type": RelationTypes.REPLACE, + "event_id": original_id, + }, + }, + tok=self.other_access_token, + ) + + body = self._redact_event( + self.other_access_token, self.room_id, edit["event_id"], expect_code=403 + ) + self.assertEqual(body["errcode"], Codes.FORBIDDEN) + + @override_config({"redaction_allowed_period": "1h"}) + def test_can_redact_old_non_message_event(self) -> None: + # the restriction only applies to m.room.message. other event types + # (a reaction in this test) can still be redacted after the period. + msg = self.helper.send(room_id=self.room_id, tok=self.other_access_token) + reaction = self.helper.send_event( + self.room_id, + "m.reaction", + content={ + "m.relates_to": { + "rel_type": RelationTypes.ANNOTATION, + "event_id": msg["event_id"], + "key": "👍", + } + }, + tok=self.other_access_token, + ) + + self.reactor.advance(60 * 60 + 1) + + self._redact_event( + self.other_access_token, + self.room_id, + reaction["event_id"], + expect_code=200, + ) + def test_redact_nonexistent_event(self) -> None: # control case: an existing event b = self.helper.send(room_id=self.room_id, tok=self.other_access_token) diff --git a/tests/rest/client/test_register.py b/tests/rest/client/test_register.py index a9f3ac2462..67f78ee24f 100644 --- a/tests/rest/client/test_register.py +++ b/tests/rest/client/test_register.py @@ -183,7 +183,7 @@ class RegisterRestServletTestCase(unittest.HomeserverTestCase): self.assertEqual(channel.code, 400, channel.json_body) self.assertEqual( channel.json_body.get("errcode"), - Codes.APPSERVICE_LOGIN_UNSUPPORTED, + "M_APPSERVICE_LOGIN_UNSUPPORTED", channel.json_body, ) @@ -753,10 +753,11 @@ class RegisterRestServletTestCase(unittest.HomeserverTestCase): "POST", b"register/email/requestToken", {"client_secret": "foobar", "email": email, "send_attempt": 1}, - # The endpoint intentionally adds up to 1000ms of jitter to avoid - # leaking whether the email address is already bound to an account. - timeout_ms=3000, + await_result=False, ) + # Note: The endpoint intentionally adds up to 1000ms of jitter to avoid + # leaking whether the email address is bound to an account. + channel.await_result(timeout_ms=1000) self.assertEqual(200, channel.code, channel.result) self.assertIsNotNone(channel.json_body.get("sid")) diff --git a/tests/rest/client/test_reporting.py b/tests/rest/client/test_reporting.py index 96697b96d5..ed33ff1b79 100644 --- a/tests/rest/client/test_reporting.py +++ b/tests/rest/client/test_reporting.py @@ -211,6 +211,24 @@ class ReportRoomTestCase(unittest.HomeserverTestCase): msg=channel.result["body"], ) + @override_config({"rc_reports": {"per_second": 0.5, "burst_count": 1}}) + def test_ratelimit(self) -> None: + """ + Tests that the room report endpoint is rate limited. + """ + data = {"reason": "this makes me sad"} + + self._assert_status(200, data) + self._assert_status(429, data) + + # Add the current user to the ratelimit overrides, allowing them no ratelimiting. + self.get_success( + self.hs.get_datastores().main.set_ratelimit_for_user(self.other_user, 0, 0) + ) + + # Test that the request isn't ratelimited anymore. + self._assert_status(200, data) + @override_config({"experimental_features": {"msc4277_enabled": True}}) def test_room_existence_hidden(self) -> None: """ diff --git a/tests/rest/client/test_room_membership.py b/tests/rest/client/test_room_membership.py new file mode 100644 index 0000000000..265c1f0f49 --- /dev/null +++ b/tests/rest/client/test_room_membership.py @@ -0,0 +1,235 @@ +# +# This file is licensed under the Affero General Public License (AGPL) version 3. +# +# Copyright (C) 2026 Element Creations Ltd +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# See the GNU Affero General Public License for more details: +# . +# + +from http import HTTPStatus +from unittest.mock import AsyncMock, patch + +from twisted.internet.testing import MemoryReactor + +from synapse.api.errors import Codes +from synapse.appservice import ApplicationService, Scopes +from synapse.rest import admin +from synapse.rest.client import login, room, room_membership +from synapse.server import HomeServer +from synapse.types import JsonDict, UserID, create_requester +from synapse.util.clock import Clock + +from tests import unittest +from tests.test_utils import event_injection +from tests.unittest import override_config + +AS_TOKEN = "i_am_an_app_service" +AS_TOKEN_NO_SCOPE = "i_am_an_app_service_without_scope" + + +class AppserviceRoomMembershipRestServletTestCase(unittest.HomeserverTestCase): + servlets = [ + admin.register_servlets_for_client_rest_resource, + login.register_servlets, + room.register_servlets, + room_membership.register_servlets, + ] + + def default_config(self) -> JsonDict: + config = super().default_config() + config["experimental_features"] = { + "msc4502_enabled": True, + # Merge in this order to allow `override_config` to override the flag + **config.get("experimental_features", {}), + } + return config + + def prepare( + self, reactor: MemoryReactor, clock: Clock, homeserver: HomeServer + ) -> None: + self.creator = self.register_user("owner", "pass") + self.creator_tok = self.login("owner", "pass") + self.room_id = self.helper.create_room_as(self.creator, tok=self.creator_tok) + + self.joined_user = self.register_user("joined_user", "pass") + self.joined_user_tok = self.login("joined_user", "pass") + self.helper.join(self.room_id, self.joined_user, tok=self.joined_user_tok) + + self.not_joined_user = self.register_user("not_joined_user", "pass") + self.not_joined_user_tok = self.login("not_joined_user", "pass") + + self.remote_server = "elsewhere.com" + self.remote_user = UserID.from_string(f"@joined_user:{self.remote_server}") + self.get_success( + event_injection.inject_member_event( + self.hs, self.room_id, self.remote_user.to_string(), "join" + ) + ) + self.not_joined_remote_user = UserID.from_string( + f"@not_joined_user:{self.remote_server}" + ) + + self.unknown_server = "unknown.org" + self.unknown_room_id = "!unknown:unknown.org" + + main_store = self.hs.get_datastores().main + main_store.services_cache.append( + ApplicationService( + AS_TOKEN, + id="as_with_scope", + sender=UserID.from_string("@as:test"), + scopes=[Scopes.QUERY_ROOM_MEMBERSHIP], + ) + ) + main_store.services_cache.append( + ApplicationService( + AS_TOKEN_NO_SCOPE, + id="as_without_scope", + sender=UserID.from_string("@as2:test"), + ) + ) + + def _get_joined( + self, room_id: str, params: str, access_token: str | None + ) -> tuple[int, JsonDict]: + channel = self.make_request( + "GET", + f"/_matrix/client/unstable/io.element.msc4502/rooms/{room_id}/is_joined?{params}", + access_token=access_token, + ) + return channel.code, channel.json_body + + def test_invalid_room_id_format(self) -> None: + code, body = self._get_joined( + "not-a-room-id", f"mxid={self.joined_user}", AS_TOKEN + ) + self.assertEqual(code, HTTPStatus.BAD_REQUEST, body) + self.assertEqual(body["errcode"], Codes.INVALID_PARAM) + + def test_both_mxid_and_server_name_given(self) -> None: + code, body = self._get_joined( + self.room_id, + f"mxid={self.joined_user}&server_name={self.hs.hostname}", + AS_TOKEN, + ) + self.assertEqual(code, HTTPStatus.BAD_REQUEST, body) + self.assertEqual(body["errcode"], Codes.MISSING_PARAM) + + def test_neither_mxid_nor_server_name_given(self) -> None: + code, body = self._get_joined(self.room_id, "", AS_TOKEN) + self.assertEqual(code, HTTPStatus.BAD_REQUEST, body) + self.assertEqual(body["errcode"], Codes.MISSING_PARAM) + + def test_invalid_mxid_format(self) -> None: + code, body = self._get_joined(self.room_id, "mxid=not-a-userid", AS_TOKEN) + self.assertEqual(code, HTTPStatus.BAD_REQUEST, body) + self.assertEqual(body["errcode"], Codes.INVALID_PARAM) + + def test_invalid_server_name_format(self) -> None: + code, body = self._get_joined(self.room_id, "server_name=foo_bar", AS_TOKEN) + self.assertEqual(code, HTTPStatus.BAD_REQUEST, body) + self.assertEqual(body["errcode"], Codes.INVALID_PARAM) + + def test_local_user_joined(self) -> None: + code, body = self._get_joined( + self.room_id, f"mxid={self.joined_user}", AS_TOKEN + ) + self.assertEqual(code, HTTPStatus.OK, body) + self.assertEqual(body, {"joined": True}) + + def test_local_user_not_joined(self) -> None: + code, body = self._get_joined( + self.room_id, f"mxid={self.not_joined_user}", AS_TOKEN + ) + self.assertEqual(code, HTTPStatus.OK, body) + self.assertEqual(body, {"joined": False}) + + def test_remote_user_joined(self) -> None: + code, body = self._get_joined( + self.room_id, f"mxid={self.remote_user.to_string()}", AS_TOKEN + ) + self.assertEqual(code, HTTPStatus.OK, body) + self.assertEqual(body, {"joined": True}) + + def test_remote_user_not_joined(self) -> None: + code, body = self._get_joined( + self.room_id, f"mxid={self.not_joined_remote_user.to_string()}", AS_TOKEN + ) + self.assertEqual(code, HTTPStatus.OK, body) + self.assertEqual(body, {"joined": False}) + + def test_local_server_name_joined(self) -> None: + code, body = self._get_joined( + self.room_id, f"server_name={self.hs.hostname}", AS_TOKEN + ) + self.assertEqual(code, HTTPStatus.OK, body) + self.assertEqual(body, {"joined": True}) + + def test_remote_server_name_joined(self) -> None: + code, body = self._get_joined( + self.room_id, f"server_name={self.remote_server}", AS_TOKEN + ) + self.assertEqual(code, HTTPStatus.OK, body) + self.assertEqual(body, {"joined": True}) + + def test_remote_server_name_not_joined(self) -> None: + code, body = self._get_joined( + self.room_id, f"server_name={self.unknown_server}", AS_TOKEN + ) + self.assertEqual(code, HTTPStatus.OK, body) + self.assertEqual(body, {"joined": False}) + + def test_nonexistent_room_returns_false(self) -> None: + code, body = self._get_joined( + self.unknown_room_id, f"server_name={self.unknown_server}", AS_TOKEN + ) + self.assertEqual(code, HTTPStatus.OK, body) + self.assertEqual(body, {"joined": False}) + + def test_no_token_unauthorized(self) -> None: + code, body = self._get_joined(self.room_id, f"mxid={self.joined_user}", None) + self.assertEqual(code, HTTPStatus.UNAUTHORIZED, body) + self.assertEqual(body["errcode"], Codes.MISSING_TOKEN) + + def test_normal_user_token_forbidden(self) -> None: + code, body = self._get_joined( + self.room_id, f"mxid={self.joined_user}", self.creator_tok + ) + self.assertEqual(code, HTTPStatus.FORBIDDEN, body) + self.assertEqual(body["errcode"], Codes.FORBIDDEN) + + def test_same_user_token_forbidden(self) -> None: + code, body = self._get_joined( + self.room_id, f"mxid={self.joined_user}", self.joined_user_tok + ) + self.assertEqual(code, HTTPStatus.FORBIDDEN, body) + self.assertEqual(body["errcode"], Codes.FORBIDDEN) + + def test_user_with_oauth_scope_allowed(self) -> None: + requester = create_requester(self.creator, scope={Scopes.QUERY_ROOM_MEMBERSHIP}) + with patch.object( + self.hs.get_auth(), "get_user_by_req", AsyncMock(return_value=requester) + ): + code, body = self._get_joined( + self.room_id, f"mxid={self.joined_user}", "doesnt-matter" + ) + self.assertEqual(code, HTTPStatus.OK, body) + self.assertEqual(body, {"joined": True}) + + def test_appservice_without_scope_forbidden(self) -> None: + code, body = self._get_joined( + self.room_id, f"mxid={self.joined_user}", AS_TOKEN_NO_SCOPE + ) + self.assertEqual(code, HTTPStatus.FORBIDDEN, body) + self.assertEqual(body["errcode"], Codes.FORBIDDEN) + + @override_config({"experimental_features": {"msc4502_enabled": False}}) + def test_unreachable_when_experimental_flag_disabled(self) -> None: + code, _ = self._get_joined(self.room_id, f"mxid={self.joined_user}", AS_TOKEN) + self.assertEqual(code, HTTPStatus.NOT_FOUND) diff --git a/tests/rest/client/test_rooms.py b/tests/rest/client/test_rooms.py index 7dd83e60f3..67b5a9f351 100644 --- a/tests/rest/client/test_rooms.py +++ b/tests/rest/client/test_rooms.py @@ -26,12 +26,14 @@ import json from http import HTTPStatus from typing import Any, Iterable, Literal -from unittest.mock import AsyncMock, Mock, call, patch +from unittest.mock import AsyncMock, Mock, call, create_autospec, patch from urllib import parse as urlparse from parameterized import param, parameterized +from twisted.internet import defer from twisted.internet.testing import MemoryReactor +from twisted.web.client import Agent import synapse.rest.admin from synapse.api.constants import ( @@ -61,11 +63,13 @@ from synapse.rest.client import ( from synapse.server import HomeServer from synapse.types import JsonDict, JsonMapping, RoomAlias, UserID, create_requester from synapse.util.clock import Clock +from synapse.util.duration import Duration from synapse.util.stringutils import random_string from tests import unittest from tests.http.server._base import make_request_with_cancellation_test from tests.storage.test_stream import PaginationTestCase +from tests.test_utils import FakeResponse from tests.test_utils.event_injection import ( create_event, inject_event, @@ -503,7 +507,8 @@ class RoomPermissionsTestCase(RoomBase): ) ) assert pl_event is not None - self.assertEqual(50, pl_event.content.get("m.call.invite")) + self.assertEqual(50, pl_event.content.get("events", {}).get("m.call.invite")) + self.assertEqual(50, pl_event.content.get("events", {}).get("m.room.name")) private_pl_event = self.get_success( self.store_controllers.state.get_current_state_event( @@ -511,7 +516,9 @@ class RoomPermissionsTestCase(RoomBase): ) ) assert private_pl_event is not None - self.assertEqual(None, private_pl_event.content.get("m.call.invite")) + self.assertEqual( + None, private_pl_event.content.get("events", {}).get("m.call.invite") + ) class RoomStateTestCase(RoomBase): @@ -1956,7 +1963,8 @@ class RoomPowerLevelOverridesTestCase(RoomBase): def test_default_power_levels_with_room_override(self) -> None: """ Create a room, providing power level overrides. - Confirm that the room's power levels reflect the overrides. + When the `power_level_content_override` was provided, it should replace the + default power levels. See https://github.com/matrix-org/matrix-spec/issues/492 - currently we overwrite each key of power_level_content_override @@ -1985,9 +1993,9 @@ class RoomPowerLevelOverridesTestCase(RoomBase): ) def test_power_levels_with_server_override(self) -> None: """ - With a server configured to modify the room-level defaults, - Create a room, without providing any extra power level overrides. - Confirm that the room's power levels reflect the server-level overrides. + With a server configured `default_power_level_content_override`, creating a room + without `power_level_content_override` should result in the server-level overrides + being applied. Similar to https://github.com/matrix-org/matrix-spec/issues/492, we overwrite each key of power_level_content_override completely. @@ -2278,7 +2286,7 @@ class RoomMessageListTestCase(RoomBase): self.room_id = self.helper.create_room_as(self.user_id) def test_topo_token_is_accepted(self) -> None: - token = "t1-0_0_0_0_0_0_0_0_0_0_0_0_0" + token = "t1-0_0_0_0_0_0_0_0_0_0_0_0_0_0" channel = self.make_request( "GET", "/rooms/%s/messages?access_token=x&from=%s" % (self.room_id, token) ) @@ -2289,7 +2297,7 @@ class RoomMessageListTestCase(RoomBase): self.assertTrue("end" in channel.json_body) def test_stream_token_is_accepted_for_fwd_pagianation(self) -> None: - token = "s0_0_0_0_0_0_0_0_0_0_0_0_0" + token = "s0_0_0_0_0_0_0_0_0_0_0_0_0_0" channel = self.make_request( "GET", "/rooms/%s/messages?access_token=x&from=%s" % (self.room_id, token) ) @@ -2533,7 +2541,12 @@ class RoomDelayedEventTestCase(RoomBase): {}, ) self.assertEqual(HTTPStatus.BAD_REQUEST, channel.code, channel.result) - self.assertNotIn("org.matrix.msc4140.errcode", channel.json_body) + # Assert that the standard error response uses a valid errcode. + # The specific errcode is irrelevant for the purpose of this test. + self.assertIsInstance( + channel.json_body.get("errcode"), + str, + ) def test_delayed_event_unsupported_by_default(self) -> None: """Test that sending a delayed event is unsupported with the default config.""" @@ -2545,10 +2558,35 @@ class RoomDelayedEventTestCase(RoomBase): ).encode("ascii"), {"body": "test", "msgtype": "m.text"}, ) - self.assertEqual(HTTPStatus.BAD_REQUEST, channel.code, channel.result) + self.assertEqual(HTTPStatus.FORBIDDEN, channel.code, channel.result) self.assertEqual( - "M_MAX_DELAY_UNSUPPORTED", - channel.json_body.get("org.matrix.msc4140.errcode"), + Codes.FORBIDDEN, + channel.json_body.get("errcode"), + channel.json_body, + ) + + @unittest.override_config( + { + "max_event_delay_duration": "24h", + "experimental_features": { + "msc4140_max_delayed_events_per_user": 0, + }, + } + ) + def test_delayed_event_disabled_by_limit(self) -> None: + """Test that delayed events are disabled by configuring the per-user limit to 0.""" + channel = self.make_request( + "PUT", + ( + "rooms/%s/send/m.room.message/mid1?org.matrix.msc4140.delay=2000" + % self.room_id + ).encode("ascii"), + {"body": "test", "msgtype": "m.text"}, + ) + self.assertEqual(HTTPStatus.FORBIDDEN, channel.code, channel.result) + self.assertEqual( + Codes.FORBIDDEN, + channel.json_body.get("errcode"), channel.json_body, ) @@ -2565,11 +2603,176 @@ class RoomDelayedEventTestCase(RoomBase): ) self.assertEqual(HTTPStatus.BAD_REQUEST, channel.code, channel.result) self.assertEqual( - "M_MAX_DELAY_EXCEEDED", - channel.json_body.get("org.matrix.msc4140.errcode"), + "ORG.MATRIX.MSC4140_DELAY_TOO_LARGE", + channel.json_body.get("errcode"), channel.json_body, ) + @unittest.override_config( + { + "max_event_delay_duration": "24h", + "experimental_features": { + "msc4140_max_delayed_events_per_user": 1, + }, + } + ) + def test_delayed_event_user_limit_reached(self) -> None: + """Test that users cannot have more delayed events scheduled at once than allowed.""" + # Disable rate-limits for this user. We want to specifically test the storage-based limit, not the request limits + self.get_success( + self.hs.get_datastores().main.set_ratelimit_for_user(self.user_id, 0, 0) + ) + + make_delayed_event_request = lambda: self.make_request( + "POST", + ( + "rooms/%s/send/m.room.message?org.matrix.msc4140.delay=15000" + % self.room_id + ).encode("ascii"), + {"body": "test", "msgtype": "m.text"}, + ) + # Send a delayed event to eat up the limit + channel = make_delayed_event_request() + self.assertEqual(HTTPStatus.OK, channel.code, channel.result) + + # Try to send another delayed event (we expect to hit the limit on the max number of delayed events that can be scheduled at once) + channel = make_delayed_event_request() + self.assertEqual(HTTPStatus.TOO_MANY_REQUESTS, channel.code, channel.result) + self.assertEqual( + Codes.LIMIT_EXCEEDED, + channel.json_body["errcode"], + channel.json_body, + ) + # Confirm that the response includes the time remaining until the next of the user's + # delayed events to be sent, at which point another delayed event may be scheduled + # without exceeding the limit + retry_after_headers = channel.headers.getRawHeaders("Retry-After") + assert retry_after_headers + retry_after_sec = int(retry_after_headers[0]) + self.assertGreater(retry_after_sec, 0) + # Confirm that there is only a single value to the Retry-After header, as per RFC9110 + self.assertEqual(1, len(retry_after_headers)) + + # Wait until we're able to retry again (the retry time from the error response) + self.reactor.advance(retry_after_sec) + + # We should be able to send another delayed event again + channel = make_delayed_event_request() + self.assertEqual(HTTPStatus.OK, channel.code, channel.result) + + @unittest.override_config( + { + "max_event_delay_duration": "24h", + "experimental_features": { + "msc4140_max_delayed_events_per_user": 1, + }, + } + ) + def test_delayed_event_processed_user_limit_reached(self) -> None: + """ + Test that delayed events in the midst of being sent still count towards the limit of + how many delayed events a user may have scheduled at once. + """ + send_after = Duration(seconds=1) + make_delayed_event_request = lambda: self.make_request( + "POST", + ( + f"rooms/%s/send/m.room.message?org.matrix.msc4140.delay={send_after.as_millis()}" + % self.room_id + ).encode("ascii"), + {"body": "test", "msgtype": "m.text"}, + ) + channel = make_delayed_event_request() + self.assertEqual(HTTPStatus.OK, channel.code, channel.result) + + # Simulate the server taking a long time to persist delayed events + simulated_send_lag = Duration(seconds=5) + event_creation_handler = self.hs.get_event_creation_handler() + orig_send_fn = event_creation_handler.create_and_send_nonmember_event + + async def slow_send_fn(*args: Any, **kwargs: Any) -> Any: + await self.clock.sleep(simulated_send_lag) + return await orig_send_fn(*args, **kwargs) + + with patch.object(event_creation_handler, orig_send_fn.__name__, slow_send_fn): + self.reactor.advance(send_after.as_secs()) + channel = make_delayed_event_request() + self.assertEqual(HTTPStatus.TOO_MANY_REQUESTS, channel.code, channel.result) + self.assertEqual( + Codes.LIMIT_EXCEEDED, + channel.json_body["errcode"], + channel.json_body, + ) + # Confirm that the response lacks a Retry-After header, because the reason for this limit + # is the server taking an indeterminitely long time to process a delayed event, and the + # server doesn't know how much longer the client should wait before sending more requests + retry_after_headers = channel.headers.getRawHeaders("Retry-After") + assert not retry_after_headers + + # Wait until the delayed event gets persisted + self.reactor.advance(simulated_send_lag.as_secs()) + + # We should be able to send another delayed event again + channel = make_delayed_event_request() + self.assertEqual(HTTPStatus.OK, channel.code, channel.result) + + @unittest.override_config( + { + "max_event_delay_duration": "24h", + "experimental_features": { + "msc4140_max_delayed_events_per_user": 5, + }, + } + ) + def test_delayed_event_user_limit_exceeded(self) -> None: + """ + Test that delayed event limits work properly when + the number of already scheduled events exceeds the configured limit. + + This can be invoked by the server admin lowering the configured limit & restarting the server + while a user has fewer scheduled delayed events than the old limit, but more than the new limit. + """ + send_after: Duration + make_delayed_event_request = lambda: self.make_request( + "POST", + ( + f"rooms/%s/send/m.room.message?org.matrix.msc4140.delay={send_after.as_millis()}" + % self.room_id + ).encode("ascii"), + {"body": f"test (send after {send_after.as_secs()}s)", "msgtype": "m.text"}, + ) + + for i in range(4): + send_after = Duration(seconds=i) + channel = make_delayed_event_request() + self.assertEqual(HTTPStatus.OK, channel.code, channel.result) + + # Simulate restarting the server after having reconfigured the limit + # to be lower than the number of delayed events we just scheduled. + # + # Set the limit > 1 to test not having to wait for _all_ delayed events + # to be sent before being able to schedule a new one. + self.hs.config.server.max_delayed_events_per_user = 2 + + channel = make_delayed_event_request() + self.assertEqual(HTTPStatus.TOO_MANY_REQUESTS, channel.code, channel.result) + self.assertEqual( + Codes.LIMIT_EXCEEDED, + channel.json_body["errcode"], + channel.json_body, + ) + retry_after_header = channel.headers.getRawHeaders("Retry-After") + assert retry_after_header + retry_after_sec = int(retry_after_header[0]) + assert retry_after_sec > 0 + + # Wait until we're able to retry again (the retry time from the error response) + self.reactor.advance(retry_after_sec) + + # We should be able to send another delayed event again + channel = make_delayed_event_request() + self.assertEqual(HTTPStatus.OK, channel.code, channel.result) + @unittest.override_config({"max_event_delay_duration": "24h"}) def test_delayed_event_with_negative_delay(self) -> None: """Test that sending a delayed event fails if its delay is negative.""" @@ -2625,7 +2828,7 @@ class RoomDelayedEventTestCase(RoomBase): """ # Test that new delayed events are correctly ratelimited. - args = ( + make_delayed_event_request = lambda: self.make_request( "POST", ( "rooms/%s/send/m.room.message?org.matrix.msc4140.delay=2000" @@ -2633,9 +2836,9 @@ class RoomDelayedEventTestCase(RoomBase): ).encode("ascii"), {"body": "test", "msgtype": "m.text"}, ) - channel = self.make_request(*args) + channel = make_delayed_event_request() self.assertEqual(HTTPStatus.OK, channel.code, channel.result) - channel = self.make_request(*args) + channel = make_delayed_event_request() self.assertEqual(HTTPStatus.TOO_MANY_REQUESTS, channel.code, channel.result) # Add the current user to the ratelimit overrides, allowing them no ratelimiting. @@ -2644,7 +2847,7 @@ class RoomDelayedEventTestCase(RoomBase): ) # Test that the new delayed events aren't ratelimited anymore. - channel = self.make_request(*args) + channel = make_delayed_event_request() self.assertEqual(HTTPStatus.OK, channel.code, channel.result) @@ -4750,7 +4953,7 @@ class MSC4293RedactOnBanKickTestCase(unittest.FederatingHomeserverTestCase): bad_user = "@remote_bad_user:" + self.OTHER_SERVER_NAME channel = self.make_signed_federation_request( "GET", - f"/_matrix/federation/v1/make_join/{self.room_id}/{bad_user}?ver=10", + f"/_matrix/federation/v1/make_join/{self.room_id}/{bad_user}?ver=11", ) self.assertEqual(channel.code, HTTPStatus.OK, channel.json_body) join_result = channel.json_body @@ -4758,7 +4961,7 @@ class MSC4293RedactOnBanKickTestCase(unittest.FederatingHomeserverTestCase): join_event_dict = join_result["event"] self.add_hashes_and_signatures_from_other_server( join_event_dict, - RoomVersions.V10, + RoomVersions.V11, ) channel = self.make_signed_federation_request( "PUT", @@ -4793,7 +4996,7 @@ class MSC4293RedactOnBanKickTestCase(unittest.FederatingHomeserverTestCase): "prev_events": auth_ids, } ), - room_version=RoomVersions.V10, + room_version=RoomVersions.V11, ) self.get_success( @@ -4844,7 +5047,7 @@ class MSC4293RedactOnBanKickTestCase(unittest.FederatingHomeserverTestCase): "prev_events": auth_ids, } ), - room_version=RoomVersions.V10, + room_version=RoomVersions.V11, ) self.get_success( @@ -4869,7 +5072,7 @@ class MSC4293RedactOnBanKickTestCase(unittest.FederatingHomeserverTestCase): bad_user = "@remote_bad_user:" + self.OTHER_SERVER_NAME channel = self.make_signed_federation_request( "GET", - f"/_matrix/federation/v1/make_join/{self.room_id}/{bad_user}?ver=10", + f"/_matrix/federation/v1/make_join/{self.room_id}/{bad_user}?ver=11", ) self.assertEqual(channel.code, HTTPStatus.OK, channel.json_body) join_result = channel.json_body @@ -4877,7 +5080,7 @@ class MSC4293RedactOnBanKickTestCase(unittest.FederatingHomeserverTestCase): join_event_dict = join_result["event"] self.add_hashes_and_signatures_from_other_server( join_event_dict, - RoomVersions.V10, + RoomVersions.V11, ) channel = self.make_signed_federation_request( "PUT", @@ -4912,7 +5115,7 @@ class MSC4293RedactOnBanKickTestCase(unittest.FederatingHomeserverTestCase): "prev_events": auth_ids, } ), - room_version=RoomVersions.V10, + room_version=RoomVersions.V11, ) self.get_success( @@ -4953,7 +5156,7 @@ class MSC4293RedactOnBanKickTestCase(unittest.FederatingHomeserverTestCase): # user should be able to join again channel = self.make_signed_federation_request( "GET", - f"/_matrix/federation/v1/make_join/{self.room_id}/{bad_user}?ver=10", + f"/_matrix/federation/v1/make_join/{self.room_id}/{bad_user}?ver=11", ) self.assertEqual(channel.code, HTTPStatus.OK, channel.json_body) join_result = channel.json_body @@ -5001,7 +5204,7 @@ class MSC4293RedactOnBanKickTestCase(unittest.FederatingHomeserverTestCase): "prev_events": auth_ids, } ), - room_version=RoomVersions.V10, + room_version=RoomVersions.V11, ) self.get_success( @@ -5115,7 +5318,7 @@ class MSC4293RedactOnBanKickTestCase(unittest.FederatingHomeserverTestCase): bad_user = "@remote_bad_user:" + self.OTHER_SERVER_NAME channel = self.make_signed_federation_request( "GET", - f"/_matrix/federation/v1/make_join/{self.room_id}/{bad_user}?ver=10", + f"/_matrix/federation/v1/make_join/{self.room_id}/{bad_user}?ver=11", ) self.assertEqual(channel.code, HTTPStatus.OK, channel.json_body) join_result = channel.json_body @@ -5123,7 +5326,7 @@ class MSC4293RedactOnBanKickTestCase(unittest.FederatingHomeserverTestCase): join_event_dict = join_result["event"] self.add_hashes_and_signatures_from_other_server( join_event_dict, - RoomVersions.V10, + RoomVersions.V11, ) channel = self.make_signed_federation_request( "PUT", @@ -5158,7 +5361,7 @@ class MSC4293RedactOnBanKickTestCase(unittest.FederatingHomeserverTestCase): "prev_events": auth_ids, } ), - room_version=RoomVersions.V10, + room_version=RoomVersions.V11, ) self.get_success( @@ -5209,7 +5412,7 @@ class MSC4293RedactOnBanKickTestCase(unittest.FederatingHomeserverTestCase): "prev_events": auth_ids, } ), - room_version=RoomVersions.V10, + room_version=RoomVersions.V11, ) self.get_success( @@ -5231,7 +5434,7 @@ class MSC4293RedactOnBanKickTestCase(unittest.FederatingHomeserverTestCase): bad_user = "@remote_bad_user:" + self.OTHER_SERVER_NAME channel = self.make_signed_federation_request( "GET", - f"/_matrix/federation/v1/make_join/{self.room_id}/{bad_user}?ver=10", + f"/_matrix/federation/v1/make_join/{self.room_id}/{bad_user}?ver=11", ) self.assertEqual(channel.code, HTTPStatus.OK, channel.json_body) join_result = channel.json_body @@ -5239,7 +5442,7 @@ class MSC4293RedactOnBanKickTestCase(unittest.FederatingHomeserverTestCase): join_event_dict = join_result["event"] self.add_hashes_and_signatures_from_other_server( join_event_dict, - RoomVersions.V10, + RoomVersions.V11, ) channel = self.make_signed_federation_request( "PUT", @@ -5274,7 +5477,7 @@ class MSC4293RedactOnBanKickTestCase(unittest.FederatingHomeserverTestCase): "prev_events": auth_ids, } ), - room_version=RoomVersions.V10, + room_version=RoomVersions.V11, ) self.get_success( @@ -5310,7 +5513,7 @@ class MSC4293RedactOnBanKickTestCase(unittest.FederatingHomeserverTestCase): # user re-joins after kick channel = self.make_signed_federation_request( "GET", - f"/_matrix/federation/v1/make_join/{self.room_id}/{bad_user}?ver=10", + f"/_matrix/federation/v1/make_join/{self.room_id}/{bad_user}?ver=11", ) self.assertEqual(channel.code, HTTPStatus.OK, channel.json_body) join_result = channel.json_body @@ -5318,7 +5521,7 @@ class MSC4293RedactOnBanKickTestCase(unittest.FederatingHomeserverTestCase): join_event_dict = join_result["event"] self.add_hashes_and_signatures_from_other_server( join_event_dict, - RoomVersions.V10, + RoomVersions.V11, ) channel = self.make_signed_federation_request( "PUT", @@ -5358,7 +5561,7 @@ class MSC4293RedactOnBanKickTestCase(unittest.FederatingHomeserverTestCase): "prev_events": auth_ids, } ), - room_version=RoomVersions.V10, + room_version=RoomVersions.V11, ) self.get_success( @@ -5598,3 +5801,117 @@ class MSC4293RedactOnBanKickTestCase(unittest.FederatingHomeserverTestCase): expect_redaction=True, reason="being disruptive", ) + + +class CreateRoomRemoteInviteTestCase(unittest.FederatingHomeserverTestCase): + """ + Tests error propagation from remote invites during /createRoom. + + Regression test for https://github.com/element-hq/synapse/security/advisories/GHSA-95fh-hv8c-chvq. + """ + + servlets = [ + room.register_servlets, + login.register_servlets, + register.register_servlets, + admin.register_servlets, + ] + + hijack_auth = False + + def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None: + self.user_id = self.register_user("creator", "test") + self.token = self.login("creator", "test") + + def _mock_remote_invite_http_error( + self, + status: int, + error_body: JsonDict, + ) -> None: + """ + Make the remote homeserver reply to its `/invite` endpoint with an error. + + Args: + status: the HTTP status to return + error_body: the JSON error body to return + """ + federation_http_client = self.hs.get_federation_http_client() + + fake_agent = create_autospec(Agent, spec_set=True) + + def request( + method: bytes, + uri: bytes, + headers: object = None, + bodyProducer: object = None, + ) -> "defer.Deferred": + # For our test, we don't expect any other outbound request + assert b"/invite/" in uri, f"unexpected outbound request to {uri!r}" + return defer.succeed( + FakeResponse.json( + code=status, + payload=error_body, + ) + ) + + fake_agent.request.side_effect = request + federation_http_client.agent = fake_agent + + @parameterized.expand( + ( + ( + HTTPStatus.IM_A_TEAPOT, + { + "errcode": "M_FORBIDDEN", + "error": "You can't invite this user", + }, + HTTPStatus.IM_A_TEAPOT, + { + "errcode": "M_FORBIDDEN", + "error": "You can't invite this user", + }, + ), + # This case is https://github.com/element-hq/synapse/security/advisories/GHSA-95fh-hv8c-chvq + # The error is rewritten for safety. + ( + HTTPStatus.UNAUTHORIZED, + {"errcode": "M_UNKNOWN_TOKEN", "error": "unknown token"}, + HTTPStatus.BAD_REQUEST, + { + "errcode": "M_UNKNOWN", + "error": "unknown token", + }, + ), + ) + ) + def test_remote_invite_bubbles_errors( + self, + policy_server_error_status: HTTPStatus, + policy_server_error_body: JsonDict, + expected_client_facing_error_status: HTTPStatus, + expected_client_facing_error_body: JsonDict, + ) -> None: + """ + Test that, when creating a room involving a remote invite, + when the remote homeserver returns an error, we bubble it + to the client carefully. + + Regression test for https://github.com/element-hq/synapse/security/advisories/GHSA-95fh-hv8c-chvq + """ + # Mock the remote homeserver (at the HTTP level) to return the configured error + self._mock_remote_invite_http_error( + policy_server_error_status, + policy_server_error_body, + ) + + channel = self.make_request( + "POST", + "/createRoom", + {"invite": ["@alice:" + self.OTHER_SERVER_NAME]}, + access_token=self.token, + ) + + self.assertEqual( + channel.code, expected_client_facing_error_status, channel.result + ) + self.assertEqual(channel.json_body, expected_client_facing_error_body) diff --git a/tests/rest/client/test_sendtodevice.py b/tests/rest/client/test_sendtodevice.py index c06c7312f2..e9b607cf23 100644 --- a/tests/rest/client/test_sendtodevice.py +++ b/tests/rest/client/test_sendtodevice.py @@ -489,6 +489,60 @@ class SendToDeviceTestCase(HomeserverTestCase): }, ) + def test_remote_spoofed_sender(self) -> None: + """ + Tests that a to-device message whose sender domain does not match the origin + server is dropped. + """ + user2 = self.register_user("u2", "pass") + user2_tok = self.login("u2", "pass", "d2") + + federation_registry = self.hs.get_federation_registry() + + # Send a spoofed to-device message EDU + self.get_success( + federation_registry.on_edu( + EduTypes.DIRECT_TO_DEVICE, + "example.org", + { + "sender": "@user:not.the.same.example.org", + "type": "org.example.test", + "messages": {user2: {"d2": {"foo": "bar"}}}, + "message_id": "1", + }, + ) + ) + # Also send a valid one as a sentinel, to make sure our test setup + # is working as we'd expect + self.get_success( + federation_registry.on_edu( + EduTypes.DIRECT_TO_DEVICE, + "example.org", + { + "sender": "@user:example.org", + "type": "org.example.test", + "messages": {user2: {"d2": {"hiss": "meow"}}}, + "message_id": "1", + }, + ) + ) + + # Then do a /sync and be sure it didn't come down + channel = self.make_request("GET", "/sync", access_token=user2_tok) + self.assertEqual(channel.code, 200, channel.result) + messages = channel.json_body.get("to_device", {}).get("events", []) + self.assertEqual( + messages, + [ + # Only the sentinel (valid) to-device message came down, not the spoofed one + { + "content": {"hiss": "meow"}, + "sender": "@user:example.org", + "type": "org.example.test", + } + ], + ) + def test_limited_sync(self) -> None: """If a limited sync for to-devices happens the next /sync should respond immediately.""" diff --git a/tests/rest/client/test_sync.py b/tests/rest/client/test_sync.py index 74a8678ae9..039aea4d78 100644 --- a/tests/rest/client/test_sync.py +++ b/tests/rest/client/test_sync.py @@ -33,6 +33,7 @@ from synapse.api.constants import ( ReceiptTypes, RelationTypes, ) +from synapse.rest.admin.experimental_features import ExperimentalFeature from synapse.rest.client import devices, knock, login, read_marker, receipts, room, sync from synapse.server import HomeServer from synapse.types import JsonDict @@ -1272,3 +1273,117 @@ class SyncCancellationTestCase(unittest.HomeserverTestCase): ) self.assertEqual(200, channel.code, msg=channel.result["body"]) + + +class SyncStateAfterArchivedRoomTestCase(unittest.HomeserverTestCase): + """Tests MSC4222 `state_after` behaviour for rooms the syncing user has + left (i.e. rooms in the `leave` section of the sync response).""" + + servlets = [ + synapse.rest.admin.register_servlets, + room.register_servlets, + login.register_servlets, + sync.register_servlets, + ] + + def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None: + self.store = hs.get_datastores().main + + def test_archived_room_state_after_not_newer_than_leave(self) -> None: + """`state_after` for a left room must be the state at the end of that + room's timeline, i.e. at the user's leave point — never state from + after the leave. + + Scenario: with lazy-loading of members and `use_state_after` enabled, + Alice does an incremental sync covering the window in which Bob sent a + message and Alice then left. Bob changed his per-room displayname + *after* Alice's leave; that post-leave membership event must NOT + appear in Alice's `state_after` for the left room. + """ + alice = self.register_user("alice", "password") + alice_tok = self.login("alice", "password") + bob = self.register_user("bob", "password") + bob_tok = self.login("bob", "password") + + # Opt Alice in to MSC4222. + self.get_success( + self.store.set_features_for_user(alice, {ExperimentalFeature.MSC4222: True}) + ) + + # Name the room to avoid heroes: those come from the *current* + # summary — a separate leak path from the one under test. + room_id = self.helper.create_room_as( + alice, tok=alice_tok, extra_content={"name": "Some room name"} + ) + self.helper.join(room_id, bob, tok=bob_tok) + + # Bob's membership as it will stand at Alice's leave point. + channel = self.make_request( + "GET", + f"/_matrix/client/v3/rooms/{room_id}/state/m.room.member/{bob}?format=event", + access_token=alice_tok, + ) + self.assertEqual(channel.code, 200, channel.result) + bob_member_event_id_at_leave = channel.json_body["event_id"] + + # Lazy-load members; `include_redundant_members` bypasses the members + # cache so Bob's membership appears in the incremental sync below. + sync_filter = json.dumps( + { + "room": { + "state": { + "lazy_load_members": True, + "include_redundant_members": True, + }, + } + } + ) + sync_url = f"/sync?filter={sync_filter}&org.matrix.msc4222.use_state_after=true" + + # Initial sync. + channel = self.make_request("GET", sync_url, access_token=alice_tok) + self.assertEqual(channel.code, 200, channel.result) + since = channel.json_body["next_batch"] + + # Bob becomes a timeline sender in the next sync window. + self.helper.send(room_id, body="hello", tok=bob_tok) + + # Alice leaves the room. + self.helper.leave(room_id, alice, tok=alice_tok) + + # Bob's membership changes AFTER Alice's leave. + post_leave_member_event = self.helper.send_state( + room_id, + EventTypes.Member, + {"membership": "join", "displayname": "bob-post-leave"}, + tok=bob_tok, + state_key=bob, + ) + post_leave_member_event_id = post_leave_member_event["event_id"] + + # Incremental sync: the room is in the `leave` section. + channel = self.make_request( + "GET", f"{sync_url}&since={since}", access_token=alice_tok + ) + self.assertEqual(channel.code, 200, channel.result) + + left_room = channel.json_body["rooms"]["leave"][room_id] + state_after_events = left_room["org.matrix.msc4222.state_after"]["events"] + + # Post-leave state must not appear in `state_after`. + self.assertNotIn( + post_leave_member_event_id, + [e["event_id"] for e in state_after_events], + f"state_after contains state from after the user's leave: " + f"{state_after_events}", + ) + + # Bob's membership must be the one at the leave point. + self.assertEqual( + [ + e["event_id"] + for e in state_after_events + if e["type"] == EventTypes.Member and e["state_key"] == bob + ], + [bob_member_event_id_at_leave], + ) diff --git a/tests/rest/client/test_versions.py b/tests/rest/client/test_versions.py index d656098469..1ed6bb145b 100644 --- a/tests/rest/client/test_versions.py +++ b/tests/rest/client/test_versions.py @@ -142,6 +142,28 @@ class VersionsTestCase(unittest.HomeserverTestCase): channel.json_body, ) + def test_msc4446_false_by_default(self) -> None: + channel = self.make_request("GET", "/_matrix/client/versions") + self.assertEqual(channel.code, 200, channel.result) + self.assertFalse(channel.json_body["unstable_features"]["com.beeper.msc4446"]) + + @unittest.override_config({"experimental_features": {"msc4446_enabled": True}}) + def test_msc4446_true_if_enabled(self) -> None: + channel = self.make_request("GET", "/_matrix/client/versions") + self.assertEqual(channel.code, 200, channel.result) + self.assertTrue(channel.json_body["unstable_features"]["com.beeper.msc4446"]) + + def test_msc4502_false_by_default(self) -> None: + channel = self.make_request("GET", "/_matrix/client/versions") + self.assertEqual(channel.code, 200, channel.result) + self.assertFalse(channel.json_body["unstable_features"]["io.element.msc4502"]) + + @unittest.override_config({"experimental_features": {"msc4502_enabled": True}}) + def test_msc4502_true_if_enabled(self) -> None: + channel = self.make_request("GET", "/_matrix/client/versions") + self.assertEqual(channel.code, 200, channel.result) + self.assertTrue(channel.json_body["unstable_features"]["io.element.msc4502"]) + def _sanity_check_versions_response(self, versions_response: JsonDict) -> None: """ Make sure this looks like a `/_matrix/client/versions` response diff --git a/tests/rest/synapse/mas/test_users.py b/tests/rest/synapse/mas/test_users.py index 6f44761bb8..9804f0195d 100644 --- a/tests/rest/synapse/mas/test_users.py +++ b/tests/rest/synapse/mas/test_users.py @@ -17,6 +17,7 @@ from parameterized import parameterized from twisted.internet.testing import MemoryReactor +from synapse.api.constants import ProfileFields from synapse.api.errors import StoreError from synapse.appservice import ApplicationService from synapse.server import HomeServer @@ -54,9 +55,10 @@ class MasQueryUserResource(BaseTestCase): ) ) self.get_success( - store.set_profile_avatar_url( + store.set_profile_field( user_id=alice, - new_avatar_url="mxc://example.com/avatar", + field_name=ProfileFields.AVATAR_URL, + new_value="mxc://example.com/avatar", ) ) @@ -729,7 +731,13 @@ class MasDeleteUserResource(BaseTestCase): store = self.hs.get_datastores().main # Add custom profile field - self.get_success(store.set_profile_field(alice, "io.element.example", "hello")) + self.get_success( + store.set_profile_field( + user_id=alice, + field_name="io.element.example", + new_value="hello", + ) + ) # Ensure we're testing what we think we are: # check the user has profile data at the start of the test diff --git a/tests/server.py b/tests/server.py index 15a7661c34..5297e0b1ac 100644 --- a/tests/server.py +++ b/tests/server.py @@ -459,7 +459,6 @@ def make_request( await_result: bool = True, custom_headers: Iterable[CustomHeaderType] | None = None, client_ip: str = "127.0.0.1", - timeout_ms: int = 1000, ) -> FakeChannel: """ Make a web request using the given method, path and content, and render it @@ -488,8 +487,6 @@ def make_request( custom_headers: (name, value) pairs to add as request headers client_ip: The IP to use as the requesting IP. Useful for testing ratelimiting. - timeout_ms: if `await_result` is `True`, the amount of time to wait on - the request before timing out. Ignored otherwise. Returns: channel @@ -574,7 +571,7 @@ def make_request( req.requestReceived(method, path, b"1.1") if await_result: - channel.await_result(timeout_ms=timeout_ms) + channel.await_result() return channel @@ -1251,19 +1248,35 @@ def setup_test_homeserver( global PREPPED_SQLITE_DB_CONN if PREPPED_SQLITE_DB_CONN is None: temp_engine = create_engine(database_config) - PREPPED_SQLITE_DB_CONN = LoggingDatabaseConnection( + prepped_conn = LoggingDatabaseConnection( conn=sqlite3.connect(":memory:"), engine=temp_engine, default_txn_name="PREPPED_CONN", server_name=server_name, ) - database = DatabaseConnectionConfig("master", database_config) - config.database.databases = [database] prepare_database( - PREPPED_SQLITE_DB_CONN, create_engine(database_config), config + prepped_conn, + create_engine(database_config), + # We pass `config=None` here so that the template database is prepared the + # same way regardless of which test happens to be the first one to run. + # + # Notably, `prepare_database` refuses to initialise an empty database + # when given a worker config, which would otherwise make any test using + # `homeserver_to_use=GenericWorkerServer` fail when run on its own. + # + # Each test still runs `prepare_database` with its own config against its own + # copy of this template (via `hs.setup()`), so anything config specific (like + # module schemas) is still applied per-test. + config=None, ) + # Only publish the template once it's fully prepared. Previously, this was + # assigned before `prepare_database(...)` ran which meant that if + # `prepare_database(...)` failed, we ended up with an unitialized/partial + # database state and never tried to re-create it for subsequent tests. + PREPPED_SQLITE_DB_CONN = prepped_conn + database_config["_TEST_PREPPED_CONN"] = PREPPED_SQLITE_DB_CONN if db_txn_limit is not None: diff --git a/tests/storage/databases/main/test_cache.py b/tests/storage/databases/main/test_cache.py index 3900a35290..1e7ae7e5a3 100644 --- a/tests/storage/databases/main/test_cache.py +++ b/tests/storage/databases/main/test_cache.py @@ -20,7 +20,10 @@ # from unittest.mock import Mock, call +from signedjson.key import generate_signing_key, get_verify_key + from synapse.storage.database import LoggingTransaction +from synapse.storage.keys import FetchKeyResult from tests.replication._base import BaseMultiWorkerStreamTestCase from tests.unittest import HomeserverTestCase @@ -122,3 +125,51 @@ class CacheInvalidationOverReplicationTestCase(BaseMultiWorkerStreamTestCase): [call(key_list) for key_list in keys_to_invalidate], any_order=True, ) + + def test_server_keys_json_invalidation_replicates(self) -> None: + """`_get_server_keys_json` takes a single argument which is itself a + tuple, and we can't send nested keys over replication: the keys are + JSON-encoded on the sending side and decoded again in + `process_replication_rows`. Check the round trip invalidates the right + cache entry on the worker. + """ + master_invalidate = Mock() + worker_invalidate = Mock() + + self.store._get_server_keys_json.invalidate = master_invalidate + worker = self.make_worker_hs("synapse.app.generic_worker") + worker_ds = worker.get_datastores().main + worker_ds._get_server_keys_json.invalidate = worker_invalidate + + signing_key = generate_signing_key("ver1") + verify_key = get_verify_key(signing_key) + key_id = f"{verify_key.alg}:{verify_key.version}" + + assert self.store._cache_id_gen is not None + initial_token = self.store._cache_id_gen.get_current_token() + + self.get_success( + self.store.store_server_keys_response( + "server1", + from_server="server2", + ts_added_ms=self.clock.time_msec(), + verify_keys={ + key_id: FetchKeyResult( + verify_key=verify_key, valid_until_ts=200_000 + ), + }, + response_json={}, + ) + ) + second_token = self.store._cache_id_gen.get_current_token() + self.assertGreater(second_token, initial_token) + + self.get_success( + worker.get_replication_data_handler().wait_for_stream_position( + "master", "caches", second_token + ) + ) + + expected_key = (("server1", key_id),) + master_invalidate.assert_called_once_with(expected_key) + worker_invalidate.assert_called_once_with(expected_key) diff --git a/tests/storage/databases/main/test_room.py b/tests/storage/databases/main/test_room.py index 4ed775ad76..ef6193da98 100644 --- a/tests/storage/databases/main/test_room.py +++ b/tests/storage/databases/main/test_room.py @@ -20,6 +20,7 @@ # import json +from typing import Optional from twisted.internet.testing import MemoryReactor @@ -45,9 +46,13 @@ class RoomBackgroundUpdateStoreTestCase(HomeserverTestCase): self.user_id = self.register_user("foo", "pass") self.token = self.login("foo", "pass") - def _generate_room(self) -> str: + def _generate_room(self, room_version: Optional[str] = None) -> str: """Create a room and return the room ID.""" - return self.helper.create_room_as(self.user_id, tok=self.token) + return self.helper.create_room_as( + self.user_id, + tok=self.token, + room_version=room_version, + ) def run_background_updates(self, update_name: str) -> None: """Insert and run the background update.""" @@ -70,7 +75,17 @@ class RoomBackgroundUpdateStoreTestCase(HomeserverTestCase): """ # Insert a room without the creator - room_id = self._generate_room() + room_id = self._generate_room( + # Create the room as v10, because the `POPULATE_ROOMS_CREATOR_COLUMN` + # background update assumes that the room `creator` field is available + # in the `m.room.create` event content, which was removed in room + # version 11. That assumption is safe as the background update only + # backfills rooms whose `creator` column is unset, which can only be + # rooms created before Synapse started populating the column at room + # creation time (Synapse 1.43, 2021), all of which predate room version + # 11 (2023). + room_version="10" + ) self.get_success( self.store.db_pool.simple_update( table="rooms", diff --git a/tests/storage/test_appservice.py b/tests/storage/test_appservice.py index 4b9d069d6a..1e097306e4 100644 --- a/tests/storage/test_appservice.py +++ b/tests/storage/test_appservice.py @@ -21,7 +21,7 @@ import json import os import tempfile -from typing import cast +from typing import Any, cast from unittest.mock import AsyncMock, Mock import yaml @@ -29,7 +29,11 @@ import yaml from twisted.internet import defer from twisted.internet.testing import MemoryReactor -from synapse.appservice import ApplicationService, ApplicationServiceState +from synapse.appservice import ( + ApplicationService, + ApplicationServiceState, + Scopes, +) from synapse.config._base import ConfigError from synapse.events import EventBase from synapse.server import HomeServer @@ -479,8 +483,8 @@ class TestTransactionStore(ApplicationServiceTransactionStore, ApplicationServic class ApplicationServiceStoreConfigTestCase(unittest.HomeserverTestCase): - def _write_config(self, suffix: str, **kwargs: str) -> str: - vals = { + def _write_config(self, suffix: str, **kwargs: str | list[str] | None) -> str: + vals: dict[str, Any] = { "id": "id" + suffix, "url": "url" + suffix, "as_token": "as_token" + suffix, @@ -566,3 +570,297 @@ class ApplicationServiceStoreConfigTestCase(unittest.HomeserverTestCase): self.assertIn(f1, str(e)) self.assertIn(f2, str(e)) self.assertIn("as_token", str(e)) + + def test_invalid_scopes_raises(self) -> None: + f = self._write_config( + suffix="1", **{"io.element.msc4502.scopes": "not-a-list"} + ) + + self.hs.config.appservice.app_service_config_files = [f] + self.hs.config.caches.event_cache_size = 1 + + server_name = self.hs.hostname + database = self.hs.get_datastores().databases[0] + with self.assertRaises(ValueError): + ApplicationServiceStore( + database, + make_conn( + db_config=database._database_config, + engine=database.engine, + default_txn_name="test", + server_name=server_name, + ), + self.hs, + ) + + def test_known_scope_works(self) -> None: + f = self._write_config( + suffix="1", + **{"io.element.msc4502.scopes": [Scopes.QUERY_ROOM_MEMBERSHIP.value]}, + ) + + self.hs.config.appservice.app_service_config_files = [f] + self.hs.config.caches.event_cache_size = 1 + + server_name = self.hs.hostname + database = self.hs.get_datastores().databases[0] + ApplicationServiceStore( + database, + make_conn( + db_config=database._database_config, + engine=database.engine, + default_txn_name="test", + server_name=server_name, + ), + self.hs, + ) + + def test_unknown_scope_raises(self) -> None: + f = self._write_config( + suffix="1", **{"io.element.msc4502.scopes": ["does:not:exist"]} + ) + + self.hs.config.appservice.app_service_config_files = [f] + self.hs.config.caches.event_cache_size = 1 + + server_name = self.hs.hostname + database = self.hs.get_datastores().databases[0] + with self.assertRaises(ValueError): + ApplicationServiceStore( + database, + make_conn( + db_config=database._database_config, + engine=database.engine, + default_txn_name="test", + server_name=server_name, + ), + self.hs, + ) + + def test_proxy_prefix_works(self) -> None: + f1 = self._write_config( + suffix="1", + **{ + "io.element.msc4512.proxy_prefix": "rtc/livekit", + "io.element.msc4512.proxy_url": "http://proxy", + }, + ) + + self.hs.config.appservice.app_service_config_files = [f1] + self.hs.config.caches.event_cache_size = 1 + + server_name = self.hs.hostname + database = self.hs.get_datastores().databases[0] + store = ApplicationServiceStore( + database, + make_conn( + db_config=database._database_config, + engine=database.engine, + default_txn_name="test", + server_name=server_name, + ), + self.hs, + ) + (appservice,) = store.get_app_services() + self.assertEqual(appservice.proxy_prefix, "rtc/livekit") + self.assertEqual(appservice.proxy_url, "http://proxy") + + def test_proxy_prefix_requires_proxy_url(self) -> None: + f1 = self._write_config( + suffix="1", + **{"io.element.msc4512.proxy_prefix": "rtc/livekit"}, + ) + + self.hs.config.appservice.app_service_config_files = [f1] + self.hs.config.caches.event_cache_size = 1 + + with self.assertRaises(KeyError): + server_name = self.hs.hostname + database = self.hs.get_datastores().databases[0] + ApplicationServiceStore( + database, + make_conn( + db_config=database._database_config, + engine=database.engine, + default_txn_name="test", + server_name=server_name, + ), + self.hs, + ) + + def test_proxy_url_requires_proxy_prefix(self) -> None: + f1 = self._write_config( + suffix="1", + **{"io.element.msc4512.proxy_url": "http://proxy"}, + ) + + self.hs.config.appservice.app_service_config_files = [f1] + self.hs.config.caches.event_cache_size = 1 + + with self.assertRaises(KeyError): + server_name = self.hs.hostname + database = self.hs.get_datastores().databases[0] + ApplicationServiceStore( + database, + make_conn( + db_config=database._database_config, + engine=database.engine, + default_txn_name="test", + server_name=server_name, + ), + self.hs, + ) + + def test_proxy_prefix_requires_non_empty_proxy_url(self) -> None: + f1 = self._write_config( + suffix="1", + **{ + "io.element.msc4512.proxy_prefix": "rtc/livekit", + "io.element.msc4512.proxy_url": "", + }, + ) + + self.hs.config.appservice.app_service_config_files = [f1] + self.hs.config.caches.event_cache_size = 1 + + with self.assertRaises(ValueError): + server_name = self.hs.hostname + database = self.hs.get_datastores().databases[0] + ApplicationServiceStore( + database, + make_conn( + db_config=database._database_config, + engine=database.engine, + default_txn_name="test", + server_name=server_name, + ), + self.hs, + ) + + def test_proxy_url_requires_non_empty_proxy_prefix(self) -> None: + f1 = self._write_config( + suffix="1", + **{ + "io.element.msc4512.proxy_prefix": "", + "io.element.msc4512.proxy_url": "http://proxy", + }, + ) + + self.hs.config.appservice.app_service_config_files = [f1] + self.hs.config.caches.event_cache_size = 1 + + with self.assertRaises(ValueError): + server_name = self.hs.hostname + database = self.hs.get_datastores().databases[0] + ApplicationServiceStore( + database, + make_conn( + db_config=database._database_config, + engine=database.engine, + default_txn_name="test", + server_name=server_name, + ), + self.hs, + ) + + def test_proxy_prefix_does_not_allow_reserved_values(self) -> None: + f1 = self._write_config( + suffix="1", + **{ + "io.element.msc4512.proxy_prefix": "not/allowed", + "io.element.msc4512.proxy_url": "http://proxy", + }, + ) + + self.hs.config.appservice.app_service_config_files = [f1] + self.hs.config.caches.event_cache_size = 1 + + with self.assertRaises(ValueError): + server_name = self.hs.hostname + database = self.hs.get_datastores().databases[0] + ApplicationServiceStore( + database, + make_conn( + db_config=database._database_config, + engine=database.engine, + default_txn_name="test", + server_name=server_name, + ), + self.hs, + ) + + def test_duplicate_proxy_prefix(self) -> None: + f1 = self._write_config( + suffix="1", + **{ + "io.element.msc4512.proxy_prefix": "rtc/livekit", + "io.element.msc4512.proxy_url": "http://proxy", + }, + ) + f2 = self._write_config( + suffix="2", + **{ + "io.element.msc4512.proxy_prefix": "rtc/livekit", + "io.element.msc4512.proxy_url": "http://proxy2", + }, + ) + + self.hs.config.appservice.app_service_config_files = [f1, f2] + self.hs.config.caches.event_cache_size = 1 + + with self.assertRaises(ConfigError) as cm: + server_name = self.hs.hostname + database = self.hs.get_datastores().databases[0] + ApplicationServiceStore( + database, + make_conn( + db_config=database._database_config, + engine=database.engine, + default_txn_name="test", + server_name=server_name, + ), + self.hs, + ) + + e = cm.exception + self.assertIn(f1, str(e)) + self.assertIn(f2, str(e)) + self.assertIn("io.element.msc4512.proxy_prefix", str(e)) + + def test_overlapping_proxy_prefix(self) -> None: + f1 = self._write_config( + suffix="1", + **{ + "io.element.msc4512.proxy_prefix": "rtc/livekit", + "io.element.msc4512.proxy_url": "http://proxy", + }, + ) + f2 = self._write_config( + suffix="2", + **{ + "io.element.msc4512.proxy_prefix": "rtc/livekit/foobar", + "io.element.msc4512.proxy_url": "http://proxy2", + }, + ) + + self.hs.config.appservice.app_service_config_files = [f1, f2] + self.hs.config.caches.event_cache_size = 1 + + with self.assertRaises(ConfigError) as cm: + server_name = self.hs.hostname + database = self.hs.get_datastores().databases[0] + ApplicationServiceStore( + database, + make_conn( + db_config=database._database_config, + engine=database.engine, + default_txn_name="test", + server_name=server_name, + ), + self.hs, + ) + + e = cm.exception + self.assertIn(f1, str(e)) + self.assertIn(f2, str(e)) + self.assertIn("io.element.msc4512.proxy_prefix", str(e)) diff --git a/tests/storage/test_end_to_end_keys.py b/tests/storage/test_end_to_end_keys.py index 24fdb0bf6d..537393f3fc 100644 --- a/tests/storage/test_end_to_end_keys.py +++ b/tests/storage/test_end_to_end_keys.py @@ -22,6 +22,7 @@ from twisted.internet.testing import MemoryReactor from synapse.server import HomeServer +from synapse.storage.database import LoggingTransaction from synapse.util.clock import Clock from tests.unittest import HomeserverTestCase @@ -118,3 +119,98 @@ class EndToEndKeyStoreTestCase(HomeserverTestCase): self.assertIn("user2", res) self.assertNotIn("device1", res["user2"]) self.assertIn("device2", res["user2"]) + + def test_bg_signatures_migration(self) -> None: + updater = self.hs.get_datastores().main.db_pool.updates + + # drop the constraint so we can insert duplicate signatures + def f(txn: LoggingTransaction) -> None: + txn.execute("DROP INDEX e2e_cross_signing_signatures_idx3") + + self.get_success(self.store.db_pool.runInteraction("", f)) + + # save multiple copies of the same key in the database + for _i in range(2): + self.get_success( + self.store.db_pool.simple_insert( + "e2e_cross_signing_signatures", + { + "user_id": "@alice:example.org", + "key_id": "ed25519:abcdefg", + "target_user_id": "@alice:example.org", + "target_device_id": "hijklmnop", + "signature": "some+signature", + }, + ) + ) + + for _i in range(2): + self.get_success( + self.store.db_pool.simple_insert( + "e2e_cross_signing_signatures", + { + "user_id": "@alice:example.org", + "key_id": "ed25519:hijklmnop", + "target_user_id": "@alice:example.org", + "target_device_id": "abcdefg", + "signature": "some+signature", + }, + ) + ) + + # run the background task to remove duplicates + self.get_success( + self.store.db_pool.simple_insert( + "background_updates", + values={ + "update_name": "e2e_cross_signing_signatures_remove_duplicates", + "progress_json": "{}", + }, + ) + ) + + self.get_success( + updater.run_background_updates(False), + ) + + # re-add the unique index + self.get_success( + self.store.db_pool.simple_insert( + "background_updates", + values={ + "update_name": "e2e_cross_signing_signatures_add_key_id_to_index", + "progress_json": "{}", + }, + ) + ) + + self.get_success( + updater.run_background_updates(False), + ) + + # check that we only have one copy of each key + expected_values = [ + ( + "@alice:example.org", + "ed25519:abcdefg", + "@alice:example.org", + "hijklmnop", + "some+signature", + ), + ( + "@alice:example.org", + "ed25519:hijklmnop", + "@alice:example.org", + "abcdefg", + "some+signature", + ), + ] + + res = self.get_success( + self.store.db_pool.execute( + "", + "SELECT user_id, key_id, target_user_id, target_device_id, signature from e2e_cross_signing_signatures ORDER BY key_id", + ) + ) + self.assertEqual(len(res), len(expected_values)) + self.assertEqual(res, expected_values) diff --git a/tests/storage/test_event_federation.py b/tests/storage/test_event_federation.py index 59e914ca8b..55be7265bc 100644 --- a/tests/storage/test_event_federation.py +++ b/tests/storage/test_event_federation.py @@ -19,6 +19,7 @@ # import datetime +import random from typing import ( Collection, Iterable, @@ -27,6 +28,7 @@ from typing import ( TypeVar, cast, ) +from unittest import mock import attr from parameterized import parameterized @@ -40,18 +42,22 @@ from synapse.api.room_versions import ( RoomVersion, ) from synapse.events import EventBase +from synapse.events.py_protocol import MSC4242Event, supports_msc4242_state_dag +from synapse.events.snapshot import EventContext from synapse.rest import admin from synapse.rest.client import login, room from synapse.server import HomeServer from synapse.storage.database import LoggingTransaction from synapse.storage.types import Cursor from synapse.synapse_rust.events import EventInternalMetadata +from synapse.synapse_rust.room_versions import RoomVersions from synapse.types import JsonDict from synapse.util.clock import Clock from synapse.util.json import json_encoder import tests.unittest import tests.utils +from tests.test_utils.event_builders import make_test_event # The silly auth graph we use to test the auth difference algorithm, # where the top are the most recent events. @@ -1414,6 +1420,358 @@ class EventFederationWorkerStoreTestCase(tests.unittest.HomeserverTestCase): # elapsed past the backoff range so there is no events to backoff from. self.assertEqual(event_ids_with_backoff, {}) + def _create_msc4242_room(self) -> tuple[str, str]: + """Create an MSC4242 room with an additional join rules state event. + + Returns: + A tuple of the room ID and the event ID of the most recent state event. + """ + user_id = self.register_user("alice", "test") + tok = self.login("alice", "test") + room_id = self.helper.create_room_as( + room_creator=user_id, + tok=tok, + room_version=RoomVersions.MSC4242v12.identifier, + ) + resp = self.helper.send_state( + room_id, + "m.room.join_rules", + {"join_rule": "knock"}, + tok=tok, + ) + return room_id, resp["event_id"] + + @tests.unittest.override_config( + {"experimental_features": {"msc4242_enabled": True}} + ) + def test_get_state_dag(self) -> None: + """ + Test that MSC4242 state dag rooms can return the complete state dag on request. + """ + room_id, latest = self._create_msc4242_room() + state_dag = self.get_success( + self.store.get_state_dag(room_id, {latest}), + ) + # create <- member <- pl <- join_rules <- his vis <- join_rules + self.assertEquals(len(state_dag), 6) + want_types = [ + EventTypes.Create, + EventTypes.Member, + EventTypes.PowerLevels, + EventTypes.JoinRules, + EventTypes.RoomHistoryVisibility, + EventTypes.JoinRules, + ] + got_types = [] + curr = {latest} + while len(curr) > 0: + event_id = curr.pop() + ev = state_dag[event_id] + got_types.append(ev.type) + curr.update(ev.prev_state_events) + got_types.reverse() # we walked up the graph but want_types is walking down + self.assertEqual(got_types, want_types) + + @tests.unittest.override_config( + {"experimental_features": {"msc4242_enabled": True}} + ) + def test_get_state_dag_at_create_event(self) -> None: + """ + Test that asking for the state DAG at the create event returns just the create event: + we always return the requested forward extremities, and there are no earlier events to + walk back to. + """ + room_id, _ = self._create_msc4242_room() + # Room IDs are the hash of the create event in this room version, so the create event + # ID is derivable from the room ID. + create_event_id = f"${room_id[1:]}" + state_dag = self.get_success( + self.store.get_state_dag(room_id, {create_event_id}), + ) + self.assertEqual(list(state_dag), [create_event_id]) + + +class EventFederationGetMissingEventsStateDAGTestCase( + tests.unittest.HomeserverTestCase +): + servlets = [ + admin.register_servlets, + room.register_servlets, + login.register_servlets, + ] + + def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None: + self.store = hs.get_datastores().main + persist_events = hs.get_datastores().persist_events + assert persist_events is not None + self.persist_events = persist_events + + # Primarily testing to make sure that we sort events + # correctly when there are multiple prev_state_events + # .- C -- D ---. + # A <- B E + # `- R -- W --` + # `-- T -` + graph: dict[str, list[str]] = { + "A": [], + "B": ["A"], + "C": ["B"], + "R": ["B"], + "D": ["C"], + "W": ["R"], + "T": ["R"], + "E": ["W", "D", "T"], + } + self.graph = graph + (self.room_id, self.graph_events) = self._persist_state_dag( + "@test_get_missing_events_state_dag:localhost", graph + ) + + def _persist_state_dag( + self, creator: str, graph: dict[str, list[str]] + ) -> tuple[str, dict[str, MSC4242Event]]: + """Build and persist a state DAG in its own room, as `build_state_dag` returns it.""" + (room_id, graph_events) = build_state_dag(creator, graph) + + def insert(txn: LoggingTransaction) -> None: + mock_context = mock.Mock(spec=EventContext) + mock_context.rejected = False + for ev in graph_events.values(): + # store the event first to satisfy fk constraints + self.persist_events._store_event_txn( + txn, + [(ev, mock_context)], + ) + self.persist_events._store_state_dag_edges( + txn, + ev, + ) + + # satisfy fk constraints + self.get_success( + self.store.store_room(room_id, creator, False, RoomVersions.MSC4242v12) + ) + self.get_success( + self.store.db_pool.runInteraction( + "_store_state_dag_edges", + insert, + ) + ) + return room_id, graph_events + + def _get_missing_events( + self, + latest: list[str], + earliest: list[str], + limit: int, + room_id: str | None = None, + graph_events: dict[str, MSC4242Event] | None = None, + ) -> list[str]: + """Run `get_missing_events_state_dag` in terms of fake event IDs, returning the fake + event IDs which came back. Queries the room built in `prepare` unless told otherwise. + """ + if room_id is None or graph_events is None: + room_id = self.room_id + graph_events = self.graph_events + fake_event_ids = {ev.event_id: fake for fake, ev in graph_events.items()} + got = self.get_success( + self.store.get_missing_events_state_dag( + room_id=room_id, + earliest_event_ids=[graph_events[fake].event_id for fake in earliest], + latest_event_ids=[graph_events[fake].event_id for fake in latest], + limit=limit, + ), + ) + return [fake_event_ids[ev.event_id] for ev in got] + + @parameterized.expand( + [ + (["E"], ["D", "T", "W"], 3), + (["E"], ["D", "T"], 2), + (["E"], ["D"], 1), + (["W", "T", "D"], ["C", "R"], 2), + # breadth first and new entries are added to the end, sorted lexicographically + (["E"], ["D", "T", "W", "C", "R", "B", "A"], 100), + # we should sort the latest values initially + (["E", "C"], ["B", "D", "T", "W"], 4), + (["C", "E"], ["B", "D", "T", "W"], 4), + # dupes are ignored + (["E", "E", "C", "C", "C"], ["B", "D", "T", "W"], 4), + # include latest events in response. W included because reachable from E. + # sort order is based on # hops not processing order of parents + # (which would produce D,T,W,R as E is processed first, then W). + (["W", "E"], ["D", "R", "T", "W"], 4), + ] + ) + @tests.unittest.override_config( + {"experimental_features": {"msc4242_enabled": True}} + ) + def test_get_missing_events_state_dag( + self, latest: list[str], want: list[str], limit: int + ) -> None: + # .- C -- D ---. + # A <- B E + # `- R -- W --` + # `-- T -` + self.assertEquals( + self._get_missing_events(latest=latest, earliest=[], limit=limit), + want, + f"latest={latest} want={want} limit={limit}", + ) + # These expectations are written by hand, so use them to check `walk_state_dag`, which + # the randomly generated graphs below are compared against. + self.assertEquals( + walk_state_dag(self.graph, self.graph_events, latest, [], limit), + want, + f"latest={latest} want={want} limit={limit}", + ) + + @tests.unittest.override_config( + {"experimental_features": {"msc4242_enabled": True}} + ) + def test_get_missing_events_state_dag_limit_truncates(self) -> None: + """Lowering `limit` must truncate the result rather than change it. + + `limit` also bounds how many hops the walk takes, so this checks the two uses agree: + as every extra hop contributes at least one event, a shallower walk cannot miss an + event that a `limit`-sized response should have contained. + """ + full = self._get_missing_events( + latest=["E"], earliest=[], limit=len(self.graph_events) + ) + self.assertEquals(full, ["D", "T", "W", "C", "R", "B", "A"]) + for limit in range(1, len(full) + 1): + self.assertEquals( + self._get_missing_events(latest=["E"], earliest=[], limit=limit), + full[:limit], + f"limit={limit}", + ) + + @tests.unittest.override_config( + {"experimental_features": {"msc4242_enabled": True}} + ) + def test_get_missing_events_state_dag_returns_nothing(self) -> None: + """The cases where there is nothing to walk back to.""" + # Nothing was asked for. + self.assertEquals( + self._get_missing_events(latest=["E"], earliest=[], limit=0), [] + ) + # Every event in the room has already been seen. + self.assertEquals( + self._get_missing_events( + latest=["E"], earliest=list(self.graph), limit=100 + ), + [], + ) + # Every event walked back from has already been seen. + self.assertEquals( + self._get_missing_events(latest=["E", "C"], earliest=["C", "E"], limit=100), + [], + ) + + @tests.unittest.override_config( + {"experimental_features": {"msc4242_enabled": True}} + ) + def test_get_missing_events_state_dag_random_graphs(self) -> None: + """Check the query against a direct implementation of the MSC4242 ordering. + + Uses randomly generated DAGs, as the hand-written cases above can only cover the + shapes we thought to write down. The seed is fixed so that a failure is reproducible + and cannot appear on an unrelated change. + """ + rand = random.Random(42) + # Single letters keep `build_state_dag`'s hash mining cheap. At least 4 events, as + # smaller graphs have too little to order for the walk to be interesting. + names = "ABCDEFGH" + for room_number in range(20): + # Each event picks its prev_state_events from the events before it, which keeps + # the graph acyclic and in causal order. "A" is the create event. + graph: dict[str, list[str]] = {names[0]: []} + for index in range(1, rand.randint(4, len(names))): + graph[names[index]] = sorted( + rand.sample(names[:index], rand.randint(1, index)) + ) + built = list(graph) + (room_id, graph_events) = self._persist_state_dag( + f"@test_random_state_dag_{room_number}:localhost", graph + ) + + for _ in range(3): + # `latest` is sampled with replacement, as callers can repeat an event ID. + latest = [ + rand.choice(built) for _ in range(rand.randint(1, len(built))) + ] + earliest = rand.sample(built, rand.randint(0, len(built) // 2)) + limit = rand.randint(0, len(built) + 1) + message = ( + f"graph={graph} latest={latest} earliest={earliest} limit={limit}" + ) + self.assertEquals( + self._get_missing_events( + latest=latest, + earliest=earliest, + limit=limit, + room_id=room_id, + graph_events=graph_events, + ), + walk_state_dag(graph, graph_events, latest, earliest, limit), + message, + ) + + # Lowering `limit` must truncate the result rather than change it. Take the + # limit from the events actually available rather than from the random one, + # which is often high enough that nothing is dropped. + full = self._get_missing_events( + latest=latest, + earliest=earliest, + limit=len(built), + room_id=room_id, + graph_events=graph_events, + ) + if len(full) > 1: + self.assertEquals( + self._get_missing_events( + latest=latest, + earliest=earliest, + limit=len(full) - 1, + room_id=room_id, + graph_events=graph_events, + ), + full[:-1], + message, + ) + + @tests.unittest.override_config( + {"experimental_features": {"msc4242_enabled": True}} + ) + def test_get_missing_events_state_dag_with_cycle(self) -> None: + """The walk must terminate even if the edges table contains a cycle. + + A cycle cannot be produced by the normal write path, as an event's prev_state_events + are fixed at creation and its event ID is a hash of them. Insert one directly to check + that the query is bounded regardless. + """ + # Claim that A (the create event) has E as a prev_state_event, so walking back from E + # eventually reaches A and then E again. + self.get_success( + self.store.db_pool.simple_insert( + table="msc4242_state_dag_edges", + values={ + "room_id": self.room_id, + "event_id": self.graph_events["A"].event_id, + "prev_state_event_id": self.graph_events["E"].event_id, + }, + desc="insert_state_dag_cycle", + ) + ) + + # Same as the acyclic walk from E, with E itself now reachable via A at the end. Each + # event appears exactly once: the CTE groups by event ID and keeps the fewest hops. + self.assertEquals( + self._get_missing_events(latest=["E"], earliest=[], limit=100), + ["D", "T", "W", "C", "R", "B", "A", "E"], + ) + @attr.s(auto_attribs=True) class FakeEvent: @@ -1431,3 +1789,126 @@ class FakeEvent: def is_state(self) -> bool: return True + + +def walk_state_dag( + graph: dict[str, list[str]], + graph_events: dict[str, MSC4242Event], + latest: list[str], + earliest: list[str], + limit: int, +) -> list[str]: + """Work out what `get_missing_events_state_dag` should return, from the MSC4242 rules. + + Walks back from `latest` via prev_state_events, then orders the events found by how many + hops they are from `latest`, breaking ties on the real event ID. An `earliest` event is + never returned and is never walked through, though its predecessors are still returned if + some other path reaches them. + + Takes the graph and events as `build_state_dag` returns them, in terms of fake event IDs, + and returns the fake event IDs which should come back, in order. Ties break on the real + event ID rather than the fake one because the create event is the one event whose real ID + does not start with its fake ID. + """ + hops: dict[str, int] = {} + # An event `limit` hops away can only be reached by returning an event at every hop + # before it, so a walk deeper than `limit` cannot contribute to the response. + frontier = sorted(set(latest) - set(earliest)) + for hop in range(1, limit + 1): + next_frontier = set() + for fake_event_id in frontier: + for prev_fake_event_id in graph[fake_event_id]: + if prev_fake_event_id in earliest or prev_fake_event_id in hops: + continue + # The first time we see an event is by definition its fewest hops. + hops[prev_fake_event_id] = hop + next_frontier.add(prev_fake_event_id) + frontier = sorted(next_frontier) + return sorted( + hops, + key=lambda fake: (hops[fake], graph_events[fake].event_id), + )[:limit] + + +def build_state_dag( + creator: str, graph: dict[str, list[str]] +) -> tuple[str, dict[str, MSC4242Event]]: + """Build an MSC4242 state DAG. + + Args: + creator: The user ID creating the graph. Should be unique per-test to ensure room IDs change between tests. + graph: A map of fake event ID e.g. "B" to a list of prev_state_events e.g. ["A"]. Graphs must + be created in causal order (earliest events first). The first entry is assumed to be the + create event, so it must have no prev_state_events; every later entry must have at least one. + Returns: + A tuple of the room ID and a map from fake event ID e.g. "B" to real event which you can use .event_id + to extract the real event ID. Guarantees that the real event IDs start with the fake event ID e.g. + the real event for "B" is guarantees to start "$B...." which makes sorting tests much easier to reason about. + The create event is the exception: its ID is fixed by the room ID, so it does not start with its + fake event ID. + """ + graph_events: dict[str, MSC4242Event] = {} # graph ID => built event + create_event = make_test_event( + { + "type": EventTypes.Create, + "state_key": "", + "content": { + "room_version": RoomVersions.MSC4242v12.identifier, + }, + "sender": creator, + "origin_server_ts": 1, + }, + room_version=RoomVersions.MSC4242v12, + ) + # Narrow to an MSC4242 event, so that `prev_state_events` can be used. + assert supports_msc4242_state_dag(create_event) + room_id = create_event.room_id + entropy = 1 + for index, graph_event_id in enumerate(graph): + if index == 0: + # The first entry is the create event, which roots the state DAG. + assert len(graph[graph_event_id]) == 0, ( + f"the first graph event {graph_event_id} is the create event, so it cannot " + "have any prev_state_events" + ) + graph_events[graph_event_id] = create_event + continue + assert len(graph[graph_event_id]) > 0, ( + f"graph event {graph_event_id} has no prev_state_events, but only the create event " + "(the first entry) may be missing them" + ) + # Map previous event IDs to real event IDs. Requires us to build events in causal order. + prev_state_event_ids = [ + graph_events[prev_graph_event_id].event_id + for prev_graph_event_id in graph[graph_event_id] + ] + # Event IDs are hashes of the event, so we cannot choose them directly. Instead, keep + # rebuilding the event with a different `origin_server_ts` until the hash happens to + # start with the fake event ID (e.g. "B" produces "$B..."). Ordering in the state DAG + # breaks ties lexicographically on the real event ID, so having the real IDs sort in + # the same order as the fake names is what makes the expectations in these tests + # readable. + while graph_event_id not in graph_events: + graph_event = make_test_event( + { + "type": "foo", + # All the fake events share a `type`, so the `state_key` is what makes + # each one a distinct piece of state. + "state_key": graph_event_id, + "content": {}, + "sender": creator, + "origin_server_ts": 1 + entropy, + "prev_state_events": prev_state_event_ids, + # Nothing in these tests looks at the message DAG, so just mirror the + # state DAG here. + "prev_events": prev_state_event_ids, + "room_id": room_id, + }, + room_version=RoomVersions.MSC4242v12, + ) + assert supports_msc4242_state_dag(graph_event) + if not graph_event.event_id[1:].startswith(graph_event_id): + entropy += 1 + continue + graph_events[graph_event_id] = graph_event + return (room_id, graph_events) diff --git a/tests/storage/test_id_generators.py b/tests/storage/test_id_generators.py index 9a338607ee..a42247b498 100644 --- a/tests/storage/test_id_generators.py +++ b/tests/storage/test_id_generators.py @@ -19,8 +19,12 @@ # # +from unittest import mock + +from twisted.internet.defer import CancelledError, Deferred, ensureDeferred from twisted.internet.testing import MemoryReactor +from synapse.logging.context import LoggingContext, make_deferred_yieldable from synapse.server import HomeServer from synapse.storage.database import ( DatabasePool, @@ -225,6 +229,90 @@ class MultiWriterIdGeneratorTestCase(MultiWriterIdGeneratorBase): self.assertEqual(id_gen.get_positions(), {"master": 8}) self.assertEqual(id_gen.get_current_token_for_writer("master"), 8) + def test_cancelled_enter_does_not_wedge_position(self) -> None: + """Reproduces presence getting stuck. + + If the `get_next()` async context manager is cancelled while + `__aenter__` is allocating a stream ID, the DB interaction that runs the + sequence has already added the ID to `_unfinished_ids`, but `__aexit__` + is never called (Python only invokes `__aexit__` if `__aenter__` + returned). The abandoned ID is therefore leaked into `_unfinished_ids` + forever, which permanently pins the persisted stream position: new rows + keep getting higher IDs, but `get_current_token()` can never advance past + `leaked_id - 1` until the process restarts. + + This mirrors a `/sync` request being cancelled part-way through + persisting a presence update. `/sync` became `@cancellable` in #19499, + and on a monolith the presence write in `PresenceStore.update_presence` + is awaited inside that cancellable request scope. + """ + # Prefill table with 7 rows written by 'master'; position starts at 7. + self._insert_rows("master", 7) + + id_gen = self._create_id_generator() + self.assertEqual(id_gen.get_current_token_for_writer("master"), 7) + + # We model the cancellation at the seam it actually happens in + # production: `__aenter__` awaits `runInteraction("_load_next_mult_id")`, + # whose transaction runs in a thread pool and so *always* completes - + # allocating stream ID 8 and adding it to `_unfinished_ids` - but the + # awaiting coroutine is handed a `CancelledError` because the enclosing + # `/sync` request was cancelled. We reproduce that by letting the real + # interaction run (applying its side effects) and then failing the + # awaited deferred with `CancelledError`. + cancel_enter: "Deferred[None]" = Deferred() + original_run_interaction = id_gen._db.runInteraction + + async def blocking_run_interaction(desc, func, *args, **kwargs): # type: ignore[no-untyped-def] + result = await original_run_interaction(desc, func, *args, **kwargs) + if desc == "_load_next_mult_id": + # Stream ID 8 is now allocated and recorded in `_unfinished_ids`. + # Deliver the cancellation here, exactly as a cancelled `/sync` + # would land it on this `await`. + await make_deferred_yieldable(cancel_enter) + return result + + async def presence_like_write() -> None: + # Mirrors `PresenceStore.update_presence`: allocate an ID and + # "persist" under the context manager. + with LoggingContext(name="sync", server_name=self.hs.hostname): + async with id_gen.get_next(): + pass + + with mock.patch.object( + id_gen._db, "runInteraction", new=blocking_run_interaction + ): + write = ensureDeferred(presence_like_write()) + + # The write is now blocked inside `__aenter__`, i.e. after stream ID + # 8 has been allocated and added to `_unfinished_ids`. + self.assertNoResult(write) + + # The client goes away and the `/sync` request is cancelled. + cancel_enter.errback(CancelledError()) + + # The cancellation must surface as a `CancelledError`. + self.get_failure(write, CancelledError) + + # The cancelled write never persisted a row for ID 8, so the generator + # must not let that abandoned ID wedge the position. A subsequent + # *successful* write should be able to advance the persisted token. + async def _successful_write() -> None: + async with id_gen.get_next(): + pass + + self.get_success(_successful_write()) + + # On the buggy code the token is still stuck at 7 (ID 8 is leaked in + # `_unfinished_ids`, blocking everything behind it). Once the leak is + # fixed, the token advances to 9: ID 8 was allocated (and abandoned) by + # the cancelled write, so the successful write above takes ID 9. + self.assertEqual( + id_gen.get_current_token_for_writer("master"), + 9, + "presence stream position is wedged by the cancelled allocation", + ) + def test_out_of_order_finish(self) -> None: """Test that IDs persisted out of order are correctly handled""" diff --git a/tests/storage/test_main.py b/tests/storage/test_main.py index 7b5774b8c1..dc98876930 100644 --- a/tests/storage/test_main.py +++ b/tests/storage/test_main.py @@ -18,8 +18,7 @@ # [This file includes modifications made by New Vector Limited] # # - - +from synapse.api.constants import ProfileFields from synapse.types import UserID from tests import unittest @@ -38,7 +37,11 @@ class DataStoreTestCase(unittest.HomeserverTestCase): self.get_success(self.store.register_user(self.user.to_string(), "pass")) self.get_success(self.store.create_profile(self.user)) self.get_success( - self.store.set_profile_displayname(self.user, self.displayname) + self.store.set_profile_field( + user_id=self.user, + field_name=ProfileFields.DISPLAYNAME, + new_value=self.displayname, + ) ) users, total = self.get_success( diff --git a/tests/storage/test_profile.py b/tests/storage/test_profile.py index dbaf298697..0d9abaff47 100644 --- a/tests/storage/test_profile.py +++ b/tests/storage/test_profile.py @@ -19,8 +19,12 @@ # # +from http import HTTPStatus + from twisted.internet.testing import MemoryReactor +from synapse.api.constants import ProfileFields +from synapse.api.errors import StoreError from synapse.server import HomeServer from synapse.storage.database import LoggingTransaction from synapse.storage.engines import PostgresEngine @@ -39,7 +43,13 @@ class ProfileStoreTestCase(unittest.HomeserverTestCase): def test_displayname(self) -> None: self.get_success(self.store.create_profile(self.u_frank)) - self.get_success(self.store.set_profile_displayname(self.u_frank, "Frank")) + self.get_success( + self.store.set_profile_field( + user_id=self.u_frank, + field_name=ProfileFields.DISPLAYNAME, + new_value="Frank", + ) + ) self.assertEqual( "Frank", @@ -47,7 +57,13 @@ class ProfileStoreTestCase(unittest.HomeserverTestCase): ) # test set to None - self.get_success(self.store.set_profile_displayname(self.u_frank, None)) + self.get_success( + self.store.set_profile_field( + user_id=self.u_frank, + field_name=ProfileFields.DISPLAYNAME, + new_value=None, + ) + ) self.assertIsNone( self.get_success(self.store.get_profile_displayname(self.u_frank)) @@ -57,7 +73,11 @@ class ProfileStoreTestCase(unittest.HomeserverTestCase): self.get_success(self.store.create_profile(self.u_frank)) self.get_success( - self.store.set_profile_avatar_url(self.u_frank, "http://my.site/here") + self.store.set_profile_field( + user_id=self.u_frank, + field_name=ProfileFields.AVATAR_URL, + new_value="http://my.site/here", + ) ) self.assertEqual( @@ -66,12 +86,57 @@ class ProfileStoreTestCase(unittest.HomeserverTestCase): ) # test set to None - self.get_success(self.store.set_profile_avatar_url(self.u_frank, None)) + self.get_success( + self.store.set_profile_field( + user_id=self.u_frank, + field_name=ProfileFields.AVATAR_URL, + new_value=None, + ) + ) self.assertIsNone( self.get_success(self.store.get_profile_avatar_url(self.u_frank)) ) + def test_get_profile_field_without_profile(self) -> None: + """ + Getting a custom profile field for a user that has no row in the + `profiles` table at all should raise a 404. + + Regression test (we previously would trigger an unhandled exception). + Can happen for users whose profile was erased upon deactivation. + """ + f = self.get_failure( + self.store.get_profile_field(self.u_frank, "org.example.field"), + StoreError, + ) + self.assertEqual(f.value.code, HTTPStatus.NOT_FOUND) + + def test_set_profile_field_without_profile(self) -> None: + """ + Setting a custom profile field for a user that has no row in the + `profiles` table at all should create the row and store the field. + + Regression test (we previously would trigger an unhandled exception in + the profile size check, and then store the field under a wrong key on + SQLite). Can happen for users whose profile was erased upon + deactivation. + """ + self.get_success( + self.store.set_profile_field( + user_id=self.u_frank, + field_name="org.example.field", + new_value="test", + ) + ) + + self.assertEqual( + "test", + self.get_success( + self.store.get_profile_field(self.u_frank, "org.example.field") + ), + ) + def test_profiles_bg_migration(self) -> None: """ Test background job that copies entries from column user_id to full_user_id, adding diff --git a/tests/storage/test_room_search.py b/tests/storage/test_room_search.py index 2c0ef19e9e..ccfe5b012d 100644 --- a/tests/storage/test_room_search.py +++ b/tests/storage/test_room_search.py @@ -19,6 +19,7 @@ # # +import json from unittest.case import SkipTest from twisted.internet.testing import MemoryReactor @@ -202,6 +203,86 @@ class EventSearchInsertionTest(HomeserverTestCase): ) self.assertCountEqual(values, ["hi", "2"]) + def _rebuild_search_index(self) -> None: + """Simulate a full search-index rebuild: wipe `event_search` so nothing + is indexed, then schedule and run the `event_search` background reindex, + which exercises `_background_reindex_search`. + """ + store = self.hs.get_datastores().main + + self.get_success( + store.db_pool.runInteraction( + "clear_event_search", + lambda txn: txn.execute("DELETE FROM event_search"), + ) + ) + + max_stream_id = store.get_room_max_stream_ordering() + store.db_pool.updates._all_done = False + self.get_success( + store.db_pool.simple_insert( + "background_updates", + { + "update_name": store.EVENT_SEARCH_UPDATE_NAME, + "progress_json": json.dumps( + { + "target_min_stream_id_inclusive": 0, + "max_stream_id_exclusive": max_stream_id + 1, + "rows_inserted": 0, + } + ), + }, + ) + ) + self.wait_for_background_updates() + + def test_reindex_search(self) -> None: + """The `event_search` background reindex must index all searchable event + types: `m.room.message`, `m.room.name` and `m.room.topic`. + """ + store = self.hs.get_datastores().main + + # Create a room and set a searchable topic, name and message through the + # normal client API so the events land in `events`/`event_json`. + self.register_user("alice", "password") + access_token = self.login("alice", "password") + room_id = self.helper.create_room_as("alice", tok=access_token) + + self.helper.send_state( + room_id, + "m.room.topic", + {"topic": "searchable topic keyword"}, + tok=access_token, + ) + self.helper.send_state( + room_id, + "m.room.name", + {"name": "searchable name keyword"}, + tok=access_token, + ) + self.helper.send(room_id, "searchable message keyword", tok=access_token) + + self._rebuild_search_index() + + message_results = self.get_success( + store.search_msgs([room_id], "searchable message keyword", ["content.body"]) + ) + self.assertEqual(message_results["count"], 1, "message was not reindexed") + + name_results = self.get_success( + store.search_msgs([room_id], "searchable name keyword", ["content.name"]) + ) + self.assertEqual(name_results["count"], 1, "name was not reindexed") + + topic_results = self.get_success( + store.search_msgs([room_id], "searchable topic keyword", ["content.topic"]) + ) + self.assertEqual( + topic_results["count"], + 1, + "room topic was not indexed by the search reindex", + ) + class MessageSearchTest(HomeserverTestCase): """ diff --git a/tests/storage/test_state_deletion.py b/tests/storage/test_state_deletion.py index d4079c372e..066d69c93d 100644 --- a/tests/storage/test_state_deletion.py +++ b/tests/storage/test_state_deletion.py @@ -14,12 +14,15 @@ import logging +from collections.abc import Collection +from unittest.mock import patch from twisted.internet.testing import MemoryReactor from synapse.rest import admin from synapse.rest.client import login, room from synapse.server import HomeServer +from synapse.storage.database import LoggingTransaction from synapse.util.clock import Clock from tests.test_utils.event_injection import create_event @@ -48,8 +51,20 @@ class StateDeletionStoreTestCase(HomeserverTestCase): self.purge_events._delete_state_loop_call.stop() self.user_id = self.register_user("test", "password") - tok = self.login("test", "password") - self.room_id = self.helper.create_room_as(self.user_id, tok=tok) + self.tok = self.login("test", "password") + self.room_id = self.helper.create_room_as(self.user_id, tok=self.tok) + + def get_persisting_marker_rows(self) -> list[tuple[int, str, int]]: + """Return the contents of the `state_groups_persisting` table.""" + + return self.get_success( + self.state_deletion_store.db_pool.simple_select_list( + table="state_groups_persisting", + keyvalues=None, + retcols=("state_group", "instance_name", "inserted_ts"), + desc="get_persisting_marker_rows", + ) + ) def check_if_can_be_deleted(self, state_group: int) -> bool: """Check if the state group is pending deletion.""" @@ -107,6 +122,39 @@ class StateDeletionStoreTestCase(HomeserverTestCase): self.get_success(ctx_mgr.__aenter__()) self.get_success(ctx_mgr.__aexit__(Exception, Exception("test"), None)) + def test_retry_send_after_failed_clean_up(self) -> None: + """Test that we can retry sending an event after the clean up of the + `state_groups_persisting` rows failed, as it does when the database goes + away while we're persisting.""" + + fail_clean_up = True + orig_finish_persisting_txn = self.state_deletion_store._finish_persisting_txn + + def _finish_persisting_txn( + txn: LoggingTransaction, state_groups: Collection[int], error: bool + ) -> None: + if fail_clean_up: + raise Exception("Database has gone away") + + orig_finish_persisting_txn(txn, state_groups, error) + + with patch.object( + self.state_deletion_store, + "_finish_persisting_txn", + new=_finish_persisting_txn, + ): + self.helper.send(self.room_id, body="first", tok=self.tok, expect_code=500) + + # The rows are still in place, as the transaction that would have + # removed them was rolled back. + self.assertNotEqual(self.get_persisting_marker_rows(), []) + + # The database has come back, so the retry should now go through. + fail_clean_up = False + self.helper.send(self.room_id, body="second", tok=self.tok) + + self.assertEqual(self.get_persisting_marker_rows(), []) + def test_existing_pending_deletion_is_cleared(self) -> None: """Test that the pending deletion flag gets cleared when the state group gets persisted.""" diff --git a/tests/storage/test_stream.py b/tests/storage/test_stream.py index de127e3971..4ece16e9a0 100644 --- a/tests/storage/test_stream.py +++ b/tests/storage/test_stream.py @@ -1455,6 +1455,7 @@ class GetCurrentStateDeltaMembershipChangesForUserFederationTestCase( auth_chain=[create_event, creator_join_event], partial_state=False, servers_in_room=frozenset(), + state_dag=None, ) ) diff --git a/tests/unittest.py b/tests/unittest.py index 202f7120ef..152fb341cb 100644 --- a/tests/unittest.py +++ b/tests/unittest.py @@ -317,7 +317,11 @@ class TestCase(unittest.TestCase): ) diff_message = f"{first_message}\n{expected_string}\n{actual_string}" - self.fail(f"{diff_message}\n{message}") + extra_message = "" + if message is not None: + extra_message = "f\n{message}" + + self.fail(f"{diff_message}{extra_message}") def DEBUG(target: TV) -> TV: @@ -571,7 +575,6 @@ class HomeserverTestCase(TestCase): await_result: bool = True, custom_headers: Iterable[CustomHeaderType] | None = None, client_ip: str = "127.0.0.1", - timeout_ms: int = 1000, ) -> FakeChannel: """ Create a SynapseRequest at the path using the method and containing the @@ -600,8 +603,6 @@ class HomeserverTestCase(TestCase): client_ip: The IP to use as the requesting IP. Useful for testing ratelimiting. - timeout_ms: if `await_result` is `True`, the amount of time to wait on - the request before timing out. Ignored otherwise. Returns: The FakeChannel object which stores the result of the request. @@ -621,7 +622,6 @@ class HomeserverTestCase(TestCase): await_result, custom_headers, client_ip, - timeout_ms, ) def setup_test_homeserver( diff --git a/tests/util/test_httpresourcetree.py b/tests/util/test_httpresourcetree.py new file mode 100644 index 0000000000..b5dc2e0657 --- /dev/null +++ b/tests/util/test_httpresourcetree.py @@ -0,0 +1,66 @@ +# +# This file is licensed under the Affero General Public License (AGPL) version 3. +# +# Copyright (C) 2026 Element Creations Ltd +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# See the GNU Affero General Public License for more details: +# . +# +import os +from http import HTTPStatus + +from twisted.web.resource import Resource + +import synapse +from synapse.api.errors import Codes +from synapse.api.urls import STATIC_PREFIX +from synapse.http.server import StaticResource + +from tests import unittest + + +class ResourceTreeTestCase(unittest.HomeserverTestCase): + servlets = [] + + def create_resource_dict(self) -> dict[str, Resource]: + """ + Register /_matrix/static for the test. + """ + resources = super().create_resource_dict() + resources[STATIC_PREFIX] = StaticResource( + # as in `synapse/app/homeserver.py` `_configure_named_resource` + os.path.join(os.path.dirname(synapse.__file__), "static") + ) + return resources + + def test_inserted_segment_is_silently_swallowed(self) -> None: + """ + Regression test for https://github.com/element-hq/synapse/security/advisories/GHSA-vh4c-pqh4-w3wq + + The path `/_matrix/INSERTED/static/client/login/style.css` used to resolve to the same + as `/_matrix/static/client/login/style.css`. + """ + PATH_SUFFIX = "/static/client/login/style.css" + correct_channel = self.make_request( + "GET", + f"/_matrix{PATH_SUFFIX}", + shorthand=False, + ) + # The correct path should give a 200 OK static resource + self.assertEqual(correct_channel.code, HTTPStatus.OK, correct_channel.result) + + wrong_channel = self.make_request( + "GET", + f"/_matrix/INSERTED{PATH_SUFFIX}", + shorthand=False, + ) + # This prefixed version of the same path should give a 404 + self.assertEqual(wrong_channel.code, HTTPStatus.NOT_FOUND, wrong_channel.result) + self.assertEqual( + wrong_channel.json_body["errcode"], Codes.UNRECOGNIZED, wrong_channel.result + )